From 23ca10cb6c38e4e3fa208368520e1c5b62b83534 Mon Sep 17 00:00:00 2001 From: gongjy <2474590974@qq.com> Date: Tue, 8 Oct 2024 09:29:39 +0800 Subject: [PATCH] update readme & requirements --- 1-pretrain_vlm.py | 17 ++--------------- 2-sft_vlm.py | 13 ------------- 3-eval_chat.py | 4 ---- README.md | 9 +++++---- README_en.md | 13 +++++++++++-- model/LMConfig.py | 4 ++-- requirements.txt | 1 + 7 files changed, 21 insertions(+), 40 deletions(-) diff --git a/1-pretrain_vlm.py b/1-pretrain_vlm.py index b2ed593..8cec330 100644 --- a/1-pretrain_vlm.py +++ b/1-pretrain_vlm.py @@ -122,23 +122,10 @@ def init_model(lm_config): if k.startswith(unwanted_prefix): state_dict[k[len(unwanted_prefix):]] = state_dict.pop(k) - for k, v in list(state_dict.items()): - if 'mask' in k: - del state_dict[k] - # 加载到模型中 model.load_state_dict(state_dict, strict=False) model = model.to(args.device) - # # 第一阶段冻结llm，只学习vision_proj层 - # for name, param in model.named_parameters(): - # # 如果参数名不包含'vision_proj'，则冻结该参数 - # if 'vision_proj' not in name: - # param.requires_grad_(False) - # else: - # # 确保'vision_proj'中的参数可以更新 - # param.requires_grad_(True) - print(f'模型可学习参数: {count_parameters(model) / 1e6} 百万 = {count_parameters(model) / 1e9} B (Billion)') (vision_model, preprocess) = get_vision_model() @@ -162,7 +149,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser(description="MiniMind-V Pretrain") parser.add_argument("--out_dir", type=str, default="out", help="Output directory") parser.add_argument("--epochs", type=int, default=19, help="Number of epochs") - parser.add_argument("--batch_size", type=int, default=128, help="Batch size") + parser.add_argument("--batch_size", type=int, default=32, help="Batch size") parser.add_argument("--learning_rate", type=float, default=4e-4, help="Learning rate") parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", help="Device to use") @@ -176,7 +163,7 @@ if __name__ == "__main__": parser.add_argument("--accumulation_steps", type=int, default=1, help="Gradient accumulation steps") parser.add_argument("--grad_clip", type=float, default=1.0, help="Gradient clipping threshold") parser.add_argument("--warmup_iters", type=int, default=0, help="Number of warmup iterations") - parser.add_argument("--log_interval", type=int, default=100, help="Logging interval") + parser.add_argument("--log_interval", type=int, default=10, help="Logging interval") parser.add_argument("--save_interval", type=int, default=100, help="Model saving interval") parser.add_argument('--local_rank', type=int, default=-1, help='local rank for distributed training') diff --git a/2-sft_vlm.py b/2-sft_vlm.py index 4df9706..321dfd6 100644 --- a/2-sft_vlm.py +++ b/2-sft_vlm.py @@ -122,23 +122,10 @@ def init_model(lm_config): if k.startswith(unwanted_prefix): state_dict[k[len(unwanted_prefix):]] = state_dict.pop(k) - for k, v in list(state_dict.items()): - if 'mask' in k: - del state_dict[k] - # 加载到模型中 model.load_state_dict(state_dict, strict=False) model = model.to(args.device) - # # 第一阶段冻结llm，只学习vision_proj层 - # for name, param in model.named_parameters(): - # # 如果参数名不包含'vision_proj'，则冻结该参数 - # if 'vision_proj' not in name: - # param.requires_grad_(False) - # else: - # # 确保'vision_proj'中的参数可以更新 - # param.requires_grad_(True) - print(f'模型可学习参数: {count_parameters(model) / 1e6} 百万 = {count_parameters(model) / 1e9} B (Billion)') (vision_model, preprocess) = get_vision_model() diff --git a/3-eval_chat.py b/3-eval_chat.py index b62eed6..b0d8301 100644 --- a/3-eval_chat.py +++ b/3-eval_chat.py @@ -36,10 +36,6 @@ def init_model(lm_config): if k.startswith(unwanted_prefix): state_dict[k[len(unwanted_prefix):]] = state_dict.pop(k) - for k, v in list(state_dict.items()): - if 'mask' in k: - del state_dict[k] - # 加载到模型中 model.load_state_dict(state_dict, strict=False) else: diff --git a/README.md b/README.md index 920c4ed..a7b71b4 100644 --- a/README.md +++ b/README.md @@ -110,8 +110,8 @@ CPU: Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz ```bash pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple ``` - - ```python + + ```text # 测试torch是否可用cuda import torch print(torch.cuda.is_available()) @@ -152,7 +152,7 @@ CPU: Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple ``` - ```python + ```text # 测试torch是否可用cuda import torch print(torch.cuda.is_available()) @@ -173,7 +173,8 @@ CPU: Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz 目录下，并解压`pretrain_images.zip` 和 `sft_images.zip` * 3.2 在`./model/LMConfig.py` 中调整model的参数配置 > 这里仅需调整dim和n_layers参数，分别是`(512+8)`或`(768+16)`，对应于`minimind-v-v1-small`和`minimind-v-v1` - * 3.3 下载minimind语言模型的[预训练权重](https://pan.baidu.com/s/1LE1SPoPYGS7VNtT1tpf7DA?pwd=6666)，放到`./out/` + * 3.3 下载minimind语言模型的预训练权重([百度网盘](https://pan.baidu.com/s/1LE1SPoPYGS7VNtT1tpf7DA?pwd=6666) + or [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset/tree/main/out))，放到`./out/` 目录下，命名为`*_llm.pth` * 3.4 `python 1-pretrain_vlm.py` 执行预训练，得到 `*_vlm_pretrain.pth` 作为预训练的输出权重 * 3.5 `python 2-sft_vlm.py` 执行指令微调，得到 `*_vlm_sft.pth` 作为指令微调的输出权重 diff --git a/README_en.md b/README_en.md index cf8fed4..fbbb89b 100644 --- a/README_en.md +++ b/README_en.md @@ -113,7 +113,7 @@ Environment: python 3.9 + Torch 2.1.2 + DDP single-machine multi-GPU training ```bash pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple ``` - ```python + ```text # Test if torch can use CUDA import torch print(torch.cuda.is_available()) @@ -153,7 +153,16 @@ Environment: python 3.9 + Torch 2.1.2 + DDP single-machine multi-GPU training ```bash pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple ``` + ```text + # Test if torch can use CUDA + import torch + print(torch.cuda.is_available()) + ``` + > If it is not available, please go to [torch_stable](https://download.pytorch.org/whl/torch_stable.html) + to download the whl file for installation. Refer to [this link](https://blog.csdn.net/weixin_45456738/article/details/141029610?ops_request_misc=&request_id=&biz_id=102&utm_term=安装torch&utm_medium=distribute.pc_search_result.none-task-blog-2~all~sobaiduweb~default-2-141029610.nonecase&spm=1018.2226.3001.4187) + + * 2.Download the `clip-vit-base-patch32` model and place it in the `./model/clip_model` directory: ```bash cd ./model/clip_model & git clone https://hf-mirror.com/openai/clip-vit-base-patch32 @@ -164,7 +173,7 @@ Environment: python 3.9 + Torch 2.1.2 + DDP single-machine multi-GPU training * 3.1 Download all contents of the dataset ([Baidu Netdisk](https://pan.baidu.com/s/1Nz36OBBvVBGEx-PwIb7ofg?pwd=6666) or [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset)) to the `./dataset` directory, and unzip `pretrain_images.zip` and `sft_images.zip` * 3.2 Adjust the model parameters in `./model/LMConfig.py` > Only need to adjust the dim and n_layers parameters, which are `(512+8)` or `(768+16)`, corresponding to `minimind-v-v1-small` and `minimind-v-v1` - * 3.3 Download the [pre-trained weight file](https://pan.baidu.com/s/1LE1SPoPYGS7VNtT1tpf7DA?pwd=6666) of the MiniMind language model and place it in the `./out/` directory, named `*_llm.pth` + * 3.3 Download the pre-trained weight file ([Baidu Netdisk](https://pan.baidu.com/s/1LE1SPoPYGS7VNtT1tpf7DA?pwd=6666) or [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset/tree/main/out)) of the MiniMind language model and place it in the `./out/` directory, named `*_llm.pth` * 3.4 Execute `python 1-pretrain_vlm.py` for pre-training, obtaining `*_vlm_pretrain.pth` as the output weights * 3.5 Execute `python 2-sft_vlm.py` for instruction fine-tuning, obtaining `*_vlm_sft.pth` as the output weights for fine-tuning diff --git a/model/LMConfig.py b/model/LMConfig.py index 9aa9291..4a1c349 100644 --- a/model/LMConfig.py +++ b/model/LMConfig.py @@ -7,8 +7,8 @@ class LMConfig(PretrainedConfig): def __init__( self, - dim: int = 768, - n_layers: int = 16, + dim: int = 512, + n_layers: int = 8, n_heads: int = 16, n_kv_heads: int = 8, vocab_size: int = 6400, diff --git a/requirements.txt b/requirements.txt index 9c71f03..ca7ba5c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,6 +20,7 @@ sentence_transformers==2.3.1 simhash==2.1.2 tiktoken==0.5.1 torch==2.1.2 +torchvision==0.16.2 transformers==4.44.0 jinja2==3.1.2 jsonlines==4.0.0