update readme & requirements

1-pretrain_vlm.py

@@ -122,23 +122,10 @@ def init_model(lm_config):
         if k.startswith(unwanted_prefix):
             state_dict[k[len(unwanted_prefix):]] = state_dict.pop(k)
 
-    for k, v in list(state_dict.items()):
-        if 'mask' in k:
-            del state_dict[k]
-
     # 加载到模型中
     model.load_state_dict(state_dict, strict=False)
     model = model.to(args.device)
 
-    # # 第一阶段冻结llm，只学习vision_proj层
-    # for name, param in model.named_parameters():
-    #     # 如果参数名不包含'vision_proj'，则冻结该参数
-    #     if 'vision_proj' not in name:
-    #         param.requires_grad_(False)
-    #     else:
-    #         # 确保'vision_proj'中的参数可以更新
-    #         param.requires_grad_(True)
-
     print(f'模型可学习参数: {count_parameters(model) / 1e6} 百万 = {count_parameters(model) / 1e9} B (Billion)')
 
     (vision_model, preprocess) = get_vision_model()
@@ -162,7 +149,7 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="MiniMind-V Pretrain")
     parser.add_argument("--out_dir", type=str, default="out", help="Output directory")
     parser.add_argument("--epochs", type=int, default=19, help="Number of epochs")
-    parser.add_argument("--batch_size", type=int, default=128, help="Batch size")
+    parser.add_argument("--batch_size", type=int, default=32, help="Batch size")
     parser.add_argument("--learning_rate", type=float, default=4e-4, help="Learning rate")
     parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu",
                         help="Device to use")
@@ -176,7 +163,7 @@ if __name__ == "__main__":
     parser.add_argument("--accumulation_steps", type=int, default=1, help="Gradient accumulation steps")
     parser.add_argument("--grad_clip", type=float, default=1.0, help="Gradient clipping threshold")
     parser.add_argument("--warmup_iters", type=int, default=0, help="Number of warmup iterations")
-    parser.add_argument("--log_interval", type=int, default=100, help="Logging interval")
+    parser.add_argument("--log_interval", type=int, default=10, help="Logging interval")
     parser.add_argument("--save_interval", type=int, default=100, help="Model saving interval")
     parser.add_argument('--local_rank', type=int, default=-1, help='local rank for distributed training')
 

2-sft_vlm.py

@@ -122,23 +122,10 @@ def init_model(lm_config):
         if k.startswith(unwanted_prefix):
             state_dict[k[len(unwanted_prefix):]] = state_dict.pop(k)
 
-    for k, v in list(state_dict.items()):
-        if 'mask' in k:
-            del state_dict[k]
-
     # 加载到模型中
     model.load_state_dict(state_dict, strict=False)
     model = model.to(args.device)
 
-    # # 第一阶段冻结llm，只学习vision_proj层
-    # for name, param in model.named_parameters():
-    #     # 如果参数名不包含'vision_proj'，则冻结该参数
-    #     if 'vision_proj' not in name:
-    #         param.requires_grad_(False)
-    #     else:
-    #         # 确保'vision_proj'中的参数可以更新
-    #         param.requires_grad_(True)
-
     print(f'模型可学习参数: {count_parameters(model) / 1e6} 百万 = {count_parameters(model) / 1e9} B (Billion)')
 
     (vision_model, preprocess) = get_vision_model()

3-eval_chat.py

@@ -36,10 +36,6 @@ def init_model(lm_config):
             if k.startswith(unwanted_prefix):
                 state_dict[k[len(unwanted_prefix):]] = state_dict.pop(k)
 
-        for k, v in list(state_dict.items()):
-            if 'mask' in k:
-                del state_dict[k]
-
         # 加载到模型中
         model.load_state_dict(state_dict, strict=False)
     else:

README.md

@@ -110,8 +110,8 @@ CPU: Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz
   ```bash
   pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
   ```
-  
+
-  ```python
+  ```text
   # 测试torch是否可用cuda
   import torch
   print(torch.cuda.is_available())
@@ -152,7 +152,7 @@ CPU: Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz
   pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
   ```
 
-  ```python
+  ```text
   # 测试torch是否可用cuda
   import torch
   print(torch.cuda.is_available())
@@ -173,7 +173,8 @@ CPU: Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz
       目录下，并解压`pretrain_images.zip` 和 `sft_images.zip`
     * 3.2 在`./model/LMConfig.py` 中调整model的参数配置
       > 这里仅需调整dim和n_layers参数，分别是`(512+8)`或`(768+16)`，对应于`minimind-v-v1-small`和`minimind-v-v1`
-    * 3.3 下载minimind语言模型的[预训练权重](https://pan.baidu.com/s/1LE1SPoPYGS7VNtT1tpf7DA?pwd=6666)，放到`./out/`
+    * 3.3 下载minimind语言模型的预训练权重([百度网盘](https://pan.baidu.com/s/1LE1SPoPYGS7VNtT1tpf7DA?pwd=6666)
+      or [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset/tree/main/out))，放到`./out/`
       目录下，命名为`*_llm.pth`
     * 3.4 `python 1-pretrain_vlm.py` 执行预训练，得到 `*_vlm_pretrain.pth` 作为预训练的输出权重
     * 3.5 `python 2-sft_vlm.py` 执行指令微调，得到 `*_vlm_sft.pth` 作为指令微调的输出权重

README_en.md

@@ -113,7 +113,7 @@ Environment: python 3.9 + Torch 2.1.2 + DDP single-machine multi-GPU training
   ```bash
   pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
   ```
-  ```python
+  ```text
   # Test if torch can use CUDA
   import torch
   print(torch.cuda.is_available())
@@ -153,7 +153,16 @@ Environment: python 3.9 + Torch 2.1.2 + DDP single-machine multi-GPU training
   ```bash
   pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
   ```
+  ```text
+  # Test if torch can use CUDA
+  import torch
+  print(torch.cuda.is_available())
+  ```
 
+  > If it is not available, please go to [torch_stable](https://download.pytorch.org/whl/torch_stable.html)
+  to download the whl file for installation. Refer to [this link](https://blog.csdn.net/weixin_45456738/article/details/141029610?ops_request_misc=&request_id=&biz_id=102&utm_term=安装torch&utm_medium=distribute.pc_search_result.none-task-blog-2~all~sobaiduweb~default-2-141029610.nonecase&spm=1018.2226.3001.4187)
+  
+  
 * 2.Download the `clip-vit-base-patch32` model and place it in the `./model/clip_model` directory:
     ```bash
     cd ./model/clip_model & git clone https://hf-mirror.com/openai/clip-vit-base-patch32
@@ -164,7 +173,7 @@ Environment: python 3.9 + Torch 2.1.2 + DDP single-machine multi-GPU training
     * 3.1 Download all contents of the dataset ([Baidu Netdisk](https://pan.baidu.com/s/1Nz36OBBvVBGEx-PwIb7ofg?pwd=6666) or [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset)) to the `./dataset` directory, and unzip `pretrain_images.zip` and `sft_images.zip`
     * 3.2 Adjust the model parameters in `./model/LMConfig.py`
       > Only need to adjust the dim and n_layers parameters, which are `(512+8)` or `(768+16)`, corresponding to `minimind-v-v1-small` and `minimind-v-v1`
-    * 3.3 Download the [pre-trained weight file](https://pan.baidu.com/s/1LE1SPoPYGS7VNtT1tpf7DA?pwd=6666) of the MiniMind language model and place it in the `./out/` directory, named `*_llm.pth`
+    * 3.3 Download the pre-trained weight file ([Baidu Netdisk](https://pan.baidu.com/s/1LE1SPoPYGS7VNtT1tpf7DA?pwd=6666) or [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset/tree/main/out)) of the MiniMind language model and place it in the `./out/` directory, named `*_llm.pth`
     * 3.4 Execute `python 1-pretrain_vlm.py` for pre-training, obtaining `*_vlm_pretrain.pth` as the output weights
     * 3.5 Execute `python 2-sft_vlm.py` for instruction fine-tuning, obtaining `*_vlm_sft.pth` as the output weights for fine-tuning
 

model/LMConfig.py

@@ -7,8 +7,8 @@ class LMConfig(PretrainedConfig):
 
     def __init__(
             self,
-            dim: int = 768,
+            dim: int = 512,
-            n_layers: int = 16,
+            n_layers: int = 8,
             n_heads: int = 16,
             n_kv_heads: int = 8,
             vocab_size: int = 6400,

requirements.txt

@@ -20,6 +20,7 @@ sentence_transformers==2.3.1
 simhash==2.1.2
 tiktoken==0.5.1
 torch==2.1.2
+torchvision==0.16.2
 transformers==4.44.0
 jinja2==3.1.2
 jsonlines==4.0.0