detail adjustment & formatting

.envrc

@@ -1 +0,0 @@
-source activate minimind

.gitignore

@@ -1,9 +0,0 @@
-model/clip_model/clip-vit-base-patch32/
-model/siglip_model/siglip-vit-base-patch16/
-out/*.pth
-full.json
-trans_json.py
-dataset/*
-!dataset/eval_images/
-!dataset/eval_multi_images/
-minimind-v/model/__pycache__/*

3-eval_chat.py

@@ -79,7 +79,7 @@ if __name__ == "__main__":
     # -------------------------- 问题和目录设置 -----------------------------------
     if multi:
         image_dir = './dataset/eval_multi_images/bird/'
-        prompt = "<image>\n<image>\nName all the differences between these two birds."
+        prompt = f"{lm_config.image_special_token}\n{lm_config.image_special_token}\nName all the differences between these two birds."
     else:
         image_dir = './dataset/eval_images/'
         prompt = lm_config.image_special_token + '\n这个图片描述的是什么内容？'

model/LMConfig.py

@@ -19,7 +19,7 @@ class LMConfig(PretrainedConfig):
             dropout: float = 0.0,
             flash_attn: bool = True,
             image_special_token: str = '<' * 25 + '>' * 25,
-            image_ids=[30] * 25 + [32] * 25,
+            image_ids: List = [30] * 25 + [32] * 25,
             ####################################################
             # Here are the specific configurations of MOE
             # When use_moe is false, the following is invalid

model/model.py

@@ -326,12 +326,12 @@ class Transformer(PreTrainedModel):
     config_class = LMConfig
     last_loss: Optional[torch.Tensor]
 
-    def __init__(self, params: LMConfig = None, vocab_size = 6400):
+    def __init__(self, params: LMConfig = None):
         super().__init__(params)
         if not params:
             params = LMConfig()
         self.params = params
-        self.vocab_size = vocab_size
+        self.vocab_size = params.vocab_size
         self.n_layers = params.n_layers
         # image的特殊占位符，对应每张图切分成M个token，和get_img_process中的数量对应
         self.image_ids = params.image_ids
@@ -373,10 +373,8 @@ class Transformer(PreTrainedModel):
         def find_indices(tokens, image_ids):
             image_ids_tensor = torch.tensor(image_ids).to(tokens.device)
             len_image_ids = len(image_ids)
-
             # .generate时，在初始化后直接跳过
             if len_image_ids > tokens.size(1):
-                # print(f"len_image_ids ({len_image_ids}) is greater than sequence length ({tokens.size(1)}), skipping.")
                 return None
 
             # 使用view来创建一个视图，便于处理滑动窗口
@@ -451,7 +449,6 @@ class Transformer(PreTrainedModel):
             self.last_loss = None
 
         self.OUT.__setitem__('logits', logits)
-        # self.OUT.__setitem__('last_loss', self.last_loss)
         self.OUT.__setitem__('loss', self.last_loss)
         return self.OUT
 

model/vision_utils.py

@@ -40,7 +40,8 @@ def get_img_embedding(batch_encoding, vision_model):
         embeddings.append(output.last_hidden_state)
 
     # 从 BatchEncoding 中提取图像张量
-    if isinstance(batch_encoding, transformers.tokenization_utils_base.BatchEncoding) or isinstance(batch_encoding, transformers.feature_extraction_utils.BatchFeature):
+    if (isinstance(batch_encoding, transformers.tokenization_utils_base.BatchEncoding)
+            or isinstance(batch_encoding, transformers.feature_extraction_utils.BatchFeature)):
         image_tensor = batch_encoding['pixel_values']
     else:
         image_tensor = batch_encoding  # torch.Size([32, 4, 3, 224, 224])
@@ -66,5 +67,6 @@ def get_img_embedding(batch_encoding, vision_model):
         hook.remove()
 
     # 拼接所有特征向量成为一个张量
-    all_embeddings = torch.stack(embeddings, dim=0).squeeze() # torch.Size([32, 4, 50, 768]) or torch.Size([32, 2, 196, 768])
+    all_embeddings = torch.stack(embeddings, dim=0).squeeze()
+    # torch.Size([32, 4, 50, 768]) or torch.Size([32, 2, 196, 768])
     return all_embeddings