speedup and siglip

3-eval_chat.py

@@ -41,7 +41,7 @@ def init_model(lm_config, device, multi):
     model = model.to(device)
     print(f'模型参数: {count_parameters(model) / 1e6} 百万 = {count_parameters(model) / 1e9} B (Billion)')
 
-    vision_model, preprocess = get_vision_model()
+    vision_model, preprocess = get_vision_model(encoder_type="clip")
     vision_model = vision_model.to(device)
     return model, tokenizer, vision_model, preprocess
 
@@ -66,7 +66,8 @@ if __name__ == "__main__":
     device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
     dtype = 'bfloat16'
     max_seq_len = 1024
-    lm_config = LMConfig()
+    # lm_config = LMConfig()
+    lm_config = LMConfig(image_special_token='<'*2+'>'*2, image_ids=[30]*2+[32]*2)
     lm_config.max_seq_len = max_seq_len
     model, tokenizer, vision_model, preprocess = init_model(lm_config, device, multi)
     model.eval()

model/model.py

@@ -374,6 +374,11 @@ class Transformer(PreTrainedModel):
             image_ids_tensor = torch.tensor(image_ids).to(tokens.device)
             len_image_ids = len(image_ids)
 
+            # .generate时，在初始化后直接跳过
+            if len_image_ids > tokens.size(1):
+                # print(f"len_image_ids ({len_image_ids}) is greater than sequence length ({tokens.size(1)}), skipping.")
+                return None
+            
             # 使用view来创建一个视图，便于处理滑动窗口
             tokens_view = tokens.unfold(1, len_image_ids, 1)  # 在第二维度创建滑动窗口
             # 检查每个滑动窗口是否与image_ids_tensor相等