speedup and siglip

.envrc

@@ -0,0 +1 @@
+source activate minimind

1-pretrain_vlm.py

@@ -171,9 +171,9 @@ if __name__ == "__main__":
     args = parser.parse_args()
 
     if args.visual_encoder == "clip":
-        lm_config = LMConfig(image_special_token='<'*2+'>'*2, image_ids=[30]*2+[32]*2)
+        lm_config = LMConfig()
     else:
-        lm_config = LMConfig(image_special_token='<'*2+'>'*2, image_ids=[30]*2+[32]*2)
+        lm_config = LMConfig(image_special_token='<'*98+'>'*98, image_ids=[30]*98+[32]*98)
 
     max_seq_len = lm_config.max_seq_len
     args.save_dir = os.path.join(args.out_dir)

2-sft_vlm.py

@@ -195,9 +195,9 @@ if __name__ == "__main__":
     args = parser.parse_args()
 
     if args.visual_encoder == "clip":
-        lm_config = LMConfig(image_special_token='<'*2+'>'*2, image_ids=[30]*2+[32]*2)
+        lm_config = LMConfig()
     else:
-        lm_config = LMConfig(image_special_token='<'*2+'>'*2, image_ids=[30]*2+[32]*2)
+        lm_config = LMConfig(image_special_token='<'*98+'>'*98, image_ids=[30]*98+[32]*98)
 
     max_seq_len = lm_config.max_seq_len
     args.save_dir = os.path.join(args.out_dir)

3-eval_chat.py

@@ -66,8 +66,12 @@ if __name__ == "__main__":
     device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
     dtype = 'bfloat16'
     max_seq_len = 1024
+    encoder_type="clip"
     # lm_config = LMConfig()
-    lm_config = LMConfig(image_special_token='<'*2+'>'*2, image_ids=[30]*2+[32]*2)
+    if encoder_type == "clip":
+        lm_config = LMConfig()
+    else:
+        lm_config = LMConfig(image_special_token='<'*98+'>'*98, image_ids=[30]*98+[32]*98)
     lm_config.max_seq_len = max_seq_len
     model, tokenizer, vision_model, preprocess = init_model(lm_config, device, multi)
     model.eval()