From 3400108e0be1589fc46012e857fbe59a03b8e45e Mon Sep 17 00:00:00 2001 From: xinyanghuang7 Date: Sat, 2 Nov 2024 12:51:04 -0400 Subject: [PATCH] speedup and siglip --- .envrc | 1 + 1-pretrain_vlm.py | 4 ++-- 2-sft_vlm.py | 4 ++-- 3-eval_chat.py | 6 +++++- model/__pycache__/LMConfig.cpython-310.pyc | Bin 1601 -> 1601 bytes model/__pycache__/model.cpython-310.pyc | Bin 16464 -> 16464 bytes 6 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 .envrc diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..446626f --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +source activate minimind \ No newline at end of file diff --git a/1-pretrain_vlm.py b/1-pretrain_vlm.py index 5a8270d..e42840e 100644 --- a/1-pretrain_vlm.py +++ b/1-pretrain_vlm.py @@ -171,9 +171,9 @@ if __name__ == "__main__": args = parser.parse_args() if args.visual_encoder == "clip": - lm_config = LMConfig(image_special_token='<'*2+'>'*2, image_ids=[30]*2+[32]*2) + lm_config = LMConfig() else: - lm_config = LMConfig(image_special_token='<'*2+'>'*2, image_ids=[30]*2+[32]*2) + lm_config = LMConfig(image_special_token='<'*98+'>'*98, image_ids=[30]*98+[32]*98) max_seq_len = lm_config.max_seq_len args.save_dir = os.path.join(args.out_dir) diff --git a/2-sft_vlm.py b/2-sft_vlm.py index 15c5d04..0a2b1a6 100644 --- a/2-sft_vlm.py +++ b/2-sft_vlm.py @@ -195,9 +195,9 @@ if __name__ == "__main__": args = parser.parse_args() if args.visual_encoder == "clip": - lm_config = LMConfig(image_special_token='<'*2+'>'*2, image_ids=[30]*2+[32]*2) + lm_config = LMConfig() else: - lm_config = LMConfig(image_special_token='<'*2+'>'*2, image_ids=[30]*2+[32]*2) + lm_config = LMConfig(image_special_token='<'*98+'>'*98, image_ids=[30]*98+[32]*98) max_seq_len = lm_config.max_seq_len args.save_dir = os.path.join(args.out_dir) diff --git a/3-eval_chat.py b/3-eval_chat.py index f505979..aa7c91a 100644 --- a/3-eval_chat.py +++ b/3-eval_chat.py @@ -66,8 +66,12 @@ if __name__ == "__main__": device = 'cuda:0' if torch.cuda.is_available() else 'cpu' dtype = 'bfloat16' max_seq_len = 1024 + encoder_type="clip" # lm_config = LMConfig() - lm_config = LMConfig(image_special_token='<'*2+'>'*2, image_ids=[30]*2+[32]*2) + if encoder_type == "clip": + lm_config = LMConfig() + else: + lm_config = LMConfig(image_special_token='<'*98+'>'*98, image_ids=[30]*98+[32]*98) lm_config.max_seq_len = max_seq_len model, tokenizer, vision_model, preprocess = init_model(lm_config, device, multi) model.eval() diff --git a/model/__pycache__/LMConfig.cpython-310.pyc b/model/__pycache__/LMConfig.cpython-310.pyc index 098838d25eb7fb5387a55a112037fe3c1a605a93..22d8930cde58f9b708b9d50191db2690e55d9547 100644 GIT binary patch delta 20 acmX@ebC8ESpO=@50SHR{)HZUPu>k-ujszM2 delta 20 acmX@ebC8ESpO=@50SI!xscz&pV*>y*K?Jz~ diff --git a/model/__pycache__/model.cpython-310.pyc b/model/__pycache__/model.cpython-310.pyc index 7a19d9a61c3b447e080a45cf372f555e8cd6393c..5043f059ff42598905298b9e90e921f8ba09207f 100644 GIT binary patch delta 2914 zcmZ`*YitzP753d-j~|QQHg;@d8^6NX#Evm6m>NuM)s1ax!XwCW3CplMwioZtY|qSM zn?zs~h}x!)B-bQO5kyh{5LF>bSE)#wmPl1vHELB9rA<10B&0=@G;P(VP}(0o-yH%L zQZ4Q0bI&>V-1E5S+_|(UE-i|%T~t(o?}nBDPx|Rq1ytOhNv)RTW(@HV`m&X z2<{AnzJe-2HvuB@ba{hzFUS#jvHZ=dT}%%0&(Rq2bIL_iHSLmTD^7|j*;<*MP)P`| zY?$^i>V;Iy|EL6@1C-DEneqa2J^Wk8zg{vSnqts%{nJ&>zYOkIl!#NlR0jom9I(jX zg`7k(Ltn%C3HiIKhpHZ9@+ALmIM*Ob!}M6zTB!-KBs0~oFrspUHS5H7nW~mnKE2N+X%QM;TJKhu8anO#etcwX(&et_ItLvtUy2v6=OC~JSAgX}g*8#`n zt97mIPk}rQji8fB(Zg6i1Na7bxkpU09eP>@>&L_ixud>Dd_m6Cf1J0SrQ>?O+}Tzu z&d7^vnyV#qPVvvlXF%!;OG-l|rL}UE^j7WU;MO+$^4?`RV>M1ElHyu0Z5nPgVY%EF zrg7NxgE=MNU)|8T4a(n#UO~okKA$m7&iM0cow7~dG-VQ9gwpF$Z*K270rETmiJ@-; zUIx6vFj+K}Gx$Tw& z{l<6~)^@A*dGgNFogS9I0lWwJEdx){Wpe}EM*uI%i8aIGi}KkugH`V^qgLGP&y}LQ zir!ICRp<|BYIJ;6lKyO~nh)W_35VT{rpZ2}hkWC|hsi$xt|(u&cN`E`RbN86M#Ek%GD2t~oO?J>xQI-yq4A zAKbOCsgD)BkY%Jy|3;`DS>7o7yZWj>U?qL+vJwQDK_AKUUA5wh{7IMo;LQoACOdr( zR<)$DV3I?xsfCeF^KBp`iSX#rufYykHl+v&p;mwzFB?IrDqEn^az}UMj^~*4@}O?; zwL1#IF~EoNZ1>%(krG0Z^gKV6AA#{VfU0dI``2zEMqDud4EPI!Ug#Ba5iGu(R?NIL zyZsn&ApHlRoVuSWPgCP4o!q8PsqPLx}h?DT>^_Ed}?Lhrxw(x#fvsKcvaOSK%rn#`Lu7Yv$q z2n{zGiXL>1_@S!E)tMSgyj*n$39h1jvO|6A0-J7{8TLPD0={9G@=Jkh!ZrLRe88ab zf+N-xzm6|_s%ejkQH36KOxM#~JDG4?zb5CDZYp8Q%#Fl=l&w^FJ3c^8Ilx RL0tf-!EzbYHF74}`PcWrOhZ^y3f*p6S#*r{E|sqHuqJ8@-joH*pskXUIplil%p>)jdm z%xr8YAUh$Z0!nFfojwRnq>3LcqNQO)r2r*G2r6EZRYBDbP@(`)NGX+g6k37A`R=4i zWQmpL+jGx3bDsB}J4^FoXSoSdhDFD#aJ%M9oLRoh%*Ia^>#kDsf(3 zUei#q$egqM*trZSeSTq)pBAOLa+P#f?dIgx*1q}4J8H&iTuv0l+rrC4KI(($5r!88 z!LpaRO1fWeuiH`bU1n;$G@-(yEAfF(E9GeiA>yPV>;TD+@dqO887Wr8C z|Gc{e%M>rbe>vcLna27PVw=OXCfuBuGs}Lkr8trSC|m}p!lRgOF&R^a?nDx%V^Gr2 zj~QMvPs%H+YumO$`3KO;OPlt?X~W=(e?zTPmccKL8$_=`>4#ElXz9fMp>F|D7|H;? z4S0oNEIgiMPmVP;8A-%uTro=<>Y)2=~AvQxlm%nV%pKY#SM5o_q-m9iiG+TZqR4E5r zp6yg!D!rpihv3qiTA*-d-8|iK3_k6aNpH*IySvw40(Zr@!Oa@CXj-S3w^f$yl5Fo_ zybVM9)bTlU_qn#;VEH@16~H?TJVnbu*8_S0rvWd^18auF7v+UD+sfZ!MwPgAK39s8 za(YW;RjM5fsL}BiN!o=bH6KF95;lh$nIh{U&F>liBTW7Ya81RsqxFcmE?;b|_FreF zJhG-tUTNK5ew7#9w;ffAqCC2q8EKpJt#w)C%_c9%F~3T^-$bJ;$UQgu2e# zZVrNZz`OE%$Gxjj5<-!*pjXQG!1xRu2<%Z%C=W$oru2d4^%iYX!Cm_weg*he6|J z51ZqBJHEKFe%}C?D)nsJaNK}nB@?#e_2dPmn@X5opBkWg5^xHF5D!czVOpt#o>cAA zO1+i^6;DAYz@`j6<@#vqX14P1rq;0Xm*ZHIhDncP&Ta~edsUUxE?0*H8XtdM zAoHS!~R3e?qN@VSYkte8QVN7N(ad~=PsA%C@{t?&kX32MV%BhATz z{+{+Fyy`QjRJq(z$D-&&yJ@zMr_Ct5abbe2;(hg_}Vauw7K`S1P%{{v&T+K>PM