From 1c884f47afdcd39e3c7ec070267e9967528c7497 Mon Sep 17 00:00:00 2001 From: Eric Date: Sat, 1 Mar 2025 00:40:01 +0800 Subject: [PATCH] fix load model and lr update --- model/__pycache__/LMConfig.cpython-310.pyc | Bin 2113 -> 0 bytes model/__pycache__/dataset.cpython-310.pyc | Bin 7845 -> 0 bytes model/__pycache__/model.cpython-310.pyc | Bin 14919 -> 0 bytes train_pretrain.py | 52 +++++++++++---------- 4 files changed, 28 insertions(+), 24 deletions(-) delete mode 100644 model/__pycache__/LMConfig.cpython-310.pyc delete mode 100644 model/__pycache__/dataset.cpython-310.pyc delete mode 100644 model/__pycache__/model.cpython-310.pyc diff --git a/model/__pycache__/LMConfig.cpython-310.pyc b/model/__pycache__/LMConfig.cpython-310.pyc deleted file mode 100644 index dc5419ccdd82b4cf463037b0a1bc9d454afcf35c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2113 zcmcJQNsJps6o#vx+e>?y@$CDS-C1Um5EcnYf{=zw7$Gqyb13z=%VTHS-A;8oOrky@ zB~lOv?r`Q3?i}F6kz)>=N(vAX5m{nM#1_IL{%2=Gf^y+NS?}|Eue+A_`c+j}D&+;* z8}EF&`AJcTn`}JVY#65@(P6+7p5=>{RkbYE$){GTnzFtY8O1ez;nDW&10(tjiV{`J6IJr8YRVxe<=9Tz z$v9aj=QvK@DL6%^3^66dTK6>6+jko`-~Q;{ zt&?4mg9VRx>+F$HxU1SJ0`D>YM6xQRGyM6op_qct0%A~wz%j5#- zyBCy>Og4}W<$96H2XgEDj($F@xf?QSUQ{OEXnLLsBzG&cI)2=2`$~p&j1lUVRP6|E zcUwm4ob**-@;YoQ88=kynyd#S?8GKt_uZ%=-8c?Rwi7AY@?BFvhfupxyTO(zV)B2i z^0_S;@Ho7--qh^rjr`D)@2|G^v_EF3%t+9|nzq|Y$B_hO9B!F%fUccbc{@+o6E$4@ zh^H9ULfs5DWxW&B%z)e3#&V)ax_-NXKsY-1%s?;bIBajpwhlLp!@<)qaNM#Hmq#Xv|F>9K4 zhQ=i$bcw;|o*o3&S9O{7-upwW4KsSLk3d`P55gt4uE+S%IIHWcl{|m6z-=ihOwf6Q#@+}+-=v~g_wr&&4-#8bzW*S(y$tK; zdCnIZyus6u=tCeUY>RBNp7^83ylHyew(RTh{6#7H6C?aVY5JWqbf2>H8|CO8IrJ;# z=@%-{&s3zJs6;=~Al;=h-Jv1+frjaO8lmrKl)j~D`i5rd7R}L3ny0U6fo{+ueML+3 zCGDawX!rQ_-n!Q+X;Z{S97J`T;T4Xh$-U4|?n&PB N5r5;NC|aYbzX8l*`IZ0x diff --git a/model/__pycache__/dataset.cpython-310.pyc b/model/__pycache__/dataset.cpython-310.pyc deleted file mode 100644 index 90690ba48f0a7aefee8a0568471c1a743271a0bc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7845 zcmb7JTaO&ab?)lh^z>YI_M#=1qM4QKBpzELtwffMgb^g0WFmXLVno`KI!+q1Q@zWX zo#`1?_3$$3fdvl@I{_FPuwf^bXv{v?FaaZej%^^%0rCs#HIQV!8K55wCX(`<>bdSx zPTYe&-F3REy1LFe-}$PhQ!4oejm7Y49R1-7}0L zEcJ(MP)KuWf%_ZAXV5-z(%g`x*1jWn;x=<>4$|X=+vcE{x}$tr9Qg;BXlA6jQQ^SA z)#F99^l&Y$!};JSJ4un3Qx9Wax@!$ti~wN9)e~9Ys9}rY@9rDzE2v!@vNw&pCb`ZL zXZH)4q9(+jZB|Rl?sTJAc_OND@&{!%WhYin3o#*;1s_ya3l+?`s~ohc-BfN5?)c1e z%F!LET+-1UE2rLyYC<_(f#>Mi&&JiQC0A5F>Fh^wqaTR?&ZoUA+>dSstL@s2aHqGs z8;OXAc+nmz7c#dGbxW5<773MohX{((+W26_8p@_&3v%PxTez|?Rvj0asL@jHMUqRT+hvyfWji1RpEmyL8s|bCCm7(?6DQ3Q5{<^qAG~QMDil9F;AjXkXb$)M^Quo2-rpUu{eH35+$@iTDIHaobV5UF+(XD4XyM zkcoa;yDCqm%wc?TsqEc~?H=|4FAM>TNf>?>J<1}44;!z;q9@;b(&;~bbm~t}S}dSF z1sHv?wgGj6`7xT9`wf~CGn*59`Cs6X_zc87CZ4Bz!s4-z&sW$p{c*O7Ue$T=Ivzy!PkuD^lIGqMu9=^!6j&_TPP zvlDleOW`Z}Y9Fv&6W?NO-~RxIIEfd)IsJxg#4@SH^+*(;(qE|Dnk3xOmQ)Vn|G8(C zlXOJ=it=|wrya(j&qZm0NNa$pF$WyOBKUVe<#tMGD3Du z|N5sMIakvr+f`S=9Pf3LTPqoCba2o5n6elUuj)ffd=@j-1HdFesc5{x%2>KznLQySEK6H!-_sRY0lQ+@Z=-wKik^(U?hM*ui7n%rHGo_%#iG0fr z;2xnv0ep|qBssDN^JVk!6xu2sMq@zEQ#J@G#8gh!+TB(pt9E8ee;RmFP&@G|1oBWm z4C}3$lwmk?cHR3Hp6e;}nTubA^^YI=vT496V1LpR@j~FaXg}p4J1Br%;9JfafBT7n zi2%^x$}O;v4T3}H%U~yLWTrOJFSxz~B4(!u0_6V6e1?t*{;KovcNH^**|)dFD&8d4 zNIV7sM905p!l8gN@@XUB}mnb zQbXz=46Fn(%bx|4X#V`;F=<$0k5B&hgZ_UK8L_H zV;XKo6Y~+6LnLZRGy_}lcl1dVb=6!Jur*R>>k!B)kCp^PIes_XejRl)xW)9ZjKLOk z-md%);C5|pwq{66F*rF0^XSx=N8vk}uOY+i9LMZfz&DFCEI&;wKUcVgB`0PKbHq{y z3uI@*;O<+do-KHA(`*ti(=_0=g}KT1ozxesPKmGCB-~QI;uhfz+ZM`%4 z<=el!^X?CBz5TO)`_<3xisw*Evv6hiB-U8@skiHe5$%77PAgj_UASi|Z?YUyk5M4( zMOZ;;APup5(AHRcgSw-h5ma|I%4X}FyqZQ{#5J}0vu8@Hj0aBJXG}X{_rwF=}EoYH$GV9%@YQ$c|K8Y3t zxQOC2<2w+l#!NyAPsv>vDXh7O)qzTf~Pv>d{#EsVIN4s{bh3k}(s_QS=w-@e)xI9TGDXn9-9i9*|SYG6c~R&+{yt#mb7HXCF>bp2O2A zLGk}T1Vs}P+G#0Z`}g9@Q$kWXL`Wv-(4ukuE5wHo60-@5_n25saR-3cD=T=QCb?rO zLe#_|(mSChNginx%!zwgsca7HJ$Ps6r#2GDGQq_NdzkRR;;2L`6{J<8>5t0e#QLZ6 ziIqqW66?kJ#Hxu;ejNA$L%$#-r09`YB{5sdEIpF%+@VT)B(y7%Z8TtH>mI#;(OGzt z9#E2+&&T5}YQ5^2v2RMvIw51*dZ+m)R`d(hkhIT`tqs?amg(o}{Wmx#Gv9mi{r>9A zXul6j!e-1zX#{%ggd1~?2-9bL`akF4iz#jO3MjIp# zY{k%aA9;WU`(_$!%#AfrdsJK&((g(LNswZ3>nzmk^H za=MgO(&co87vFLbdZlgU)~|GAw)IT^S48}RNHLd}bsDN#;tC2G_nx`oA=t#;&JB=< ztQ#UH`mR6;Yf|uUgd)oNSN_e}eMwiEJOX@bx6sb22{U1@cofjC&H?QfRr)@Oe})K3 z*#O2IDp;EIK2?mLPj>NxynrTiAR;JBnwPw0$J@GFzvLBFmlf+vj z?vnT+i5U!Q?|&J(|HLU@fWSf)KU$f%AGkJ2+d@o^rH8ETnLZN4>}OgTP198EVSjxg zdZw%0g&a=NGoff=h=RC9;u|EsMdHv1$r=wx|4$J8Q}b^)$`-J${w}J)uotPx;<|da zbk0oNp(?Yt&C=3$@Zje-CGDf=v{d%2${!2T)4pPx{w@q@mqI3ZBmD_c{fOA4r!I&U z@3fUEB9Ye$Y>?uwo7pNJDH>I35}O-bIeLRM+dr-7jnmV;q|uV6Idpk|lSgDdBCZ;t@aAjv@$dTojg!JSI21b532_Qg8`~qzM@B{V_1Fpo55kDltcHjib zkC+lS-*;|z&&)oQ^q_9tI(6&Tt$WYoJLlXPRw{vk-?2A-b?b>24dc&warjrj#j`k) zhG`g)5t%)sq0eT+lxwSD$=Pn$krmlJr|&czlWm-y+xHq?ztAZ3{f6H!Hi|+wdqJZl z&y*WwIam5s%P@`pLZc=v++Mj~ZB(W1^%nc}MqTQK-coRN6D6a@&e3s8?|;#A|IYszr-8-NwnN9xdH88mFRVJaala zhWjh5p~dlN^`_G}gI-TWYv^??)_uMQl#|gZP)!6&D&VX`eO8E{@-V?2ZvOcAp z1?4-Uv!I-vQr-*7d!vUyc_^-9tPe%zE5^pdZ*$f+-^MH(8)ojmbn*G&pxxQpaB};( z!HwMi#3=0y2hCpYUK)>jaqhg-Nm8`51_P7}pH%TB)$9!7=;APndmybm-yA2+-b)uh zF-}M0^anfXl}|S15&Pmx&$alv!#^%Gf60eX#0D3)Va0aWX`7K5yO9-pk&Wv@+k4Xh z&57Kbm4+XAQ2`TKeA94@Mt~3T^TNv)U)db0er`t{6!9p@-6*{=id&AnK^LrhIBXor z`%t9Do-r}^!3LR?8KITh6DM`MUS?)?WZt&+O;8Jw@rv=5wfUB*7Vr$-<|gA&ta3jL zJA+OdhN=p>@=@f)Pc~JvAE&tOrbE@bk~_meoaEkXacApFim7#yxYt${J|T~>&$rCH z7>2E0GfBemE#vPWePVE}A3yoTNDaGjD@~s0<3%{4C;FUeU7j1=P)m52N5anvOwU|1 zcWTqQI5!(!%fijsk9Cgpc^t`iqsWY|k(r!@)SB3+Igz<r9XYo#FJ14rv2QLL zlR}0&%b1M~ck^v_^!6S6E@9yd+i^U?)NlCeIOw_C?hTu%Iwp0MTvcgqkA|=1PCsrA z*X-KI&Sv#9X*hO4C=d{%k*u40&MQ!B(v2=1DZk-Qiexc{J=b zRbC`7hrP~ryy0mUKF7|Ukd|!~Urll|%?qPp61FC~>tEs`!bIKF9}5BHhj;BQGO`n4+Bj+d)Qs#eKw*5reA)Os zm{%qW6E|;bew(&#WRY#Yhq_ZgoxO@k#q34`w9b5F1}KI(+gwzjQg zop_RL*MEI#1#lv8h0FE^z}ok7v!>d$^0k&8n{^f)S<#8>N|wszSI zv@zaP%gkFatgEQ2dS(qk{epFspJyuRl|r&%=hnFohahP0bj^YF2p=_a?X zU&XPlKjH<@8us<8A&Tv=;h_eZ#Ce}uJ-wF7ptcRnBn^Be{fH&d8aD4t&2ycR!p{qc1$)Q6vZd|ls5J}g7t z5C+`DYIBWqFTOr%4kGn2TtGd^;$P$q>vGa?l3qN>gF(1`ExZypqa^ov3g?U}93sq7 zoDN|78qK53UDfiTnbxf;bd>ci^$O@M97ldM4E!aZK`{d}fH8L45x_lE*|+Z`3z+K5YjQb0MBCs%Vu`%0X2tUA6{X3trch_$o)ytKSB!~(t(6roLP6* zu-qX~QWuDz3p<4?AGIPBX3#Co=xFILgDoME!w(v`_t?naIbr6%}vWbP}*2f!4Ga-m-G%a;KRHw8&2-&}T8& zY?y`sbJ4h!h9c@*H(}quci!I_v;m>~sMFjMwC(hp$u`tz0FaqLablG>-lGUQG*tNq zsO5#hl#eTeP_*XsvNpS%F}k7*Xk+~{A1sNKg@TSXX7yeIhKF^xUW3|sZ5zim9K*cu zf*OvXWAZ{Yt&}nLK{&grK&5r?ZPS~@aT15EX6s6v7p^oDG*wz52JP{nHT5I{sJi3| z=U_mPhNl^4NqjYgoNA5fq^dK{7gW5(IZ)x{al0L>4M*Uj`T>3wv9tOS z76Q=bGA#2%rI0*^B7jm~Hhuh6@#jIw>pxzRx?-M!WG|aL_2*_HHOJ1IT1OTW*eERF zFcGDkS4VJb!tMA*c&*tR$1;W%1#b4Mp)!ZrzYPHdOBtWGcfH-ht`E!tvrblp9@J+5 z?|sv@Eqk}P8|;>{B4C}D0aJ7-bl$$RTg-wvjdaq2taJ~pl=>OkX7-*9Lyq1uprWy< zelRU|SvLVCf-UN48_XO`sRS6MIkklTOX^3_OQ|g;f47VgmGO+TZ>1H`jc#?%Iz-_U zhbV`>#ON-lpUohw+cwTdx28UaauGnn-L0hc?vjiazIj%mV|Uxztq#72H1zIA#eJ)0 zq|1|Ic*ff|0f85?1=Lr7^Z<`%WmaIFW!pJnd@ov|=i}MJWObjOI=*yrA}a#D;n~Sr zwji~ItW-00t66pS9g~xsbG*|A)tH>ZaXPE+V`bPXgJ*vr=pwvu3B{2knw|n|!?*-A zR1`s>9OuQUe1xOC*a0XI5BcqP;W!C?*f^sR3hjxCSi~%zXA!cX(5la{5R)ws!*0R3 z+6FBzras2}1P$|Z%&(IpeE%?rt}okI9T=mdeN z4jPIXRbAHFC^l+Z?8K_3qzb|svqk~T?aRpy`qBcNo1ywX-uiytvaVl)PapzwP2C{H zgI0kXkb6mM2qkhCh!J0mgH6t(D0^q8n5Aa<)(WJxZU%Op<+rMC4R+ZxE&21md8+V; z0656BRfA;jEFMVo)M11pzqwIX)C-DWR51>qXeYIDmHG89#e-z16fvpe~_!7Dqyy?HhH|I%%2CJC14Hu`KId$F{yzcdT#vZv7o;%N0NJqtWJ1 zLK6>05E2G48m?dg4;<;`r3-N!UC`cg?zJR}^I$yiuknui;DLwXg~FOI8?fRIZ7XcI zeNuZAhz9_$p*)Td+bWeAFaWJ^NU8@5S?Cf0K#c)iz=4W9vG~O)ovmO$=k(wK?MrE4 zQbyZ~*zt;3*23o7oEZVy>P6P*cU(}PLb*{ru#0(cr4vOl>=T?15lb!~49mEeb^ye~ zVOur%IQ6o+%zM^rAH?w_j_*=iq*7F77UnhyKgB}K%zZ4(8@QP)qX>Yj17O2b;Kfz> ztD8H^hZbdSPmZh#zoUqQ!k$pFpm2DY5{aQ_57i@P0;q(M7hWM2Wo`oBa;WO3S^O9Z zNOY$+R$t)tKW6bTivu&h|Njy0-em8T74mpA%ny8$J&t(C0K<+5rUo5ClZPuK|UJ=_ddPd)r*gOIe2+dNh!0||3L|_n-r8S>|ZbyUex7hAV{soSNs;JH|>+0<5 z?w+R43B%4o47o_>8co?t}UWRw84}0Y%!}Z8t0oJvAeQ5mx zK%coSLI+fX$KpuL)Fm*L*rNBK*RA({^ry@FU;eG9)oZvy;QL9wPpr!aQ3D5;s30T{ z08|8SF*Z(Wcu4usir(vqx2Adb7f9x)^QFyU`fcO=Vp zZ(wQ$D`SMR;WSB(ZO?4@VswiT5UzubXNi3tY~FIER|Kn9*y?;=j{ytd8%fi)RFr{VW; zBy|+D%?NW?!MlM&CU)-Z9Dl%<95GxHGRE`IOGje3m=owC?ZOhF3X1J^J* zFe*r~h#vt%e!`$(LrR1u#1x}qs}7zP0|uj}LlCC+1yB~5Z`>`VG(SkCzhJz&3}aT_ ztz?K*BIr=wMwnvHgojWP522)K=Z&i$0>#3r`(37fOr7?D`Kt2GFenQ!jWnPow#)3# z8@r_eU3|n0(TC3wBDrJj+q1hEfeRDEz{$-F@Na_L$G!=(iO>w#b17pG7TFd0dOF2& zjkQG<#5BNu&$0fiEI5avar4D{g4>$;ewHmnTL%KmB*(_Q0J5kb{2ujD-u@L7xdoL- zgOL}yo zC}}n>W;XcExZzaP0kbXO8>HLnySCq|PK8*5Q-a&9mlX z(3Gd_GMLdb9dl>pE_R$M%GOM8p&x^vJcm*L1sBsiDB{TQE0L++MUrmfAOi|Nb>adL zcsQWPCO#n}_>_Rk>K4}kc)(wxbYA^9xQjfCHUL_WN7ldvl=35c72$XAYmgN||K6)- z$&Z;2S_zN=RHhgKiOj5wa|QfS5w3K|l>&aM066ZfCFFn*$C)g^t1eG!(6r3gE=H83 zdW3-0zJ)LoBFT6YQz7oAW+X_R<|Z8}TAFf**lOVr%{)ys%L~i_0C(j6<<6ir91oCj zZ;i&t?u`4R8wd)F`?UMQXmfT_zsWZUKRIxdL-lWXO?G8Eh-nz%;*Es^7HTXBJLts+ z!$n9RCr63eo+pj+w2_VpJ7Auqu?UX`u@!NC^zB9V{+~D`G*gF{vO=Totip6#V9hd3I%tnrjs;wQ z#K@hS%hg8kV2drDvB+;>Fbua;Gm>{c*jfG>M)P|dQGkN>+b?;mbyJbv!g1s`lZ500 zARl&^u|vjbJ7Q3=9{}kky2`ZN<{@Xfm{%^s|G^P`6q$nU-0wAS0MB(IZNTkA?_zy! zzzG=-40vWu^F)w1Xh39`aRyDYKajqzt4^j~2B&0zXb{h6g_sbHx#Vti`!ZTqAhGO2 z=XCC+FQ82=c`;p__Oe97pF#smpK`T<{W#Xa2AU(T_K#xR8g##mT3%pYqL1whCDJdE z#PF=nohj-n?HZSP#}_Xe`-j9a&a3l=A#dbmr>;QedLm2bDVRqjEW=Ds6)BIJAQ#Fo1J4vXJIL<|tUIhI(sE0ziB9!kZO|YF5vIv5b{unm`zFUW@5LPZL((aKy+8zFq>nLW@7uyXd zT88}1(z}$uO2aLhMKd~v^e$4nU04jHcs(R`177jasa>RW9k{qYQo5xW5zl?2SS~I^ zE3#D(`#2hv>1M&Ipzj6PgyVTIn~Ik%ZswkZX^tj+DZKN@7uDS@V}uPtdqm!dFYIx- zv7v;<23>eHwp$;&0w z@2j)M5$C}*Y>K%YCY>Glu*mLGd298gy^7+C{mXF_G1J5T@+vGOchhKEauC>7|B-$E zC${mXy-??=KPAPXev_!UxLEzP>UEM~Y5?lA?`zNSx^FU~_*G*9? z<@P6j0GTkYCS(Q)GqkJn4c25X`GJpeE{RTw2$#CTouF2?9K{sqVulhc<+?G zou&KCH&SZ@B$Qy&l6>K4ZZ>IgQfEI7f-W-?QwcWvf&(USRRG&y!ba7|*fLW_mtbm` z2+8b;>@bES0|1GDJokg4wLnVvt^j%1A9h?e>g^V`pjsnF){tdIB<(p$6L!2fZ7YRq z1LGwT%I+2LzehQKDtH{|RR{lll zJP$evf^KPZ(2BL8`5Nfrf~db@P4u@EWSzWtek&dT@~LNDCIj)c*6U}Dore#x$JAUL zlE&FH>fg>gr>5LE#smrk%;xrkzh(dSR-Ec2%ipl+Q!I$^)QH78izitKZ#PjxARikR zLUGn|5BsnHW3?>(ceeW_7JtWfr4Sn(;Dp8fFq}c_|G~Q)iEL~@2r<6Ok;sJqGRb^@ z?oYF*iI`^fb<$iBWo>8=KI-UXNigYYP7@Ji;4BmVLe?NvFtU%Jrp>Me=>s5sr(CEZ ze#QL}^wD`AZUtLHYAMaOJRPfRu8!fPD}`MRYm9R@$1# z+X+P8B`B5!TDm@GylK2?Vz)mW#+$yTV!Z4_V)3`RTcD^j6!%C8?er_0hjkQUkID#@ zAon4=8gRx=1~{41t(Mj_(7;uN*?1bK+|!=aU1Fi0#l46tP3yFN)H;#*xl8hpJj^#{ z>PQBI825~IF41eaD-!K;}KOyr6Y~#%K zyG&=<(UBWVN90Ag&!B5b2q57q!k>{aV!>}b)Dj96pPUEW#staT!X}II2pbG%?&@6t zqQkGRq6@?q9p_t*KI7E3_|$TEFGD_^N}~{~uiDHeR%`sLd;tM_!Jx z>w5OKyvlJ5un$#p2_o%?D{=A1D%KfC?D*6#$}Kfg9>$|pPjNQ=k$B?`IJ5&K1taz6 zcuNBjd-31NH&l@Wgom(wi#xZ*F*Ua+9*4b&miA}8{SZHx)9E;;Ob7cn*347TF5Jyg zmIK!Po7JP%efDgtfahkvWmG6V+c>b@TsRT|NroFCg!m=MB7RLEa4%!?z)HTERP) zKxa602^!IqQOxxjLTXceMs1NTO&0g@4>t&{{Tc25Y+h&UlpG^f(0o;+kxOD3FX$+k zrOl*RMy$TE^)r9;$Tv#o7M~HTvjGu|;-m%JI|IZbzq$Q!*<~pSwxr2KS+Y^rFlfF!RW_%}unw{s2ucp;S4qwS$3um`+9ycXw+(~#Z)sZpI{YiN|~_e6XP z81+>gf-S7qWq%fA`vcaxM0#gP!pBuj5XCCwodGzI-I*kY#nzR1rM7r&r){k%)U z9Twmp_)v3vfo*Azk-Wj;eJlv}^RnIor8B8|Ul*fVT8I85Yt$$Qo4Rs*A|mHhVEhlF zSS@lT_+bx@;Qa-9c!-}(rRE!sxola-sKpUeyt9IQi+7r!s(h!ms@QhpVf8vrqIIzi zI4=O~;{Vgq+(|q5=LH$JhyRHp+tMr^+IJ>{{R2L14My@2jivc6hgnJ*tVVF9`Ye8o zjmxq(5EvNUCyGccKYf^l&?f$wj2j~KcDzI!Hy!RFM$0{&uaf_7(tF|7LDs(DORULM z62?eaO^3q1#LKU-_+u9Q^h4%f%!9TJ=ka=E!=Aw$8p$~nKK@66e`SQgc&25LKWo`u V4}5>ERxj6Is4v!^s-It8`@i}8aTovq diff --git a/train_pretrain.py b/train_pretrain.py index 56e41e5..af1b871 100644 --- a/train_pretrain.py +++ b/train_pretrain.py @@ -59,16 +59,17 @@ def train_epoch(epoch, wandb): total_steps = iter_per_epoch * args.epochs warmup_steps = int(total_steps * 0.005) decay_steps = int(total_steps * 0.9) + + if epoch == 0: + for param_group in optimizer.param_groups: # 在此处设置学习率 + param_group['lr'] = 5e-6 + start_time = time.time() for step, (X, Y, loss_mask) in enumerate(train_loader): X = X.to(args.device) Y = Y.to(args.device) loss_mask = loss_mask.to(args.device) - lr = get_dynamic_lr(step_base + step, warmup_steps, decay_steps, args.learning_rate) - for param_group in optimizer.param_groups: - param_group['lr'] = lr - with ctx: res = model(X) loss = loss_fct( @@ -82,6 +83,11 @@ def train_epoch(epoch, wandb): scaler.scale(loss).backward() if (step + 1) % args.accumulation_steps == 0: + current_step = step_base + step # 计算当前总步数 + lr = get_dynamic_lr(current_step, warmup_steps, decay_steps, args.learning_rate) + for param_group in optimizer.param_groups: # 在此处设置学习率 + param_group['lr'] = lr + scaler.unscale_(optimizer) torch.nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip) @@ -140,25 +146,6 @@ def save_checkpoint(buffer: io.BytesIO, path): f.write(buffer.getvalue()) -def init_model(cli_args): - lm_config = LMConfig( - dim=cli_args.dim, - n_layers=cli_args.n_layers, - n_heads=cli_args.n_heads, - n_kv_heads=cli_args.n_kv_heads, - max_seq_len=cli_args.max_seq_len, - use_moe=cli_args.use_moe, - use_mla=cli_args.use_mla, - torch_dtype=cli_args.dtype, - ) - tokenizer = AutoTokenizer.from_pretrained(f'./model/{cli_args.vocab}_tokenizer') - # tokenizer = AutoTokenizer.from_pretrained('./deepseek_v3_tokenizer') - lm_config.vocab_size = tokenizer.vocab_size - model = MiniMindLM(lm_config) - Logger(f'LLM总参数量:{sum(p.numel() for p in model.parameters() if p.requires_grad) / 1e6:.3f} 百万') - return model, tokenizer - - def init_distributed_mode(): if not ddp: return global ddp_local_rank, DEVICE @@ -173,6 +160,9 @@ def init_distributed_mode(): # torchrun --nproc_per_node 2 1-pretrain.py if __name__ == "__main__": + # 启用动态显存段扩展 + os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True" + parser = argparse.ArgumentParser(description="MiniMind Pretraining") parser.add_argument("--save_dir", type=str, default="out") # 若要以最快速度实现zero则epochs设置为1轮;否则应当利用有限的数据训练2~6个epochs。 @@ -233,7 +223,21 @@ if __name__ == "__main__": else: wandb = None - model, tokenizer = init_model(args) + lm_config = LMConfig( + dim=args.dim, + n_layers=args.n_layers, + n_heads=args.n_heads, + n_kv_heads=args.n_kv_heads, + max_seq_len=args.max_seq_len, + use_moe=args.use_moe, + use_mla=args.use_mla, + torch_dtype=args.dtype, + ) + tokenizer = AutoTokenizer.from_pretrained(f'./model/{args.vocab}_tokenizer') + lm_config.vocab_size = tokenizer.vocab_size + model = MiniMindLM(lm_config) + Logger(f'LLM总参数量:{sum(p.numel() for p in model.parameters() if p.requires_grad) / 1e6:.3f} 百万') + if args.ckpt_path is not None: # 接着训练 state_dict = torch.load(args.ckpt_path, map_location=args.device)