From 9faf7a6fea23aba89ec2ceed2b67c4c3fc69d0ce Mon Sep 17 00:00:00 2001 From: whitesword <1790513390@qq.com> Date: Tue, 23 Dec 2025 20:10:01 +0800 Subject: [PATCH] feat: implement merge_lora function --- .gitignore | 3 ++- eval_llm.py | 3 +-- model/model_lora.py | 16 ++++++++++++++++ .../__pycache__/trainer_utils.cpython-312.pyc | Bin 0 -> 8352 bytes 4 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 trainer/__pycache__/trainer_utils.cpython-312.pyc diff --git a/.gitignore b/.gitignore index aee52b1..ae53ad5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ model/__pycache__ out website/ -docs-minimind/ \ No newline at end of file +docs-minimind/ +MiniMind2 \ No newline at end of file diff --git a/eval_llm.py b/eval_llm.py index 8af17d4..6955697 100755 --- a/eval_llm.py +++ b/eval_llm.py @@ -22,8 +22,7 @@ def init_model(args): ckp = f'./{args.save_dir}/{args.weight}_{args.hidden_size}{moe_suffix}.pth' model.load_state_dict(torch.load(ckp, map_location=args.device), strict=True) if args.lora_weight != 'None': - apply_lora(model) - load_lora(model, f'./{args.save_dir}/lora/{args.lora_weight}_{args.hidden_size}.pth') + merge_lora(model, f'./{args.save_dir}/lora/{args.lora_weight}_{args.hidden_size}.pth') else: model = AutoModelForCausalLM.from_pretrained(args.load_from, trust_remote_code=True) print(f'MiniMind模型参数: {sum(p.numel() for p in model.parameters()) / 1e6:.2f} M(illion)') diff --git a/model/model_lora.py b/model/model_lora.py index c675f3b..1d4bf60 100644 --- a/model/model_lora.py +++ b/model/model_lora.py @@ -42,6 +42,22 @@ def load_lora(model, path): module.lora.load_state_dict(lora_state) +def merge_lora(model, path): + state_dict = torch.load(path, map_location=model.device) + # 移除可能的module前缀,确保key与模型层级名称一致 + state_dict = {(k[7:] if k.startswith('module.') else k): v for k, v in state_dict.items()} + + for name, module in model.named_modules(): + if isinstance(module, nn.Linear): + key_A = f"{name}.lora.A.weight" + key_B = f"{name}.lora.B.weight" + if key_A in state_dict and key_B in state_dict: + # 直接合并权重: W_new = W_old + B @ A + module.weight.data += state_dict[key_B] @ state_dict[key_A] + + return model + + def save_lora(model, path): state_dict = {} for name, module in model.named_modules(): diff --git a/trainer/__pycache__/trainer_utils.cpython-312.pyc b/trainer/__pycache__/trainer_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c1215376254b0120df9ac1f2a85b279085062a93 GIT binary patch literal 8352 zcmb_BZBQF$db`p}TJ(X$hY$iA<1dU12pcy@=%LS=zqh!KJz z8BLHJA>~ptqLEAOh*mDC5lSv~BYL?sj2PsS4pod8Njz5@G=KGeizDkZbm@38sWxa+NAehRl z1Y>2*?`oAZk2o)DT-C7#Xc<5M%U}HV?|yXu{Xe_^C-2|C_Q7X=y7a}(AKy^o+}BTcMq{Hvzpry@!q2nZR0J`d<9>c3HX22pA(#q3 zX5estCy%`TFpJdE6XX3s&KN*YT?J>49;t)%Npue zX-i`KwaN!tf_5tBE9?hQH2mcA1);J)2r`>9+DDU?2UsNq%8*hZrr|mXQi6fkHRDmdHD%R>x*!bChCU{voD4ae6@$;*{EdV zBfK{#*8w1iIQ#|8%_=r9X9N*ygia^?xP6e#5P6d&!6i?p_P)ixbLqwdaoXR3Pz8Sq0WmFJZ2p^1uE5Au?$od z>uRl(Jq3J64Z{-k{lmk9U;Gy&`MOF}*yjskHXZf)K(a8Bap>8jhle~E%vB+2fr%vz zkk!D3FZxj=ENOvS2qQ?U#@AcDd!BI=iA5zHA3?qeiSor5uVmzaunc?A?_*&HV=&?a zCcq+~^e))tRj|lF{?QoEGM-R`VNo0Guo-^b2xRzb7-zfYy03T7^(im_GIq)wWty`?K#ajLeUu&7!?yrOG{XVwI}6cH+v39OV=# zXKHtr+L<@mGn8HC8HwUpmMJd=ezJjc%1O>rJoAMM7J!3qOo7s?ygvAi*#*uE;ETQ? zeFGBnww%gwT%f>@DimDs0xtm%K6*BmqZ?A`Y=$hEUyc{C4Bn7Dg%dHF|HJIUt-1{2 zX880mmr2qCCoz!_YKNU9O*o2n;fg8b4acyo0g58HorGj9>G61jL5W^xbxzU&n})-Z zg<*LXVU@%2exFn^%7%RtAupPAk@D)L?236h7FCvi9G1tMa#+3sk5a~KgI5NVRF<-@ zQu=GpTzTf&@Ri}@{w!4w{FkHbB4tl9S*o$XjSV8zkb?2%LQj)OHKq70wIlE9xZ02B z=csy-s?Sm_`JKBP18VMXyP8uQ&qYaY` z6|WEsHwi{3kdW&Ew!lNM-+H=92o*4Dc-s&+erISJpiP1%NnZJysbGvjDx`~p9mW^9 zP|yfwRraC=LGxYZpQ^IMR+v*pKC0t!i(m0^%FkL`5gpt4)j30 zGHw+r1#7^*aaPeb3OK%f?5CR97K(^lsz(Bw ziZE(TunEQ@EJMVr1U)*)Zz+uHnIcZN3HrcRb);CT-yg3Qs*#1?u8s)RjD;aodlq;D zRu7ks>r2OBXN#bH6#fND(D6If)dMYRtyrojLA51t_;zndDjb}tQmA6AZ|UPT0{v*( zp8vr*MQh`B!9GLIR+M1L&J-|?dP_LcUc!-0g8hHXkwT47+XH8xeG2A2f-?(nrv!VU z{vt&Prge^~`yV)_MyM-tL5Y!Aj$`TsQxWzh%S$zUn~L557P1?XifiD+dUYt^R%^vF zu(w!OYsI;ji4xJXPJUR=+Cl5FCLgH-JEs zsbMM$F!d!I_W#6*ai>r(I42Lld3Op9rWRjvp{w04{Vm%|69txp@b#w(`?SN-s5Ftlj)5-2iHWKyEQmSQDBB0DA0 z^EKs;@)Of04`R7kh{X-XVhXhZ$?;yE^)P-P56(g~ zc8iqEA#c=!EkoWP38Q{k9Q8pK!zRSQb7!zed~NRjTuN$+R7rHmJIOMB1jcKOV`WsBCj1P;hLv+w z$+oIH9Ck<7zZdo5c}73y55q~rK33AhXU4$&9>4=S@KZyaL``^uW9T^aC?*HRc`ve( z0kP4b7p52{yquTk5qb*G!MKxK*s?=dRZEo^ohA6ujQg;c2nhG_ipgSPLs9;+$LIA; zuo4x;zLj1kDr_E%co~F+rR+Vr8l;L~$OAs+n15W-PqF^-2_7BB!$)v-6lWNrl1`?A zMB~E(1(2klobsY^4$u@~J@Ov3Z!)UXObU<44H+p40?0{;ya;CnFyISx46mz2z!)k# z1I#07C)vxA?xHssVjLIK^>TDaojrG6xO3tzld%p0h<$VNM9Q1ir!J*0EIM&=ga`tx7-k!C)Go$0VQ6!F{l~F!n%RUX#cp>RcRlQSpqwZ$?JDYB7y4fVw?aJ5(5=PNJ0LM%{FII0) z7$6WZXS{AqcBiJY^sZIfD7Oxzj%Vqe`5H%p{*B4DYOPDgQvR&9HDA}b5SkDDG?LW* z=0Nvb2h#f%sYNE+xbH4CPv2|YmVRt;#}c*D_*jyDpeO2@W$bi_&W;slXWrhBudQF* zcObX#n7Hp)cHi;0cH|m5#D3* z{j_aQuI-@Mc5tQbklMAYJvo@xF4yjOPz5VLs3q#QrXLsWu7puV#KE*NOSh?r=uZ7H zwDRV7}yT2>vK7N+N?(`|S9vyLN)<9WxH z1z}!D4=rt7aqLeVU+sE4*VQj}^=G>V-m1+x+eK%4*6IH6dx;Zyt0UFDY~7jP-nMv5 z+`eyVLfqbacel8GC`qU4L|gN!tuEtqEw(N;EbUvGUJ7Q8oXebNGh;#Vd?<4+oN+|5 zwrIYlG4X8PTJzP~S(2zbPIB0Y{>N6=V3*zp7JDIKL4l>++QskUe`-{%5ZCdARf-;4 zTrWUWRMtP=D{r@hjjsWlNgFo^TF^L@Y~$+y7N2870vjLu0eZ07Y2Ks`1`5#$wFmSX zEl{@&1dRT?0;NI-x-ktzORNwr2}dCEh)o96jWevE5mwdc5YC=~>=dnt-~r#H*d8Clo6d*lEP*Xu3}b1;d+F2(1>f6frg zdnfObnX~6|X9Fu|14~=q-}P6!-gk+6C%@t_8@*iRB7bIgk>gM&X0EyqPix)##$s46 zHvjg<-`~4)&1FUuv?S{oii9Ar1c4?T`DGC1X*7hhlQ`Rjv-6NiR;&#nuBciSvQCt= zVBbhatkgr;v4@Z&A8{EG4uVT2d2K~cN|n4CO; zFG(h40YzCFlm^G)AVpDoDGJM2L?F5b<9_&Y{{R_iE8FIrtwXeREQVKXhi0BxHE&5* zteAJt^nYroPCmbEXv}YE&28DUyk*b)^GlnZ3t9 zZ1~WeIsHPW?!|;@wc4?;dwzH7?9#<-b#KCur!6^pvq*1FjW4npdUKZEe~&U|EFHIf zK$*K|a{Xt-{xd86KgjN#Tyalk!WTsMg$#;iIxc1|#xtguveXarCfnQ-*PlopTQN1J zUd)(UGE~bS23s));k4Ea*#f~5@DxM9GmMTQ;VDGRXyCDgVziL!E>mDZ+D=XSqer~F zZ({w4!-t8lCcK&~0I^2_w}KQ1k2(fXZ)kyFRFa^DmbyD|y~6QY$-pU3Cz5d#@1hue zW)61s$Q>-_qHyRBjJa1~B{Yk(*C6BYK`Ale?4o+)>Y`91#7lnQT2$Xa(If33l z+9*6VY5mNlGBgBH@NG1dhvKjnhZmHkS>9Hi(B-RZ;0o0^7JBA;u)))qIGQ(C%?({2 zO4Ve|n^&#X$;V!uPMv72_n2 zMYfz64ZfG<_y(_`s&r7LKvV2DD*mPpy^iaJJ11#F-b?bugU2(Opcdu5X#JL!>L$tH z@c<719qY^7MR^2YV-EXXAYq-GfHvoDoLh13UOfGt=axrw_T0H3I-kCK`ky?%_K5YT z6BbOwy~$9T$+a91;lJ*{iuH*M{e&#x;0abav|a)+5C(RbXlScQX0*zr@28kYv z1tF3>#t*Mw2~5!8=}Nb|v*-h1Ax5jZa+-!QSzAx*p%#fL4kdVrfHx-M^LWq;u!ig( zBP>}FHmGD{AbJ^f;ts4_rFGZ1