From be979ec9e7cee39074830f80c0889a311a1648d0 Mon Sep 17 00:00:00 2001
From: Ltimbe <69570084+Litmeb@users.noreply.github.com>
Date: Fri, 5 Dec 2025 21:13:42 +0800
Subject: [PATCH] Modify LoRA application conditions in model_lora.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

原本的lora只作用于方阵上，也就是q_proj和o_proj。修改完会同时作用于qkvo
---
 model/model_lora.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/model/model_lora.py b/model/model_lora.py
index b7c1d4c..8c0c8f0 100644
--- a/model/model_lora.py
+++ b/model/model_lora.py
@@ -20,8 +20,8 @@ class LoRA(nn.Module):
 
 def apply_lora(model, rank=8):
     for name, module in model.named_modules():
-        if isinstance(module, nn.Linear) and module.weight.shape[0] == module.weight.shape[1]:
-            lora = LoRA(module.weight.shape[0], module.weight.shape[1], rank=rank).to(model.device)
+        if 'lora' not in name and ('q_proj' in name or 'k_proj' in name or 'v_proj' in name or 'o_proj' in name):
+            lora = LoRA(module.weight.shape[1], module.weight.shape[0], rank=rank).to(model.device)
             setattr(module, "lora", lora)
             original_forward = module.forward