From 62f47fb844cff587d5effd54dedf05c2cbb5ac9d Mon Sep 17 00:00:00 2001 From: MiniMind-V User Date: Fri, 6 Mar 2026 04:49:40 +0000 Subject: [PATCH] [fix] LoRA/RAG: correct target modules, vlm_checkpoint call, and gitignore - lora_utils.py: fix target_modules to match MiniMind arch (q/k/v/o_proj instead of GPT-2 style c_attn/c_proj/c_fc) - train_sft_vlm_lora.py: fix broken vlm_checkpoint call in train_epoch (was passing model/epoch/step/args instead of vlm_config); now saves inline like train_sft_vlm.py; remove unused SkipBatchSampler import - .gitignore: add __pycache__, .claude/, dataset/sft_vlm.parquet --- .gitignore | 21 +++++++++++++++++- trainer/lora_utils.py | 7 +++--- trainer/train_sft_vlm_lora.py | 41 ++++++++++++++++++++++------------- 3 files changed, 50 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index 69225a3..974276b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,23 @@ model/__pycache__ out website/ docs-minimind-v/ -测试.py \ No newline at end of file +测试.py + +# Jupyter checkpoints +.ipynb_checkpoints/ +**/.ipynb_checkpoints/ + +# Large model weights and pretrained models +MiniMind2-V/ +llm_512.pth +model/vision_model/ + +# Python cache +__pycache__/ +*.pyc + +# Claude config +.claude/ + +# Large dataset files +dataset/sft_vlm.parquet \ No newline at end of file diff --git a/trainer/lora_utils.py b/trainer/lora_utils.py index 734c633..156bf06 100644 --- a/trainer/lora_utils.py +++ b/trainer/lora_utils.py @@ -37,9 +37,10 @@ def get_lora_config( # These are the attention projection layers if target_modules is None: target_modules = [ - "c_attn", # Combined QKV projection in attention - "c_proj", # Output projection in attention - "c_fc", # First linear in MLP (optional, but helps) + "q_proj", # Query projection in attention + "k_proj", # Key projection in attention + "v_proj", # Value projection in attention + "o_proj", # Output projection in attention ] config = LoraConfig( diff --git a/trainer/train_sft_vlm_lora.py b/trainer/train_sft_vlm_lora.py index 57707e7..6d89bfd 100644 --- a/trainer/train_sft_vlm_lora.py +++ b/trainer/train_sft_vlm_lora.py @@ -20,7 +20,7 @@ from torch.utils.data import DataLoader, DistributedSampler from transformers import AutoTokenizer from model.model_vlm import MiniMindVLM, VLMConfig from dataset.lm_dataset import VLMDataset -from trainer.trainer_utils import get_lr, Logger, is_main_process, init_distributed_mode, setup_seed, init_vlm_model, vlm_checkpoint, SkipBatchSampler +from trainer.trainer_utils import get_lr, Logger, is_main_process, init_distributed_mode, setup_seed, init_vlm_model from trainer.lora_utils import get_lora_config, apply_lora warnings.filterwarnings('ignore') @@ -54,7 +54,7 @@ def train_epoch(epoch, loader, iters, start_step=0, wandb=None): if step % args.log_interval == 0 or step == iters - 1: spend_time = time.time() - start_time - logger.log( + Logger( 'Epoch:[{}/{}]({}/{}) loss:{:.3f} lr:{:.8f} epoch_time:{}min:'.format( epoch + 1, args.epochs, @@ -72,11 +72,19 @@ def train_epoch(epoch, loader, iters, start_step=0, wandb=None): if step % args.save_interval == 0 or step == iters - 1: if is_main_process(): - vlm_checkpoint(model.module if hasattr(model, 'module') else model, - epoch, - step, - args, - suffix=f"_lora" if args.use_lora else "") + model.eval() + raw_model = model.module if hasattr(model, 'module') else model + raw_model = getattr(raw_model, '_orig_mod', raw_model) + os.makedirs(args.out_dir, exist_ok=True) + lora_suffix = '_lora' if args.use_lora else '' + ckp = f'{args.out_dir}/sft_vlm{lora_suffix}_{lm_config.hidden_size}.pth' + if args.use_lora: + raw_model.save_pretrained(ckp + '_adapters') + else: + state_dict = {k: v.half().cpu() for k, v in raw_model.state_dict().items() + if not k.startswith('vision_encoder.')} + torch.save(state_dict, ckp) + model.train() if __name__ == "__main__": @@ -95,11 +103,14 @@ if __name__ == "__main__": parser.add_argument("--log_interval", type=int, default=100, help="Log interval") parser.add_argument("--save_interval", type=int, default=1000, help="Save interval") parser.add_argument("--local_rank", type=int, default=-1, help="DDP local rank") + parser.add_argument("--ddp", action="store_true", help="Use DistributedDataParallel") + parser.add_argument("--data_path", type=str, default="./dataset/sft_vlm.parquet", help="Path to training parquet file") parser.add_argument("--use_moe", action="store_true", help="Use Mixture of Experts") parser.add_argument("--resume_step", type=int, default=0, help="Resume from step") parser.add_argument("--hidden_size", type=int, default=512, help="Hidden size") parser.add_argument("--num_heads", type=int, default=16, help="Number of attention heads") parser.add_argument("--num_layers", type=int, default=8, help="Number of layers") + parser.add_argument("--num_kv_heads", type=int, default=4, help="Number of key-value heads (GQA)") # LoRA-specific arguments parser.add_argument("--use_lora", action="store_true", help="Use LoRA for parameter-efficient training") @@ -110,9 +121,8 @@ if __name__ == "__main__": args = parser.parse_args() # Setup - init_distributed_mode(args) + init_distributed_mode() setup_seed(42) - logger = Logger(args.out_dir, is_main_process()) # Load tokenizer and model tokenizer = AutoTokenizer.from_pretrained('./model') @@ -121,12 +131,13 @@ if __name__ == "__main__": lm_config = VLMConfig( hidden_size=args.hidden_size, num_attention_heads=args.num_heads, + num_key_value_heads=args.num_kv_heads, num_hidden_layers=args.num_layers, use_moe=args.use_moe, ) # Initialize model - model = init_vlm_model(lm_config, args.device) + model, tokenizer, _ = init_vlm_model(lm_config, device=args.device, tokenizer_path='./model', vision_model_path='./model/vision_model/clip-vit-base-patch16', save_dir='./out') # Apply LoRA if requested if args.use_lora: @@ -165,7 +176,7 @@ if __name__ == "__main__": # Load dataset dataset = VLMDataset( - parquet_path='./dataset/sft_vlm.parquet', + parquet_path=args.data_path, tokenizer=tokenizer, preprocess=model.processor if not args.ddp else model.module.processor, max_length=512 @@ -203,8 +214,8 @@ if __name__ == "__main__": # Training loop iters = len(loader) - logger.log(f"Starting SFT training {'with LoRA' if args.use_lora else 'full fine-tuning'}...") - logger.log(f"Total iterations: {iters} per epoch") + Logger(f"Starting SFT training {'with LoRA' if args.use_lora else 'full fine-tuning'}...") + Logger(f"Total iterations: {iters} per epoch") for epoch in range(args.epochs): train_epoch(epoch, loader, iters, start_step=0, wandb=wandb) @@ -216,7 +227,7 @@ if __name__ == "__main__": save_path = f"{args.out_dir}/lora_adapters_sft_vlm_{args.hidden_size}" model_to_save = model.module if hasattr(model, 'module') else model model_to_save.save_pretrained(save_path) - logger.log(f"✅ LoRA adapters saved to {save_path}") + Logger(f"✅ LoRA adapters saved to {save_path}") else: vlm_checkpoint( model.module if hasattr(model, 'module') else model, @@ -229,4 +240,4 @@ if __name__ == "__main__": if wandb: wandb.finish() - logger.log("Training complete!") + Logger("Training complete!")