From c3199c72406e578aefaec734b0621d80a823b410 Mon Sep 17 00:00:00 2001 From: Justin Guese Date: Thu, 3 Sep 2026 15:13:29 +0700 Subject: [PATCH 1/2] =?UTF-8?q?[update]=20=E6=8A=8A=20num=5Fexperts=20/=20?= =?UTF-8?q?num=5Fexperts=5Fper=5Ftok=20=E6=9A=B4=E9=9C=B2=E5=88=B0?= =?UTF-8?q?=E5=91=BD=E4=BB=A4=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 此前要改专家数只能去编辑 model_minimind.py。三个入口 （train_pretrain / train_full_sft / eval_llm）的默认值与 MiniMindConfig 一致（4 / 1），默认行为不变；RL 与 agent 相关的 trainer 不动。 --- eval_llm.py | 4 ++++ trainer/train_full_sft.py | 5 ++++- trainer/train_pretrain.py | 5 ++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/eval_llm.py b/eval_llm.py index 9359ca8..c7a11a8 100755 --- a/eval_llm.py +++ b/eval_llm.py @@ -16,6 +16,8 @@ def init_model(args): hidden_size=args.hidden_size, num_hidden_layers=args.num_hidden_layers, use_moe=bool(args.use_moe), + num_experts=args.num_experts, + num_experts_per_tok=args.num_experts_per_tok, inference_rope_scaling=args.inference_rope_scaling )) moe_suffix = '_moe' if args.use_moe else '' @@ -38,6 +40,8 @@ def main(): parser.add_argument('--hidden_size', default=768, type=int, help="隐藏层维度") parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量") parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构（0=否，1=是）") + parser.add_argument('--num_experts', default=4, type=int, help="MoE专家数量（仅use_moe=1时生效，需与训练时一致）") + parser.add_argument('--num_experts_per_tok', default=1, type=int, help="每个token路由到的专家数（仅use_moe=1时生效，需与训练时一致）") parser.add_argument('--inference_rope_scaling', default=False, action='store_true', help="启用RoPE位置编码外推（4倍，仅解决位置编码问题）") parser.add_argument('--max_new_tokens', default=8192, type=int, help="最大生成长度（注意：并非模型实际长文本能力）") parser.add_argument('--temperature', default=0.85, type=float, help="生成温度，控制随机性（0-1，越大越随机）") diff --git a/trainer/train_full_sft.py b/trainer/train_full_sft.py index 7584565..cd9f763 100644 --- a/trainer/train_full_sft.py +++ b/trainer/train_full_sft.py @@ -99,6 +99,8 @@ if __name__ == "__main__": parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量") parser.add_argument('--max_seq_len', default=768, type=int, help="训练的最大截断长度（中文1token≈1.5~1.7字符）") parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构（0=否，1=是）") + parser.add_argument('--num_experts', default=4, type=int, help="MoE专家数量（仅use_moe=1时生效）") + parser.add_argument('--num_experts_per_tok', default=1, type=int, help="每个token路由到的专家数（仅use_moe=1时生效）") parser.add_argument("--data_path", type=str, default="../dataset/sft_t2t_mini.jsonl", help="训练数据路径") parser.add_argument('--from_weight', default='pretrain', type=str, help="基于哪个权重训练，为none则不基于任何权重训练") parser.add_argument('--from_resume', default=0, type=int, choices=[0, 1], help="是否自动检测&续训（0=否，1=是）") @@ -114,7 +116,8 @@ if __name__ == "__main__": # ========== 2. 配置目录、模型参数、检查ckp ========== os.makedirs(args.save_dir, exist_ok=True) - lm_config = MiniMindConfig(hidden_size=args.hidden_size, num_hidden_layers=args.num_hidden_layers, use_moe=bool(args.use_moe)) + lm_config = MiniMindConfig(hidden_size=args.hidden_size, num_hidden_layers=args.num_hidden_layers, use_moe=bool(args.use_moe), + num_experts=args.num_experts, num_experts_per_tok=args.num_experts_per_tok) ckp_data = lm_checkpoint(lm_config, weight=args.save_weight, save_dir='../checkpoints') if args.from_resume==1 else None # ========== 3. 设置混合精度 ========== diff --git a/trainer/train_pretrain.py b/trainer/train_pretrain.py index 50ba878..1994296 100644 --- a/trainer/train_pretrain.py +++ b/trainer/train_pretrain.py @@ -98,6 +98,8 @@ if __name__ == "__main__": parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量") parser.add_argument('--max_seq_len', default=340, type=int, help="训练的最大截断长度（中文1token≈1.5~1.7字符）") parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构（0=否，1=是）") + parser.add_argument('--num_experts', default=4, type=int, help="MoE专家数量（仅use_moe=1时生效）") + parser.add_argument('--num_experts_per_tok', default=1, type=int, help="每个token路由到的专家数（仅use_moe=1时生效）") parser.add_argument("--data_path", type=str, default="../dataset/pretrain_t2t_mini.jsonl", help="预训练数据路径") parser.add_argument('--from_weight', default='none', type=str, help="基于哪个权重训练，为none则从头开始") parser.add_argument('--from_resume', default=0, type=int, choices=[0, 1], help="是否自动检测&续训（0=否，1=是）") @@ -113,7 +115,8 @@ if __name__ == "__main__": # ========== 2. 配置目录、模型参数、检查ckp ========== os.makedirs(args.save_dir, exist_ok=True) - lm_config = MiniMindConfig(hidden_size=args.hidden_size, num_hidden_layers=args.num_hidden_layers, use_moe=bool(args.use_moe)) + lm_config = MiniMindConfig(hidden_size=args.hidden_size, num_hidden_layers=args.num_hidden_layers, use_moe=bool(args.use_moe), + num_experts=args.num_experts, num_experts_per_tok=args.num_experts_per_tok) ckp_data = lm_checkpoint(lm_config, weight=args.save_weight, save_dir='../checkpoints') if args.from_resume==1 else None # ========== 3. 设置混合精度 ========== From fc585a9bf98947f4eaeb529c9abe8af1d3590462 Mon Sep 17 00:00:00 2001 From: Justin Guese Date: Thu, 3 Sep 2026 15:36:14 +0700 Subject: [PATCH 2/2] =?UTF-8?q?[update]=20=E6=8A=8A=E9=9A=8F=E6=9C=BA?= =?UTF-8?q?=E7=A7=8D=E5=AD=90=E6=9A=B4=E9=9C=B2=E5=88=B0=E5=91=BD=E4=BB=A4?= =?UTF-8?q?=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit trainer 里的种子此前硬编码为 42（setup_seed(42 + rank) 与每轮的 setup_seed(42 + epoch)），要跑多种子实验只能改源码。新增 --seed， 默认 42，与改动前完全一致。 --- trainer/train_full_sft.py | 5 +++-- trainer/train_pretrain.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/trainer/train_full_sft.py b/trainer/train_full_sft.py index cd9f763..b080aef 100644 --- a/trainer/train_full_sft.py +++ b/trainer/train_full_sft.py @@ -101,6 +101,7 @@ if __name__ == "__main__": parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构（0=否，1=是）") parser.add_argument('--num_experts', default=4, type=int, help="MoE专家数量（仅use_moe=1时生效）") parser.add_argument('--num_experts_per_tok', default=1, type=int, help="每个token路由到的专家数（仅use_moe=1时生效）") + parser.add_argument('--seed', default=42, type=int, help="随机种子（DDP下每个rank为seed+rank，每轮为seed+epoch）") parser.add_argument("--data_path", type=str, default="../dataset/sft_t2t_mini.jsonl", help="训练数据路径") parser.add_argument('--from_weight', default='pretrain', type=str, help="基于哪个权重训练，为none则不基于任何权重训练") parser.add_argument('--from_resume', default=0, type=int, choices=[0, 1], help="是否自动检测&续训（0=否，1=是）") @@ -112,7 +113,7 @@ if __name__ == "__main__": # ========== 1. 初始化环境和随机种子 ========== local_rank = init_distributed_mode() if dist.is_initialized(): args.device = f"cuda:{local_rank}" - setup_seed(42 + (dist.get_rank() if dist.is_initialized() else 0)) + setup_seed(args.seed + (dist.get_rank() if dist.is_initialized() else 0)) # ========== 2. 配置目录、模型参数、检查ckp ========== os.makedirs(args.save_dir, exist_ok=True) @@ -160,7 +161,7 @@ if __name__ == "__main__": # ========== 8. 开始训练 ========== for epoch in range(start_epoch, args.epochs): train_sampler and train_sampler.set_epoch(epoch) - setup_seed(42 + epoch); indices = torch.randperm(len(train_ds)).tolist() + setup_seed(args.seed + epoch); indices = torch.randperm(len(train_ds)).tolist() skip = start_step if (epoch == start_epoch and start_step > 0) else 0 batch_sampler = SkipBatchSampler(train_sampler or indices, args.batch_size, skip) loader = DataLoader(train_ds, batch_sampler=batch_sampler, num_workers=args.num_workers, pin_memory=True) diff --git a/trainer/train_pretrain.py b/trainer/train_pretrain.py index 1994296..42384c8 100644 --- a/trainer/train_pretrain.py +++ b/trainer/train_pretrain.py @@ -100,6 +100,7 @@ if __name__ == "__main__": parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构（0=否，1=是）") parser.add_argument('--num_experts', default=4, type=int, help="MoE专家数量（仅use_moe=1时生效）") parser.add_argument('--num_experts_per_tok', default=1, type=int, help="每个token路由到的专家数（仅use_moe=1时生效）") + parser.add_argument('--seed', default=42, type=int, help="随机种子（DDP下每个rank为seed+rank，每轮为seed+epoch）") parser.add_argument("--data_path", type=str, default="../dataset/pretrain_t2t_mini.jsonl", help="预训练数据路径") parser.add_argument('--from_weight', default='none', type=str, help="基于哪个权重训练，为none则从头开始") parser.add_argument('--from_resume', default=0, type=int, choices=[0, 1], help="是否自动检测&续训（0=否，1=是）") @@ -111,7 +112,7 @@ if __name__ == "__main__": # ========== 1. 初始化环境和随机种子 ========== local_rank = init_distributed_mode() if dist.is_initialized(): args.device = f"cuda:{local_rank}" - setup_seed(42 + (dist.get_rank() if dist.is_initialized() else 0)) + setup_seed(args.seed + (dist.get_rank() if dist.is_initialized() else 0)) # ========== 2. 配置目录、模型参数、检查ckp ========== os.makedirs(args.save_dir, exist_ok=True) @@ -159,7 +160,7 @@ if __name__ == "__main__": # ========== 8. 开始训练 ========== for epoch in range(start_epoch, args.epochs): train_sampler and train_sampler.set_epoch(epoch) - setup_seed(42 + epoch); indices = torch.randperm(len(train_ds)).tolist() + setup_seed(args.seed + epoch); indices = torch.randperm(len(train_ds)).tolist() skip = start_step if (epoch == start_epoch and start_step > 0) else 0 batch_sampler = SkipBatchSampler(train_sampler or indices, args.batch_size, skip) loader = DataLoader(train_ds, batch_sampler=batch_sampler, num_workers=args.num_workers, pin_memory=True)