From 6fc918beb68a0d8c40452338df6319fe168014ba Mon Sep 17 00:00:00 2001 From: jingyaogong Date: Fri, 4 Sep 2026 22:16:38 +0800 Subject: [PATCH] [update] remove expert args --- eval_llm.py | 4 ---- trainer/train_full_sft.py | 5 +---- trainer/train_pretrain.py | 5 +---- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/eval_llm.py b/eval_llm.py index c7a11a8..9359ca8 100755 --- a/eval_llm.py +++ b/eval_llm.py @@ -16,8 +16,6 @@ def init_model(args): hidden_size=args.hidden_size, num_hidden_layers=args.num_hidden_layers, use_moe=bool(args.use_moe), - num_experts=args.num_experts, - num_experts_per_tok=args.num_experts_per_tok, inference_rope_scaling=args.inference_rope_scaling )) moe_suffix = '_moe' if args.use_moe else '' @@ -40,8 +38,6 @@ def main(): parser.add_argument('--hidden_size', default=768, type=int, help="隐藏层维度") parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量") parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构（0=否，1=是）") - parser.add_argument('--num_experts', default=4, type=int, help="MoE专家数量（仅use_moe=1时生效，需与训练时一致）") - parser.add_argument('--num_experts_per_tok', default=1, type=int, help="每个token路由到的专家数（仅use_moe=1时生效，需与训练时一致）") parser.add_argument('--inference_rope_scaling', default=False, action='store_true', help="启用RoPE位置编码外推（4倍，仅解决位置编码问题）") parser.add_argument('--max_new_tokens', default=8192, type=int, help="最大生成长度（注意：并非模型实际长文本能力）") parser.add_argument('--temperature', default=0.85, type=float, help="生成温度，控制随机性（0-1，越大越随机）") diff --git a/trainer/train_full_sft.py b/trainer/train_full_sft.py index b080aef..c22eb80 100644 --- a/trainer/train_full_sft.py +++ b/trainer/train_full_sft.py @@ -99,8 +99,6 @@ if __name__ == "__main__": parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量") parser.add_argument('--max_seq_len', default=768, type=int, help="训练的最大截断长度（中文1token≈1.5~1.7字符）") parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构（0=否，1=是）") - parser.add_argument('--num_experts', default=4, type=int, help="MoE专家数量（仅use_moe=1时生效）") - parser.add_argument('--num_experts_per_tok', default=1, type=int, help="每个token路由到的专家数（仅use_moe=1时生效）") parser.add_argument('--seed', default=42, type=int, help="随机种子（DDP下每个rank为seed+rank，每轮为seed+epoch）") parser.add_argument("--data_path", type=str, default="../dataset/sft_t2t_mini.jsonl", help="训练数据路径") parser.add_argument('--from_weight', default='pretrain', type=str, help="基于哪个权重训练，为none则不基于任何权重训练") @@ -117,8 +115,7 @@ if __name__ == "__main__": # ========== 2. 配置目录、模型参数、检查ckp ========== os.makedirs(args.save_dir, exist_ok=True) - lm_config = MiniMindConfig(hidden_size=args.hidden_size, num_hidden_layers=args.num_hidden_layers, use_moe=bool(args.use_moe), - num_experts=args.num_experts, num_experts_per_tok=args.num_experts_per_tok) + lm_config = MiniMindConfig(hidden_size=args.hidden_size, num_hidden_layers=args.num_hidden_layers, use_moe=bool(args.use_moe)) ckp_data = lm_checkpoint(lm_config, weight=args.save_weight, save_dir='../checkpoints') if args.from_resume==1 else None # ========== 3. 设置混合精度 ========== diff --git a/trainer/train_pretrain.py b/trainer/train_pretrain.py index 42384c8..83a0382 100644 --- a/trainer/train_pretrain.py +++ b/trainer/train_pretrain.py @@ -98,8 +98,6 @@ if __name__ == "__main__": parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量") parser.add_argument('--max_seq_len', default=340, type=int, help="训练的最大截断长度（中文1token≈1.5~1.7字符）") parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构（0=否，1=是）") - parser.add_argument('--num_experts', default=4, type=int, help="MoE专家数量（仅use_moe=1时生效）") - parser.add_argument('--num_experts_per_tok', default=1, type=int, help="每个token路由到的专家数（仅use_moe=1时生效）") parser.add_argument('--seed', default=42, type=int, help="随机种子（DDP下每个rank为seed+rank，每轮为seed+epoch）") parser.add_argument("--data_path", type=str, default="../dataset/pretrain_t2t_mini.jsonl", help="预训练数据路径") parser.add_argument('--from_weight', default='none', type=str, help="基于哪个权重训练，为none则从头开始") @@ -116,8 +114,7 @@ if __name__ == "__main__": # ========== 2. 配置目录、模型参数、检查ckp ========== os.makedirs(args.save_dir, exist_ok=True) - lm_config = MiniMindConfig(hidden_size=args.hidden_size, num_hidden_layers=args.num_hidden_layers, use_moe=bool(args.use_moe), - num_experts=args.num_experts, num_experts_per_tok=args.num_experts_per_tok) + lm_config = MiniMindConfig(hidden_size=args.hidden_size, num_hidden_layers=args.num_hidden_layers, use_moe=bool(args.use_moe)) ckp_data = lm_checkpoint(lm_config, weight=args.save_weight, save_dir='../checkpoints') if args.from_resume==1 else None # ========== 3. 设置混合精度 ==========