diff --git a/.trae/documents/完成 GRPO 与 SPO 训练类型的前端与后端支持.md b/.trae/documents/完成 GRPO 与 SPO 训练类型的前端与后端支持.md
new file mode 100644
index 0000000..f62b0d7
--- /dev/null
+++ b/.trae/documents/完成 GRPO 与 SPO 训练类型的前端与后端支持.md
@@ -0,0 +1,66 @@
+## 目标
+- 在现有 DPO/PPO 基础上，新增 GRPO 与 SPO 训练类型的完整前后端流程。
+- 选定对应训练类型后，仅显示该类型的特有参数；隐藏不需要的参数。
+- 参数默认值严格对齐 `trainer/train_grpo.py` 与 `trainer/train_spo.py`。
+- 保持原有 `trainer_web` 逻辑与交互不受影响。
+
+## 代码改动
+### 前端模板（index.html）
+- 在训练类型选择框中新增：`RL - GRPO`（value=`grpo`）、`RL - SPO`（value=`spo`）。
+- 新增两块参数卡片：
+ - GRPO 参数卡（class=`parameter-card grpo`）：
+ - `beta`（浮点输入，placeholder=0.02）
+ - `num_generations`（整数输入，placeholder=8）
+ - `reasoning`（下拉 0/1）
+ - `reward_model_path`（文本输入，placeholder=`../../internlm2-1_8b-reward`）
+ - SPO 参数卡（class=`parameter-card spo`）：
+ - `beta`（浮点输入，placeholder=0.02）
+ - `reasoning`（下拉 0/1）
+ - `reward_model_path`（文本输入，placeholder=`../../internlm2-1_8b-reward`）
+- 保持现有 DPO/PPO 参数卡结构不变。
+
+### 前端交互（static/js/train/form.js）
+- 在 `onTrainTypeChange` 中：
+ - 为 GRPO/SPO 添加显示/隐藏逻辑：
+ - `.grpo` 与 `.spo` 卡片只在对应训练类型显示。
+ - `pretrain-sft` 区域显示范围扩展到 `grpo` 与 `spo`（需要 `save_weight` 字段）。
+ - `from-weight` 区域在 `ppo/grpo/spo` 下隐藏（这三者均不使用 `from_weight`）。
+ - 新增默认参数设定：
+ - GRPO 默认值（严格对齐 `trainer/train_grpo.py`）：
+ - `save_dir=../out`, `save_weight=grpo`, `epochs=1`, `batch_size=2`, `learning_rate=8e-8`
+ - `data_path=../dataset/rlaif-mini.jsonl`, `log_interval=1`, `save_interval=10`
+ - `hidden_size=512`, `num_hidden_layers=8`, `max_seq_len=66`, `use_moe=0`
+ - 特有：`beta=0.02`, `num_generations=8`, `reasoning=1`, `reward_model_path=../../internlm2-1_8b-reward`
+ - SPO 默认值（严格对齐 `trainer/train_spo.py`）：
+ - `save_dir=../out`, `save_weight=spo`, `epochs=1`, `batch_size=2`, `learning_rate=1e-7`
+ - `data_path=../dataset/rlaif-mini.jsonl`, `log_interval=1`, `save_interval=10`
+ - `hidden_size=512`, `num_hidden_layers=8`, `max_seq_len=66`, `use_moe=0`
+ - 特有：`beta=0.02`, `reasoning=1`, `reward_model_path=../../internlm2-1_8b-reward`
+- 保持 GPU 选择与提交逻辑不变。
+
+### 后端启动器（trainer_web/train_web_ui.py）
+- 在 `start_training_process` 中新增 `train_type=='grpo'` 与 `train_type=='spo'` 分支：
+ - `grpo` 使用脚本 `../trainer/train_grpo.py`；显式拼接：`--beta`、`--num_generations`、`--reasoning`、`--reward_model_path`（若前端提供）。
+ - `spo` 使用脚本 `../trainer/train_spo.py`；显式拼接：`--beta`、`--reasoning`、`--reward_model_path`（若前端提供）。
+- 在通用参数拼接的跳过列表中新增：`'num_generations','beta','reasoning','reward_model_path'`（防止重复添加），并与 DPO/PPO 现有跳过策略一致；同时对 `grpo/spo` 也跳过 `from_weight`。
+- 其余逻辑（多卡、日志、监控）保持不变。
+
+### 进程显示名称
+- `static/js/processes/list.js` 的名称映射中补充：`ppo: 'PPO', grpo: 'GRPO', spo: 'SPO'`（`logfiles/list.js` 已有，无需改动）。
+
+## 验证步骤
+- 前端：切换训练类型时仅显示对应参数卡；`from_weight` 在 `ppo/grpo/spo` 下隐藏；默认值填充正确。
+- 后端：提交后查看日志首行的命令拼接，确认包含对应特有参数：
+ - GRPO：`--beta 0.02 --num_generations 8 --reasoning 1 --reward_model_path ../../internlm2-1_8b-reward`
+ - SPO：`--beta 0.02 --reasoning 1 --reward_model_path ../../internlm2-1_8b-reward`
+- 保证 DPO/PPO 流程未受影响：分别发起一次 DPO/PPO 以回归测试。
+- 多卡/单卡/CPU 选择与日志查看、进程管理均保持工作。
+
+## 兼容性与风险控制
+- 所有改动均为增量更新，不修改既有字段与 API 形态。
+- 默认值与训练脚本对齐；占位符仅用于引导，实际以默认填充值为准。
+- 若未提供特有参数，后端不拼接该 flag，训练脚本使用自身默认值。
+
+## 交付
+- 更新上述 4 处文件：`templates/index.html`、`static/js/train/form.js`、`trainer_web/train_web_ui.py`、`static/js/processes/list.js`。
+- 提供端到端验证记录与示例日志片段，确保功能闭环。
\ No newline at end of file
diff --git a/trainer_web/static/js/processes/list.js b/trainer_web/static/js/processes/list.js
index 1d96bc9..9e9b911 100644
--- a/trainer_web/static/js/processes/list.js
+++ b/trainer_web/static/js/processes/list.js
@@ -115,6 +115,9 @@ function getTrainTypeDisplayName(trainType) {
sft: '全参数监督微调 (SFT - Full)',
lora: 'LoRA监督微调 (SFT - Lora)',
dpo: '直接偏好优化 (RL - DPO)',
+ ppo: 'PPO',
+ grpo: 'GRPO',
+ spo: 'SPO',
};
return names[trainType] || trainType;
}
diff --git a/trainer_web/static/js/train/form.js b/trainer_web/static/js/train/form.js
index 72e8c09..6e9f893 100644
--- a/trainer_web/static/js/train/form.js
+++ b/trainer_web/static/js/train/form.js
@@ -21,24 +21,38 @@ function onTrainTypeChange() {
const dpoCard = document.querySelector('.parameter-card.dpo');
const ppoFields = document.querySelectorAll('.ppo');
const ppoCard = document.querySelector('.parameter-card.ppo');
- pretrainSft.forEach((f) => (f.style.display = v === 'pretrain' || v === 'sft' || v === 'dpo' || v === 'ppo' ? 'block' : 'none'));
- fromWeightFields.forEach((f) => (f.style.display = v !== 'ppo' ? 'block' : 'none'));
+ const grpoFields = document.querySelectorAll('.grpo');
+ const grpoCard = document.querySelector('.parameter-card.grpo');
+ const spoFields = document.querySelectorAll('.spo');
+ const spoCard = document.querySelector('.parameter-card.spo');
+ pretrainSft.forEach((f) => (f.style.display = v === 'pretrain' || v === 'sft' || v === 'dpo' || v === 'ppo' || v === 'grpo' || v === 'spo' ? 'block' : 'none'));
+ fromWeightFields.forEach((f) => (f.style.display = v !== 'ppo' && v !== 'grpo' && v !== 'spo' ? 'block' : 'none'));
loraFields.forEach((f) => (f.style.display = v === 'lora' ? 'block' : 'none'));
dpoFields.forEach((f) => (f.style.display = v === 'dpo' ? 'block' : 'none'));
ppoFields.forEach((f) => (f.style.display = v === 'ppo' ? 'block' : 'none'));
if (dpoCard) dpoCard.style.display = v === 'dpo' ? 'block' : 'none';
if (ppoCard) ppoCard.style.display = v === 'ppo' ? 'block' : 'none';
+ grpoFields.forEach((f) => (f.style.display = v === 'grpo' ? 'block' : 'none'));
+ spoFields.forEach((f) => (f.style.display = v === 'spo' ? 'block' : 'none'));
+ if (grpoCard) grpoCard.style.display = v === 'grpo' ? 'block' : 'none';
+ if (spoCard) spoCard.style.display = v === 'spo' ? 'block' : 'none';
if (v === 'pretrain') setDefaults({ save_dir: '../out', save_weight: 'pretrain', epochs: '1', batch_size: '32', learning_rate: '5e-4', data_path: '../dataset/pretrain_hq.jsonl', from_weight: 'none', log_interval: '100', save_interval: '100', hidden_size: '512', num_hidden_layers: '8', max_seq_len: '512', use_moe: '0' });
else if (v === 'sft') setDefaults({ save_dir: '../out', save_weight: 'full_sft', epochs: '2', batch_size: '16', learning_rate: '5e-7', data_path: '../dataset/sft_mini_512.jsonl', from_weight: 'pretrain', log_interval: '100', save_interval: '100', hidden_size: '512', num_hidden_layers: '8', max_seq_len: '512', use_moe: '0' });
else if (v === 'lora') setDefaults({ save_dir: '../out/lora', lora_name: 'lora_identity', epochs: '50', batch_size: '32', learning_rate: '1e-4', data_path: '../dataset/lora_identity.jsonl', from_weight: 'full_sft', log_interval: '10', save_interval: '1', hidden_size: '512', num_hidden_layers: '8', max_seq_len: '512', use_moe: '0' });
else if (v === 'dpo') setDefaults({ save_dir: '../out', save_weight: 'dpo', epochs: '1', batch_size: '4', learning_rate: '4e-8', data_path: '../dataset/dpo.jsonl', from_weight: 'full_sft', log_interval: '100', save_interval: '100', beta: '0.1', hidden_size: '512', num_hidden_layers: '8', max_seq_len: '1024', use_moe: '0' });
else if (v === 'ppo') setDefaults({ save_dir: '../out', save_weight: 'ppo_actor', epochs: '1', batch_size: '2', learning_rate: '8e-8', data_path: '../dataset/rlaif-mini.jsonl', log_interval: '1', save_interval: '10', clip_epsilon: '0.1', vf_coef: '0.5', kl_coef: '0.02', reasoning: '1', update_old_actor_freq: '4', reward_model_path: '../../internlm2-1_8b-reward', hidden_size: '512', num_hidden_layers: '8', max_seq_len: '66', use_moe: '0' });
+ else if (v === 'grpo') setDefaults({ save_dir: '../out', save_weight: 'grpo', epochs: '1', batch_size: '2', learning_rate: '8e-8', data_path: '../dataset/rlaif-mini.jsonl', log_interval: '1', save_interval: '10', beta: '0.02', num_generations: '8', reasoning: '1', reward_model_path: '../../internlm2-1_8b-reward', hidden_size: '512', num_hidden_layers: '8', max_seq_len: '66', use_moe: '0' });
+ else if (v === 'spo') setDefaults({ save_dir: '../out', save_weight: 'spo', epochs: '1', batch_size: '2', learning_rate: '1e-7', data_path: '../dataset/rlaif-mini.jsonl', log_interval: '1', save_interval: '10', beta: '0.02', reasoning: '1', reward_model_path: '../../internlm2-1_8b-reward', hidden_size: '512', num_hidden_layers: '8', max_seq_len: '66', use_moe: '0' });
}
function setDefaults(map) {
- Object.entries(map).forEach(([id, val]) => {
- const node = document.getElementById(id);
- if (node) node.value = val;
+ Object.entries(map).forEach(([name, val]) => {
+ const nodes = document.querySelectorAll(`[name="${name}"]`);
+ nodes.forEach((node) => {
+ const card = node.closest('.parameter-card');
+ const visible = !card || card.style.display !== 'none';
+ if (visible) node.value = val;
+ });
});
}
@@ -69,19 +83,30 @@ function initGpuSelectors() {
function onSubmit(e) {
e.preventDefault();
const form = e.currentTarget;
- const fd = new FormData(form);
const data = {};
- const trainingMode = fd.get('training_mode');
- fd.forEach((value, key) => {
- if (key === 'gpu_num') {
- const multi = document.getElementById('multi-gpu-selection');
- if (multi && multi.style.display !== 'none') data[key] = value;
- } else if (key === 'device') {
- if (trainingMode === 'single_gpu') data[key] = `cuda:${value}`;
- else if (trainingMode === 'cpu') data[key] = 'cpu';
- } else if (key !== 'training_mode') {
- data[key] = value;
+ const trainingModeSel = form.querySelector('#training_mode');
+ const trainingMode = trainingModeSel ? trainingModeSel.value : 'cpu';
+ const inputs = form.querySelectorAll('input, select, textarea');
+ inputs.forEach((el) => {
+ const name = el.name;
+ if (!name || name === 'training_mode') return;
+ const card = el.closest('.parameter-card');
+ const visible = !card || card.style.display !== 'none';
+ if (!visible) return;
+ let value = el.value;
+ if (el.type === 'checkbox') {
+ if (!el.checked) return;
}
+ if (name === 'gpu_num') {
+ const multi = document.getElementById('multi-gpu-selection');
+ if (!(multi && multi.style.display !== 'none')) return;
+ }
+ if (name === 'device') {
+ if (trainingMode === 'single_gpu') value = `cuda:${value}`;
+ else if (trainingMode === 'cpu') value = 'cpu';
+ else return;
+ }
+ data[name] = value;
});
showNotification('正在启动训练...', 'info');
setTimeout(() => {
diff --git a/trainer_web/templates/index.html b/trainer_web/templates/index.html
index 8f9d75e..d6eb535 100644
--- a/trainer_web/templates/index.html
+++ b/trainer_web/templates/index.html
@@ -34,6 +34,8 @@
+
+
@@ -103,6 +105,54 @@
+
+
+
+
强化学习参数 (GRPO)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
强化学习参数 (SPO)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/trainer_web/train_web_ui.pid b/trainer_web/train_web_ui.pid
new file mode 100644
index 0000000..b7ae382
--- /dev/null
+++ b/trainer_web/train_web_ui.pid
@@ -0,0 +1 @@
+2100766
\ No newline at end of file
diff --git a/trainer_web/train_web_ui.py b/trainer_web/train_web_ui.py
index b8dcdf8..146e302 100644
--- a/trainer_web/train_web_ui.py
+++ b/trainer_web/train_web_ui.py
@@ -130,14 +130,40 @@ def start_training_process(train_type, params):
cmd.extend(['--update_old_actor_freq', params['update_old_actor_freq']])
if 'reward_model_path' in params and params['reward_model_path']:
cmd.extend(['--reward_model_path', params['reward_model_path']])
+ elif train_type == 'grpo':
+ script_path = '../trainer/train_grpo.py'
+ if use_torchrun:
+ cmd = ['torchrun', '--nproc_per_node', str(gpu_num), script_path]
+ else:
+ cmd = [sys.executable, script_path]
+ if 'beta' in params and params['beta']:
+ cmd.extend(['--beta', params['beta']])
+ if 'num_generations' in params and params['num_generations']:
+ cmd.extend(['--num_generations', params['num_generations']])
+ if 'reasoning' in params and params['reasoning']:
+ cmd.extend(['--reasoning', params['reasoning']])
+ if 'reward_model_path' in params and params['reward_model_path']:
+ cmd.extend(['--reward_model_path', params['reward_model_path']])
+ elif train_type == 'spo':
+ script_path = '../trainer/train_spo.py'
+ if use_torchrun:
+ cmd = ['torchrun', '--nproc_per_node', str(gpu_num), script_path]
+ else:
+ cmd = [sys.executable, script_path]
+ if 'beta' in params and params['beta']:
+ cmd.extend(['--beta', params['beta']])
+ if 'reasoning' in params and params['reasoning']:
+ cmd.extend(['--reasoning', params['reasoning']])
+ if 'reward_model_path' in params and params['reward_model_path']:
+ cmd.extend(['--reward_model_path', params['reward_model_path']])
else:
return None
# 添加通用参数
for key, value in params.items():
# 跳过特殊参数和DPO、PPO特有参数，以及gpu_num参数（因为已经在torchrun命令中使用）
- # 对于PPO训练，跳过--from_weight参数
- if key in ['train_type', 'save_weight', 'lora_name', 'train_monitor', 'beta', 'accumulation_steps', 'grad_clip', 'gpu_num', 'clip_epsilon', 'vf_coef', 'kl_coef', 'reasoning', 'update_old_actor_freq', 'reward_model_path'] or (train_type == 'ppo' and key == 'from_weight'):
+ # 对于PPO/GRPO/SPO训练，跳过--from_weight参数
+ if key in ['train_type', 'save_weight', 'lora_name', 'train_monitor', 'beta', 'accumulation_steps', 'grad_clip', 'gpu_num', 'clip_epsilon', 'vf_coef', 'kl_coef', 'reasoning', 'update_old_actor_freq', 'reward_model_path', 'num_generations'] or ((train_type == 'ppo' or train_type == 'grpo' or train_type == 'spo') and key == 'from_weight'):
continue
# 对于from_resume参数，需要正确传递参数值
elif key == 'from_resume':
diff --git a/trainer_web/training_processes.json b/trainer_web/training_processes.json
new file mode 100644
index 0000000..d91960d
--- /dev/null
+++ b/trainer_web/training_processes.json
@@ -0,0 +1,13 @@
+{
+ "20251117_171339": {
+ "pid": 2060245,
+ "train_type": "grpo",
+ "log_file": "/root/minimind_web/minimind/logfile/train_grpo_20251117_171339.log",
+ "start_time": "2025-11-17 17:13:39",
+ "running": false,
+ "error": true,
+ "manually_stopped": false,
+ "train_monitor": "none",
+ "swanlab_url": null
+ }
+}
\ No newline at end of file