support rl-grpo, rl-spo

.trae/documents/完成 GRPO 与 SPO 训练类型的前端与后端支持.md

@@ -0,0 +1,66 @@
+## 目标
+- 在现有 DPO/PPO 基础上，新增 GRPO 与 SPO 训练类型的完整前后端流程。
+- 选定对应训练类型后，仅显示该类型的特有参数；隐藏不需要的参数。
+- 参数默认值严格对齐 `trainer/train_grpo.py` 与 `trainer/train_spo.py`。
+- 保持原有 `trainer_web` 逻辑与交互不受影响。
+
+## 代码改动
+### 前端模板（index.html）
+- 在训练类型选择框中新增：`RL - GRPO`（value=`grpo`）、`RL - SPO`（value=`spo`）。
+- 新增两块参数卡片：
+  - GRPO 参数卡（class=`parameter-card grpo`）：
+    - `beta`（浮点输入，placeholder=0.02）
+    - `num_generations`（整数输入，placeholder=8）
+    - `reasoning`（下拉 0/1）
+    - `reward_model_path`（文本输入，placeholder=`../../internlm2-1_8b-reward`）
+  - SPO 参数卡（class=`parameter-card spo`）：
+    - `beta`（浮点输入，placeholder=0.02）
+    - `reasoning`（下拉 0/1）
+    - `reward_model_path`（文本输入，placeholder=`../../internlm2-1_8b-reward`）
+- 保持现有 DPO/PPO 参数卡结构不变。
+
+### 前端交互（static/js/train/form.js）
+- 在 `onTrainTypeChange` 中：
+  - 为 GRPO/SPO 添加显示/隐藏逻辑：
+    - `.grpo` 与 `.spo` 卡片只在对应训练类型显示。
+    - `pretrain-sft` 区域显示范围扩展到 `grpo` 与 `spo`（需要 `save_weight` 字段）。
+    - `from-weight` 区域在 `ppo/grpo/spo` 下隐藏（这三者均不使用 `from_weight`）。
+  - 新增默认参数设定：
+    - GRPO 默认值（严格对齐 `trainer/train_grpo.py`）：
+      - `save_dir=../out`, `save_weight=grpo`, `epochs=1`, `batch_size=2`, `learning_rate=8e-8`
+      - `data_path=../dataset/rlaif-mini.jsonl`, `log_interval=1`, `save_interval=10`
+      - `hidden_size=512`, `num_hidden_layers=8`, `max_seq_len=66`, `use_moe=0`
+      - 特有：`beta=0.02`, `num_generations=8`, `reasoning=1`, `reward_model_path=../../internlm2-1_8b-reward`
+    - SPO 默认值（严格对齐 `trainer/train_spo.py`）：
+      - `save_dir=../out`, `save_weight=spo`, `epochs=1`, `batch_size=2`, `learning_rate=1e-7`
+      - `data_path=../dataset/rlaif-mini.jsonl`, `log_interval=1`, `save_interval=10`
+      - `hidden_size=512`, `num_hidden_layers=8`, `max_seq_len=66`, `use_moe=0`
+      - 特有：`beta=0.02`, `reasoning=1`, `reward_model_path=../../internlm2-1_8b-reward`
+- 保持 GPU 选择与提交逻辑不变。
+
+### 后端启动器（trainer_web/train_web_ui.py）
+- 在 `start_training_process` 中新增 `train_type=='grpo'` 与 `train_type=='spo'` 分支：
+  - `grpo` 使用脚本 `../trainer/train_grpo.py`；显式拼接：`--beta`、`--num_generations`、`--reasoning`、`--reward_model_path`（若前端提供）。
+  - `spo` 使用脚本 `../trainer/train_spo.py`；显式拼接：`--beta`、`--reasoning`、`--reward_model_path`（若前端提供）。
+- 在通用参数拼接的跳过列表中新增：`'num_generations','beta','reasoning','reward_model_path'`（防止重复添加），并与 DPO/PPO 现有跳过策略一致；同时对 `grpo/spo` 也跳过 `from_weight`。
+- 其余逻辑（多卡、日志、监控）保持不变。
+
+### 进程显示名称
+- `static/js/processes/list.js` 的名称映射中补充：`ppo: 'PPO', grpo: 'GRPO', spo: 'SPO'`（`logfiles/list.js` 已有，无需改动）。
+
+## 验证步骤
+- 前端：切换训练类型时仅显示对应参数卡；`from_weight` 在 `ppo/grpo/spo` 下隐藏；默认值填充正确。
+- 后端：提交后查看日志首行的命令拼接，确认包含对应特有参数：
+  - GRPO：`--beta 0.02 --num_generations 8 --reasoning 1 --reward_model_path ../../internlm2-1_8b-reward`
+  - SPO：`--beta 0.02 --reasoning 1 --reward_model_path ../../internlm2-1_8b-reward`
+- 保证 DPO/PPO 流程未受影响：分别发起一次 DPO/PPO 以回归测试。
+- 多卡/单卡/CPU 选择与日志查看、进程管理均保持工作。
+
+## 兼容性与风险控制
+- 所有改动均为增量更新，不修改既有字段与 API 形态。
+- 默认值与训练脚本对齐；占位符仅用于引导，实际以默认填充值为准。
+- 若未提供特有参数，后端不拼接该 flag，训练脚本使用自身默认值。
+
+## 交付
+- 更新上述 4 处文件：`templates/index.html`、`static/js/train/form.js`、`trainer_web/train_web_ui.py`、`static/js/processes/list.js`。
+- 提供端到端验证记录与示例日志片段，确保功能闭环。

trainer_web/static/js/processes/list.js

@@ -115,6 +115,9 @@ function getTrainTypeDisplayName(trainType) {
     sft: '全参数监督微调 (SFT - Full)',
     lora: 'LoRA监督微调 (SFT - Lora)',
     dpo: '直接偏好优化 (RL - DPO)',
+    ppo: 'PPO',
+    grpo: 'GRPO',
+    spo: 'SPO',
   };
   return names[trainType] || trainType;
 }

trainer_web/static/js/train/form.js

@@ -21,24 +21,38 @@ function onTrainTypeChange() {
   const dpoCard = document.querySelector('.parameter-card.dpo');
   const ppoFields = document.querySelectorAll('.ppo');
   const ppoCard = document.querySelector('.parameter-card.ppo');
-  pretrainSft.forEach((f) => (f.style.display = v === 'pretrain' || v === 'sft' || v === 'dpo' || v === 'ppo' ? 'block' : 'none'));
+  const grpoFields = document.querySelectorAll('.grpo');
-  fromWeightFields.forEach((f) => (f.style.display = v !== 'ppo' ? 'block' : 'none'));
+  const grpoCard = document.querySelector('.parameter-card.grpo');
+  const spoFields = document.querySelectorAll('.spo');
+  const spoCard = document.querySelector('.parameter-card.spo');
+  pretrainSft.forEach((f) => (f.style.display = v === 'pretrain' || v === 'sft' || v === 'dpo' || v === 'ppo' || v === 'grpo' || v === 'spo' ? 'block' : 'none'));
+  fromWeightFields.forEach((f) => (f.style.display = v !== 'ppo' && v !== 'grpo' && v !== 'spo' ? 'block' : 'none'));
   loraFields.forEach((f) => (f.style.display = v === 'lora' ? 'block' : 'none'));
   dpoFields.forEach((f) => (f.style.display = v === 'dpo' ? 'block' : 'none'));
   ppoFields.forEach((f) => (f.style.display = v === 'ppo' ? 'block' : 'none'));
   if (dpoCard) dpoCard.style.display = v === 'dpo' ? 'block' : 'none';
   if (ppoCard) ppoCard.style.display = v === 'ppo' ? 'block' : 'none';
+  grpoFields.forEach((f) => (f.style.display = v === 'grpo' ? 'block' : 'none'));
+  spoFields.forEach((f) => (f.style.display = v === 'spo' ? 'block' : 'none'));
+  if (grpoCard) grpoCard.style.display = v === 'grpo' ? 'block' : 'none';
+  if (spoCard) spoCard.style.display = v === 'spo' ? 'block' : 'none';
   if (v === 'pretrain') setDefaults({ save_dir: '../out', save_weight: 'pretrain', epochs: '1', batch_size: '32', learning_rate: '5e-4', data_path: '../dataset/pretrain_hq.jsonl', from_weight: 'none', log_interval: '100', save_interval: '100', hidden_size: '512', num_hidden_layers: '8', max_seq_len: '512', use_moe: '0' });
   else if (v === 'sft') setDefaults({ save_dir: '../out', save_weight: 'full_sft', epochs: '2', batch_size: '16', learning_rate: '5e-7', data_path: '../dataset/sft_mini_512.jsonl', from_weight: 'pretrain', log_interval: '100', save_interval: '100', hidden_size: '512', num_hidden_layers: '8', max_seq_len: '512', use_moe: '0' });
   else if (v === 'lora') setDefaults({ save_dir: '../out/lora', lora_name: 'lora_identity', epochs: '50', batch_size: '32', learning_rate: '1e-4', data_path: '../dataset/lora_identity.jsonl', from_weight: 'full_sft', log_interval: '10', save_interval: '1', hidden_size: '512', num_hidden_layers: '8', max_seq_len: '512', use_moe: '0' });
   else if (v === 'dpo') setDefaults({ save_dir: '../out', save_weight: 'dpo', epochs: '1', batch_size: '4', learning_rate: '4e-8', data_path: '../dataset/dpo.jsonl', from_weight: 'full_sft', log_interval: '100', save_interval: '100', beta: '0.1', hidden_size: '512', num_hidden_layers: '8', max_seq_len: '1024', use_moe: '0' });
   else if (v === 'ppo') setDefaults({ save_dir: '../out', save_weight: 'ppo_actor', epochs: '1', batch_size: '2', learning_rate: '8e-8', data_path: '../dataset/rlaif-mini.jsonl', log_interval: '1', save_interval: '10', clip_epsilon: '0.1', vf_coef: '0.5', kl_coef: '0.02', reasoning: '1', update_old_actor_freq: '4', reward_model_path: '../../internlm2-1_8b-reward', hidden_size: '512', num_hidden_layers: '8', max_seq_len: '66', use_moe: '0' });
+  else if (v === 'grpo') setDefaults({ save_dir: '../out', save_weight: 'grpo', epochs: '1', batch_size: '2', learning_rate: '8e-8', data_path: '../dataset/rlaif-mini.jsonl', log_interval: '1', save_interval: '10', beta: '0.02', num_generations: '8', reasoning: '1', reward_model_path: '../../internlm2-1_8b-reward', hidden_size: '512', num_hidden_layers: '8', max_seq_len: '66', use_moe: '0' });
+  else if (v === 'spo') setDefaults({ save_dir: '../out', save_weight: 'spo', epochs: '1', batch_size: '2', learning_rate: '1e-7', data_path: '../dataset/rlaif-mini.jsonl', log_interval: '1', save_interval: '10', beta: '0.02', reasoning: '1', reward_model_path: '../../internlm2-1_8b-reward', hidden_size: '512', num_hidden_layers: '8', max_seq_len: '66', use_moe: '0' });
 }
 
 function setDefaults(map) {
-  Object.entries(map).forEach(([id, val]) => {
+  Object.entries(map).forEach(([name, val]) => {
-    const node = document.getElementById(id);
+    const nodes = document.querySelectorAll(`[name="${name}"]`);
-    if (node) node.value = val;
+    nodes.forEach((node) => {
+      const card = node.closest('.parameter-card');
+      const visible = !card || card.style.display !== 'none';
+      if (visible) node.value = val;
+    });
   });
 }
 
@@ -69,19 +83,30 @@ function initGpuSelectors() {
 function onSubmit(e) {
   e.preventDefault();
   const form = e.currentTarget;
-  const fd = new FormData(form);
   const data = {};
-  const trainingMode = fd.get('training_mode');
+  const trainingModeSel = form.querySelector('#training_mode');
-  fd.forEach((value, key) => {
+  const trainingMode = trainingModeSel ? trainingModeSel.value : 'cpu';
-    if (key === 'gpu_num') {
+  const inputs = form.querySelectorAll('input, select, textarea');
-      const multi = document.getElementById('multi-gpu-selection');
+  inputs.forEach((el) => {
-      if (multi && multi.style.display !== 'none') data[key] = value;
+    const name = el.name;
-    } else if (key === 'device') {
+    if (!name || name === 'training_mode') return;
-      if (trainingMode === 'single_gpu') data[key] = `cuda:${value}`;
+    const card = el.closest('.parameter-card');
-      else if (trainingMode === 'cpu') data[key] = 'cpu';
+    const visible = !card || card.style.display !== 'none';
-    } else if (key !== 'training_mode') {
+    if (!visible) return;
-      data[key] = value;
+    let value = el.value;
+    if (el.type === 'checkbox') {
+      if (!el.checked) return;
     }
+    if (name === 'gpu_num') {
+      const multi = document.getElementById('multi-gpu-selection');
+      if (!(multi && multi.style.display !== 'none')) return;
+    }
+    if (name === 'device') {
+      if (trainingMode === 'single_gpu') value = `cuda:${value}`;
+      else if (trainingMode === 'cpu') value = 'cpu';
+      else return;
+    }
+    data[name] = value;
   });
   showNotification('正在启动训练...', 'info');
   setTimeout(() => {

trainer_web/templates/index.html

@@ -34,6 +34,8 @@
                                 <option value="lora">SFT - Lora</option>
                                 <option value="dpo">RL - DPO</option>
                                 <option value="ppo">RL - PPO</option>
+                                <option value="grpo">RL - GRPO</option>
+                                <option value="spo">RL - SPO</option>
                             </select>
                         </div>
                         <div class="form-group">
@@ -103,6 +105,54 @@
                         </div>
                     </div>
                 </div>
+
+                <!-- GRPO强化学习参数 -->
+                <div class="parameter-card grpo" style="display: none;">
+                    <h3 class="card-title">强化学习参数 (GRPO)</h3>
+                    <div class="parameter-content">
+                        <div class="form-group">
+                            <label for="beta_grpo">GRPO KL惩罚系数：</label>
+                            <input type="text" id="beta_grpo" name="beta" placeholder="0.02">
+                        </div>
+                        <div class="form-group">
+                            <label for="num_generations">每个prompt生成样本数：</label>
+                            <input type="number" id="num_generations" name="num_generations" placeholder="8" min="1">
+                        </div>
+                        <div class="form-group">
+                            <label for="reasoning_grpo">是否使用Reasoning模式：</label>
+                            <select id="reasoning_grpo" name="reasoning">
+                                <option value="0">否</option>
+                                <option value="1">是</option>
+                            </select>
+                        </div>
+                        <div class="form-group">
+                            <label for="reward_model_path_grpo">奖励模型路径：</label>
+                            <input type="text" id="reward_model_path_grpo" name="reward_model_path" placeholder="../../internlm2-1_8b-reward">
+                        </div>
+                    </div>
+                </div>
+
+                <!-- SPO强化学习参数 -->
+                <div class="parameter-card spo" style="display: none;">
+                    <h3 class="card-title">强化学习参数 (SPO)</h3>
+                    <div class="parameter-content">
+                        <div class="form-group">
+                            <label for="beta_spo">SPO KL惩罚系数：</label>
+                            <input type="text" id="beta_spo" name="beta" placeholder="0.02">
+                        </div>
+                        <div class="form-group">
+                            <label for="reasoning_spo">是否使用Reasoning模式：</label>
+                            <select id="reasoning_spo" name="reasoning">
+                                <option value="0">否</option>
+                                <option value="1">是</option>
+                            </select>
+                        </div>
+                        <div class="form-group">
+                            <label for="reward_model_path_spo">奖励模型路径：</label>
+                            <input type="text" id="reward_model_path_spo" name="reward_model_path" placeholder="../../internlm2-1_8b-reward">
+                        </div>
+                    </div>
+                </div>
                 
                 <!-- 模型结构参数 -->
                 <div class="parameter-card">

trainer_web/train_web_ui.pid

@@ -0,0 +1 @@
+2100766

trainer_web/train_web_ui.py

@@ -130,14 +130,40 @@ def start_training_process(train_type, params):
             cmd.extend(['--update_old_actor_freq', params['update_old_actor_freq']])
         if 'reward_model_path' in params and params['reward_model_path']:
             cmd.extend(['--reward_model_path', params['reward_model_path']])
+    elif train_type == 'grpo':
+        script_path = '../trainer/train_grpo.py'
+        if use_torchrun:
+            cmd = ['torchrun', '--nproc_per_node', str(gpu_num), script_path]
+        else:
+            cmd = [sys.executable, script_path]
+        if 'beta' in params and params['beta']:
+            cmd.extend(['--beta', params['beta']])
+        if 'num_generations' in params and params['num_generations']:
+            cmd.extend(['--num_generations', params['num_generations']])
+        if 'reasoning' in params and params['reasoning']:
+            cmd.extend(['--reasoning', params['reasoning']])
+        if 'reward_model_path' in params and params['reward_model_path']:
+            cmd.extend(['--reward_model_path', params['reward_model_path']])
+    elif train_type == 'spo':
+        script_path = '../trainer/train_spo.py'
+        if use_torchrun:
+            cmd = ['torchrun', '--nproc_per_node', str(gpu_num), script_path]
+        else:
+            cmd = [sys.executable, script_path]
+        if 'beta' in params and params['beta']:
+            cmd.extend(['--beta', params['beta']])
+        if 'reasoning' in params and params['reasoning']:
+            cmd.extend(['--reasoning', params['reasoning']])
+        if 'reward_model_path' in params and params['reward_model_path']:
+            cmd.extend(['--reward_model_path', params['reward_model_path']])
     else:
         return None
     
     # 添加通用参数
     for key, value in params.items():
         # 跳过特殊参数和DPO、PPO特有参数，以及gpu_num参数（因为已经在torchrun命令中使用）
-        # 对于PPO训练，跳过--from_weight参数
+        # 对于PPO/GRPO/SPO训练，跳过--from_weight参数
-        if key in ['train_type', 'save_weight', 'lora_name', 'train_monitor', 'beta', 'accumulation_steps', 'grad_clip', 'gpu_num', 'clip_epsilon', 'vf_coef', 'kl_coef', 'reasoning', 'update_old_actor_freq', 'reward_model_path'] or (train_type == 'ppo' and key == 'from_weight'):
+        if key in ['train_type', 'save_weight', 'lora_name', 'train_monitor', 'beta', 'accumulation_steps', 'grad_clip', 'gpu_num', 'clip_epsilon', 'vf_coef', 'kl_coef', 'reasoning', 'update_old_actor_freq', 'reward_model_path', 'num_generations'] or ((train_type == 'ppo' or train_type == 'grpo' or train_type == 'spo') and key == 'from_weight'):
             continue
         # 对于from_resume参数，需要正确传递参数值
         elif key == 'from_resume':

trainer_web/training_processes.json

@@ -0,0 +1,13 @@
+{
+  "20251117_171339": {
+    "pid": 2060245,
+    "train_type": "grpo",
+    "log_file": "/root/minimind_web/minimind/logfile/train_grpo_20251117_171339.log",
+    "start_time": "2025-11-17 17:13:39",
+    "running": false,
+    "error": true,
+    "manually_stopped": false,
+    "train_monitor": "none",
+    "swanlab_url": null
+  }
+}