From 5bef3f08ababcbec3b7a37f40615478ea2303ff0 Mon Sep 17 00:00:00 2001 From: yuyu5333 <1812107659@qq.com> Date: Fri, 7 Nov 2025 05:13:13 +0000 Subject: [PATCH] add: rl-dpo --- scripts/static/js/script.js | 109 +++++++++++++++++++++++++++++++++++ scripts/templates/index.html | 16 ++++- scripts/train_web_ui.py | 22 +++++-- 3 files changed, 140 insertions(+), 7 deletions(-) diff --git a/scripts/static/js/script.js b/scripts/static/js/script.js index 1502419..ea9bcd7 100644 --- a/scripts/static/js/script.js +++ b/scripts/static/js/script.js @@ -12,6 +12,14 @@ function openTab(evt, tabName) { document.getElementById(tabName).classList.remove("hidden"); evt.currentTarget.classList.add("active"); + // 当切换到其他标签页时，清除所有日志自动更新定时器 + if (tabName !== 'processes') { + for (let processId in logTimers) { + clearInterval(logTimers[processId]); + delete logTimers[processId]; + } + } + // 如果切换到进程页面，刷新进程列表 if (tabName === 'processes') { loadProcesses(); @@ -25,6 +33,8 @@ document.getElementById('train_type').addEventListener('change', function() { const trainType = this.value; const pretrainSftFields = document.querySelectorAll('.pretrain-sft'); const loraFields = document.querySelectorAll('.lora'); + const dpoFields = document.querySelectorAll('.dpo'); + const dpoParamCard = document.querySelector('.parameter-card.dpo'); pretrainSftFields.forEach(field => { field.style.display = (trainType === 'pretrain' || trainType === 'sft') ? 'block' : 'none'; @@ -34,6 +44,14 @@ document.getElementById('train_type').addEventListener('change', function() { field.style.display = trainType === 'lora' ? 'block' : 'none'; }); + dpoFields.forEach(field => { + field.style.display = trainType === 'dpo' ? 'block' : 'none'; + }); + + if (dpoParamCard) { + dpoParamCard.style.display = trainType === 'dpo' ? 'block' : 'none'; + } + // 设置默认值 if (trainType === 'pretrain') { document.getElementById('save_dir').value = '../out'; @@ -65,6 +83,17 @@ document.getElementById('train_type').addEventListener('change', function() { document.getElementById('from_weight').value = 'full_sft'; document.getElementById('log_interval').value = '10'; document.getElementById('save_interval').value = '1'; + } else if (trainType === 'dpo') { + document.getElementById('save_dir').value = '../out'; + document.getElementById('save_weight').value = 'dpo'; + document.getElementById('epochs').value = '1'; + document.getElementById('batch_size').value = '4'; + document.getElementById('learning_rate').value = '4e-8'; + document.getElementById('data_path').value = '../dataset/dpo.jsonl'; + document.getElementById('from_weight').value = 'full_sft'; + document.getElementById('log_interval').value = '100'; + document.getElementById('save_interval').value = '100'; + document.getElementById('beta').value = '0.1'; } }); @@ -100,6 +129,10 @@ function loadProcesses() { statusClass = 'status-completed'; } + // 设置进程数据属性，用于后续检查状态 + processItem.dataset.processId = process.id; + processItem.dataset.processStatus = process.status; + processItem.innerHTML = `
@@ -122,13 +155,52 @@ function loadProcesses() { }); } +// 存储训练日志定时器的ID +let logTimers = {}; + // 显示日志 function showLogs(processId) { const logsContainer = document.getElementById(`logs-${processId}`); logsContainer.classList.toggle('hidden'); if (!logsContainer.classList.contains('hidden')) { + // 加载日志内容 loadLogContent(processId, logsContainer); + + // 清除可能存在的旧定时器 + if (logTimers[processId]) { + clearInterval(logTimers[processId]); + } + + // 查找进程元素，获取其状态 + const processItem = document.querySelector(`[data-process-id="${processId}"]`); + const isRunning = processItem && processItem.dataset.processStatus === '运行中'; + + // 只有运行中的进程才设置自动刷新定时器 + if (isRunning) { + logTimers[processId] = setInterval(() => { + // 检查日志容器是否可见 + if (!logsContainer.classList.contains('hidden')) { + // 再次检查进程状态，确保仍然是运行中 + const currentProcessItem = document.querySelector(`[data-process-id="${processId}"]`); + const stillRunning = currentProcessItem && currentProcessItem.dataset.processStatus === '运行中'; + + if (stillRunning) { + loadLogContent(processId, logsContainer); + } else { + // 如果进程不再运行中，清除定时器 + clearInterval(logTimers[processId]); + delete logTimers[processId]; + } + } + }, 1000); + } + } else { + // 隐藏日志时清除定时器 + if (logTimers[processId]) { + clearInterval(logTimers[processId]); + delete logTimers[processId]; + } } } @@ -137,6 +209,34 @@ function refreshLog(processId) { const logsContainer = document.getElementById(`logs-${processId}`); if (!logsContainer.classList.contains('hidden')) { loadLogContent(processId, logsContainer); + + // 查找进程元素，获取其状态 + const processItem = document.querySelector(`[data-process-id="${processId}"]`); + const isRunning = processItem && processItem.dataset.processStatus === '运行中'; + + // 重置定时器，确保从现在开始每1秒更新一次 + if (logTimers[processId]) { + clearInterval(logTimers[processId]); + } + + // 只有运行中的进程才设置定时器 + if (isRunning) { + logTimers[processId] = setInterval(() => { + if (!logsContainer.classList.contains('hidden')) { + // 再次检查进程状态 + const currentProcessItem = document.querySelector(`[data-process-id="${processId}"]`); + const stillRunning = currentProcessItem && currentProcessItem.dataset.processStatus === '运行中'; + + if (stillRunning) { + loadLogContent(processId, logsContainer); + } else { + // 如果进程不再运行中，清除定时器 + clearInterval(logTimers[processId]); + delete logTimers[processId]; + } + } + }, 1000); + } } } @@ -214,7 +314,16 @@ function loadLogFiles() { trainType = 'sft'; } else if (logfile.filename.includes('train_lora_')) { trainType = 'lora'; + } else if (logfile.filename.includes('train_dpo_')) { + trainType = 'dpo'; + } else if (logfile.filename.includes('train_ppo_')) { + trainType = 'ppo'; + } else if (logfile.filename.includes('train_grpo_')) { + trainType = 'grpo'; + } else if (logfile.filename.includes('train_spo_')) { + trainType = 'spo'; } + fileItem.innerHTML = `
diff --git a/scripts/templates/index.html b/scripts/templates/index.html index 1c57c2a..63cc1a8 100644 --- a/scripts/templates/index.html +++ b/scripts/templates/index.html @@ -3,11 +3,11 @@ - MiniMind 训练 Web UI + MiniMind Training Lab -

MiniMind 训练 Web UI

+

MiniMind Training Lab

@@ -29,6 +29,7 @@ +
@@ -54,6 +55,17 @@
+ + +

模型结构参数

diff --git a/scripts/train_web_ui.py b/scripts/train_web_ui.py index 8beb5d2..ee80e3f 100644 --- a/scripts/train_web_ui.py +++ b/scripts/train_web_ui.py @@ -43,13 +43,23 @@ def start_training_process(train_type, params): cmd = [sys.executable, script_path] if 'lora_name' in params: cmd.extend(['--lora_name', params['lora_name']]) + elif train_type == 'dpo': + script_path = '../trainer/train_dpo.py' + cmd = [sys.executable, script_path] + # 添加DPO特定参数 + if 'beta' in params and params['beta']: + cmd.extend(['--beta', params['beta']]) + if 'accumulation_steps' in params and params['accumulation_steps']: + cmd.extend(['--accumulation_steps', params['accumulation_steps']]) + if 'grad_clip' in params and params['grad_clip']: + cmd.extend(['--grad_clip', params['grad_clip']]) else: return None # 添加通用参数 for key, value in params.items(): - # 跳过特殊参数 - if key in ['train_type', 'save_weight', 'lora_name', 'train_monitor']: + # 跳过特殊参数和DPO特有参数 + if key in ['train_type', 'save_weight', 'lora_name', 'train_monitor', 'beta', 'accumulation_steps', 'grad_clip']: continue # 特殊处理布尔标志参数 @@ -64,6 +74,8 @@ def start_training_process(train_type, params): if 'train_monitor' in params: if params['train_monitor'] == 'wandb' or params['train_monitor'] == 'swanlab': cmd.append('--use_wandb') # 对于wandb和swanlab，只添加标志，不添加值 + if params['train_monitor'] == 'wandb': + cmd.extend(['--wandb_project', 'minimind_training']) # 创建日志文件 with open(log_file, 'w') as f: @@ -170,10 +182,10 @@ def logs(process_id): if os.path.exists(log_file): try: with open(log_file, 'r', encoding='utf-8') as f: - # 只读取最后500行 + # 只读取最后200行 lines = f.readlines() - last_500_lines = lines[-500:] if len(lines) > 500 else lines - return ''.join(last_500_lines) + last_200_lines = lines[-200:] if len(lines) > 200 else lines + return ''.join(last_200_lines) except Exception as e: return f'读取日志失败: {str(e)}' return '日志文件不存在或已被删除'