diff --git a/trainer_web/static/js/app.js b/trainer_web/static/js/app.js index 072fa59..c2dcdb9 100644 --- a/trainer_web/static/js/app.js +++ b/trainer_web/static/js/app.js @@ -6,7 +6,10 @@ import { refreshLog } from './processes/logs.js'; const hooks = { onEnterProcesses: () => { - loadProcesses(); + // 当切换到进程标签页时，立即加载一次，然后开始轮询 + loadProcesses().then(() => { + startProcessPolling(); + }); }, onLeaveProcesses: () => { stopProcessPolling(); @@ -339,13 +342,21 @@ window.navigateToParent = navigateToParent; window.selectCurrentDirectory = selectCurrentDirectory; // 将进程管理函数暴露到全局作用域 -window.refreshProcesses = loadProcesses; +window.refreshProcesses = () => { + // 立即刷新进程数据，然后重置轮询计时器 + return loadProcesses().then(() => { + // 重置轮询计时器以确保平滑的更新间隔 + stopProcessPolling(); + startProcessPolling(); + }); +}; window.refreshLogs = loadLogFiles; window.refreshLog = refreshLog; window.addEventListener('load', () => { initTrainForm(); - startProcessPolling(); - loadProcesses(); + // 不再立即开始轮询，而是等待用户切换到进程标签页 + // startProcessPolling(); // 移动到钩子函数中 + loadProcesses(); // 仍然加载初始进程数据 }); diff --git a/trainer_web/static/js/processes/list.js b/trainer_web/static/js/processes/list.js index 7305d7f..5d3ea58 100644 --- a/trainer_web/static/js/processes/list.js +++ b/trainer_web/static/js/processes/list.js @@ -168,10 +168,13 @@ let processPollingTimer = null; export function startProcessPolling() { if (processPollingTimer) clearInterval(processPollingTimer); + // 使用2秒间隔进行实时更新，确保进度信息及时刷新 processPollingTimer = setInterval(() => { const tab = document.querySelector('.tab.active'); - if (tab && tab.textContent.includes('进程')) checkProcessStatusChanges(); - }, 5000); + if (tab && tab.textContent.includes('进程')) { + checkProcessStatusChanges(); + } + }, 2000); } export function stopProcessPolling() { @@ -184,16 +187,30 @@ export function stopProcessPolling() { export function checkProcessStatusChanges() { return getProcesses() .then((data) => { + let updatedCount = 0; data.forEach((p) => { const item = document.querySelector(`[data-process-id="${p.id}"]`); if (!item) return; const cur = item.dataset.processStatus; const next = p.status; + + // 如果状态发生变化，更新整个项目 if (cur !== next) { updateProcessItem(item, p); if (next === '出错') showNotification(`进程 ${p.train_type} 已出错`, 'error'); + updatedCount++; + } + // 如果进程正在运行，即使状态没变也要更新进度信息 + else if (p.running) { + updateProcessProgress(item, p); + updatedCount++; } }); + + // 调试用：在控制台显示更新信息（生产环境中可以移除） + if (updatedCount > 0) { + console.log(`[${new Date().toLocaleTimeString()}] 更新了 ${updatedCount} 个进程的进度信息`); + } }) .catch(() => { showNotification('连接服务器失败，请刷新页面重试', 'error'); @@ -347,6 +364,46 @@ function bindItemButtons(item, process) { if (delBtn) delBtn.addEventListener('click', () => deleteProcess(process.id)); } +export function updateProcessProgress(item, process) { + // 只更新进度信息，不更新整个项目 + const progressInfo = calculateProgress(process); + + // 更新进度条 + const progressFill = item.querySelector('.progress-fill'); + const progressText = item.querySelector('.progress-info span:first-child'); + const remainingText = item.querySelector('.progress-info span:last-child'); + const metricsContainer = item.querySelector('.progress-metrics'); + + if (progressFill) { + progressFill.style.width = `${progressInfo.percentage}%`; + } + + if (progressText) { + progressText.textContent = `进度: ${progressInfo.current}/${progressInfo.total}`; + } + + if (remainingText) { + remainingText.textContent = `剩余时间: ${progressInfo.remaining}`; + } + + if (metricsContainer) { + // 更新指标 - 只更新有变化的值来减少DOM操作 + const lossItem = metricsContainer.querySelector('.metric-item:nth-child(1) .metric-value'); + const epochItem = metricsContainer.querySelector('.metric-item:nth-child(2) .metric-value'); + const lrItem = metricsContainer.querySelector('.metric-item:nth-child(3) .metric-value'); + + if (progressInfo.loss && lossItem) { + lossItem.textContent = progressInfo.loss; + } + if (progressInfo.epoch && epochItem) { + epochItem.textContent = progressInfo.epoch; + } + if (progressInfo.lr && lrItem) { + lrItem.textContent = progressInfo.lr; + } + } +} + export function updateProcessItem(item, process) { item.dataset.processStatus = process.status; item.dataset.trainMonitor = process.train_monitor || 'none'; diff --git a/trainer_web/train_web_ui.py b/trainer_web/train_web_ui.py index f03649b..f1cea06 100644 --- a/trainer_web/train_web_ui.py +++ b/trainer_web/train_web_ui.py @@ -389,6 +389,106 @@ def train(): else: return jsonify({'success': False, 'error': '无效的训练类型'}) +# 测试端点 - 添加模拟训练进程 +@app.route('/test/add_process', methods=['POST']) +def add_test_process(): + """添加一个测试进程用于验证自动更新功能""" + import subprocess + import threading + + process_id = f"test_process_{int(time.time())}" + + # 创建测试训练命令 + test_command = [ + 'python', '-c', ''' +import time +import sys + +print("2024-11-21 14:30:00 - Starting pretrain training") +sys.stdout.flush() +time.sleep(1) + +print("2024-11-21 14:30:01 - Loading dataset from ../dataset/pretrain_hq.jsonl") +sys.stdout.flush() +time.sleep(1) + +print("2024-11-21 14:30:02 - Model initialized with 108M parameters") +sys.stdout.flush() +time.sleep(2) + +for epoch in range(1, 6): + print(f"2024-11-21 14:30:{5 + epoch*5} - Starting epoch {epoch}/5") + sys.stdout.flush() + time.sleep(1) + + loss = 4.5 - epoch * 0.3 + lr = 1e-4 * (0.9 ** epoch) + print(f"2024-11-21 14:30:{6 + epoch*5} - Loss: {loss:.4f}, lr: {lr:.2e}") + sys.stdout.flush() + time.sleep(2) + + remaining = (5 - epoch) * 15 + print(f"2024-11-21 14:30:{8 + epoch*5} - Epoch {epoch}/5 completed, remaining: 0:0{remaining}:00") + sys.stdout.flush() + time.sleep(1) + +print("2024-11-21 14:30:35 - Training completed successfully") +sys.stdout.flush() + ''' + ] + + # 启动进程 + process = subprocess.Popen( + test_command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + bufsize=1 + ) + + # 保存进程信息 + log_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../logfile') + log_dir = os.path.abspath(log_dir) + os.makedirs(log_dir, exist_ok=True) + + training_processes[process_id] = { + 'process': process, + 'train_type': 'pretrain', + 'log_file': os.path.join(log_dir, f'{process_id}.log'), + 'start_time': time.strftime('%Y-%m-%d %H:%M:%S'), + 'start_timestamp': time.time(), + 'running': True, + 'error': False, + 'train_monitor': 'none', + 'swanlab_url': None + } + + # 启动线程读取输出并写入日志文件 + def read_output(): + try: + log_file = training_processes[process_id]['log_file'] + with open(log_file, 'w') as f: + for line in iter(process.stdout.readline, ''): + if line: + f.write(line) + f.flush() + process.wait() + training_processes[process_id]['running'] = False + if process.returncode != 0: + training_processes[process_id]['error'] = True + except Exception as e: + print(f"读取测试进程输出时出错: {e}") + training_processes[process_id]['running'] = False + training_processes[process_id]['error'] = True + + threading.Thread(target=read_output, daemon=True).start() + + return jsonify({ + 'success': True, + 'process_id': process_id, + 'message': '测试进程已添加' + }) + @app.route('/processes') def processes(): result = []