diff --git a/trainer_web/static/js/processes/list.js b/trainer_web/static/js/processes/list.js index 5d3ea58..551c1ad 100644 --- a/trainer_web/static/js/processes/list.js +++ b/trainer_web/static/js/processes/list.js @@ -68,7 +68,11 @@ function calculateProgress(process) { remaining: process.progress.remaining_time || '计算中...', loss: process.progress.current_loss || null, epoch: process.progress.current_epoch ? `${process.progress.current_epoch}/${process.progress.total_epochs}` : null, - lr: process.progress.current_lr || null + lr: process.progress.current_lr || null, + step: process.progress.current_step && process.progress.total_steps ? + `${process.progress.current_step}/${process.progress.total_steps}` : null, + currentStep: process.progress.current_step || 0, + totalSteps: process.progress.total_steps || 0 }; } @@ -88,6 +92,9 @@ function calculateProgress(process) { let current = 0; let total = 0; let percentage = 0; + let currentStep = 0; + let totalSteps = 0; + let stepInfo = null; for (const pattern of epochPatterns) { const match = logText.match(pattern); @@ -99,6 +106,28 @@ function calculateProgress(process) { } } + // 提取step信息 - 支持多种格式 + const stepPatterns = [ + /step\s+(\d+)\s*\/\s*(\d+)/i, // step 150/1000 + /Step\s+(\d+)\s*of\s*(\d+)/i, // Step 150 of 1000 + /\[(\d+)\/(\d+)\]/i, // [150/1000] + /step\s*[:：]\s*(\d+)\s*\/\s*(\d+)/i, // step: 150/1000 + /第\s*(\d+)\s*步\s*\/\s*共\s*(\d+)\s*步/i, // 第150步/共1000步 + /步数\s*(\d+)\s*\/\s*(\d+)/i, // 步数 150/1000 + /batch\s+(\d+)\s*\/\s*(\d+)/i, // batch 150/1000 + /Batch\s+(\d+)\s*of\s*(\d+)/i // Batch 150 of 1000 + ]; + + for (const pattern of stepPatterns) { + const match = logText.match(pattern); + if (match) { + currentStep = parseInt(match[1]); + totalSteps = parseInt(match[2]); + stepInfo = `${currentStep}/${totalSteps}`; + break; + } + } + // 提取loss信息 - 支持多种格式 const lossPatterns = [ /loss[\s:=]\s*([\d.]+(?:e[+-]?\d+)?)/i, // loss: 4.32 or loss = 4.32 @@ -149,14 +178,29 @@ function calculateProgress(process) { // 如果找到了有效的epoch信息，返回进度 if (total > 0) { + // 重新计算百分比 - 支持epoch和step双重进度 + let finalPercentage = percentage; + if (totalSteps > 0 && currentStep > 0) { + // 基础epoch进度 + const epochPercentage = (current / total) * 100; + // 当前epoch内的step进度 + const stepPercentageInEpoch = (currentStep / totalSteps) * 100; + // 将step进度加到epoch进度上（每个epoch占总进度的1/total） + const stepContribution = stepPercentageInEpoch / total; + finalPercentage = Math.min(100, Math.max(0, Math.round(epochPercentage + stepContribution))); + } + return { - percentage, + percentage: finalPercentage, current, total, remaining: calculateRemainingTime(current, total, logText), loss: currentLoss, epoch: `${current}/${total}`, - lr: currentLr + lr: currentLr, + step: stepInfo, + currentStep, + totalSteps }; } } @@ -321,12 +365,13 @@ export function addProcessItemToGroup(parent, process) {
- 进度: ${progressInfo.current}/${progressInfo.total} + 进度: ${progressInfo.current}/${progressInfo.total}${progressInfo.step ? ` (${progressInfo.step})` : ''} 剩余时间: ${progressInfo.remaining}
${progressInfo.loss ? `
Loss:${progressInfo.loss}
` : ''} ${progressInfo.epoch ? `
Epoch:${progressInfo.epoch}
` : ''} + ${progressInfo.step ? `
Step:${progressInfo.step}
` : ''} ${progressInfo.lr ? `
LR:${progressInfo.lr}
` : ''}
@@ -379,7 +424,8 @@ export function updateProcessProgress(item, process) { } if (progressText) { - progressText.textContent = `进度: ${progressInfo.current}/${progressInfo.total}`; + const stepText = progressInfo.step ? ` (${progressInfo.step})` : ''; + progressText.textContent = `进度: ${progressInfo.current}/${progressInfo.total}${stepText}`; } if (remainingText) { @@ -390,7 +436,8 @@ export function updateProcessProgress(item, process) { // 更新指标 - 只更新有变化的值来减少DOM操作 const lossItem = metricsContainer.querySelector('.metric-item:nth-child(1) .metric-value'); const epochItem = metricsContainer.querySelector('.metric-item:nth-child(2) .metric-value'); - const lrItem = metricsContainer.querySelector('.metric-item:nth-child(3) .metric-value'); + const stepItem = metricsContainer.querySelector('.metric-item:nth-child(3) .metric-value'); + const lrItem = metricsContainer.querySelector('.metric-item:nth-child(4) .metric-value'); if (progressInfo.loss && lossItem) { lossItem.textContent = progressInfo.loss; @@ -398,6 +445,9 @@ export function updateProcessProgress(item, process) { if (progressInfo.epoch && epochItem) { epochItem.textContent = progressInfo.epoch; } + if (progressInfo.step && stepItem) { + stepItem.textContent = progressInfo.step; + } if (progressInfo.lr && lrItem) { lrItem.textContent = progressInfo.lr; } diff --git a/trainer_web/train_web_ui.py b/trainer_web/train_web_ui.py index f1cea06..4589d40 100644 --- a/trainer_web/train_web_ui.py +++ b/trainer_web/train_web_ui.py @@ -40,6 +40,8 @@ def calculate_training_progress(process_id, process_info): 'percentage': 0, 'current_epoch': 0, 'total_epochs': 0, + 'current_step': 0, + 'total_steps': 0, 'remaining_time': '计算中...', 'current_loss': None, 'current_lr': None @@ -110,6 +112,26 @@ def calculate_training_progress(process_id, process_info): total_epochs = int(match.group(2)) break + # 提取step信息 - 支持多种格式 + # 格式: step 150/1000, Step 150 of 1000, [150/1000], step: 150/1000 + step_patterns = [ + r'step\s+(\d+)\s*/\s*(\d+)', + r'Step\s+(\d+)\s*of\s*(\d+)', + r'\[(\d+)/(\d+)\]', + r'step\s*[:：]\s*(\d+)\s*/\s*(\d+)', + r'第\s*(\d+)\s*步\s*/\s*共\s*(\d+)\s*步', + r'步数\s*(\d+)\s*/\s*(\d+)', + r'batch\s+(\d+)\s*/\s*(\d+)', # 也支持batch格式 + r'Batch\s+(\d+)\s*of\s*(\d+)' + ] + + for pattern in step_patterns: + match = re.search(pattern, line, re.IGNORECASE) + if match: + progress['current_step'] = int(match.group(1)) + progress['total_steps'] = int(match.group(2)) + break + # 提取loss信息 - 支持多种格式 if not current_loss: # 格式: loss: 4.32, training_loss: 4.32, train_loss: 4.32, Loss: 4.32, 训练损失: 4.32 @@ -157,10 +179,29 @@ def calculate_training_progress(process_id, process_info): if total_epochs and current_loss and current_lr: break - # 计算进度百分比 + # 计算进度百分比 - 支持epoch和step双重进度 percentage = 0 if total_epochs > 0: - percentage = min(100, max(0, int((current_epoch / total_epochs) * 100))) + # 基础epoch进度 + epoch_percentage = (current_epoch / total_epochs) * 100 + + # 如果有step信息，在当前epoch内计算step进度 + if progress['total_steps'] > 0 and progress['current_step'] > 0: + # 计算当前epoch内的step进度 + step_percentage_in_epoch = (progress['current_step'] / progress['total_steps']) * 100 + # 将step进度加到epoch进度上（每个epoch占总进度的1/total_epochs） + step_contribution = step_percentage_in_epoch / total_epochs + percentage = min(100, max(0, int(epoch_percentage + step_contribution))) + else: + # 只有epoch信息的传统计算方式 + percentage = min(100, max(0, int(epoch_percentage))) + + # 更新进度字典 + progress['percentage'] = percentage + progress['current_epoch'] = current_epoch + progress['total_epochs'] = total_epochs + progress['current_loss'] = current_loss + progress['current_lr'] = current_lr # 估算剩余时间（增强计算） remaining_time = '计算中...' @@ -398,7 +439,7 @@ def add_test_process(): process_id = f"test_process_{int(time.time())}" - # 创建测试训练命令 + # 创建测试训练命令 - 包含step进度 test_command = [ 'python', '-c', ''' import time @@ -416,23 +457,28 @@ print("2024-11-21 14:30:02 - Model initialized with 108M parameters") sys.stdout.flush() time.sleep(2) -for epoch in range(1, 6): - print(f"2024-11-21 14:30:{5 + epoch*5} - Starting epoch {epoch}/5") - sys.stdout.flush() - time.sleep(1) - - loss = 4.5 - epoch * 0.3 - lr = 1e-4 * (0.9 ** epoch) - print(f"2024-11-21 14:30:{6 + epoch*5} - Loss: {loss:.4f}, lr: {lr:.2e}") - sys.stdout.flush() - time.sleep(2) - - remaining = (5 - epoch) * 15 - print(f"2024-11-21 14:30:{8 + epoch*5} - Epoch {epoch}/5 completed, remaining: 0:0{remaining}:00") - sys.stdout.flush() - time.sleep(1) +# 测试单epoch但多step的情况 +print("2024-11-21 14:30:03 - Starting epoch 1/1") +sys.stdout.flush() +time.sleep(1) -print("2024-11-21 14:30:35 - Training completed successfully") +total_steps = 20 +for step in range(1, total_steps + 1): + # 模拟step进度 + if step % 5 == 0 or step == total_steps: + print(f"2024-11-21 14:30:{4 + step} - Step {step}/{total_steps}") + sys.stdout.flush() + + # 模拟训练过程 + loss = 4.5 - step * 0.1 + lr = 1e-4 * (0.95 ** step) + if step % 3 == 0: + print(f"2024-11-21 14:30:{4 + step} - Loss: {loss:.4f}, lr: {lr:.2e}") + sys.stdout.flush() + + time.sleep(0.5) + +print("2024-11-21 14:30:25 - Training completed successfully") sys.stdout.flush() ''' ] diff --git a/trainer_web/training_processes.json b/trainer_web/training_processes.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/trainer_web/training_processes.json @@ -0,0 +1 @@ +{} \ No newline at end of file