fix training logs show

pull/524/head
yuyu5333 2025-11-07 03:11:22 +00:00
parent 9bfb58ad1e
commit e9d2ccbb2d
4 changed files with 86 additions and 60 deletions

View File

@ -89,37 +89,30 @@ h1 {
border-top: 1px solid #4d4d4d; border-top: 1px solid #4d4d4d;
} }
/* 参数内容容器 */ /* 参数内容容器 - 使用flex布局替代float */
.parameter-content { .parameter-content {
width: 100%; width: 100%;
display: flex;
flex-wrap: wrap;
justify-content: space-between;
gap: 15px;
} }
.form-group { .form-group {
width: 40%; width: 40%; /* 考虑gap间距,确保两个元素能并排 */
float: left; margin-bottom: 0;
margin-bottom: 15px; margin-right: 5%;
margin-right: 10%;
box-sizing: border-box; box-sizing: border-box;
} }
.form-group:nth-child(2n) {
margin-right: 0;
}
/* 确保复选框组占满整行 */ /* 确保复选框组占满整行 */
.form-group.checkbox-group { .form-group.checkbox-group {
width: 100%; width: 100%;
margin-right: 0;
} }
.form-group.pretrain-sft, .form-group.lora { .form-group.pretrain-sft, .form-group.lora {
/* 移除100%宽度设置,让这些参数也遵循每行两个的布局 */ /* 保持默认宽度,遵循每行两个的布局 */
} width: calc(40% - 8px);
.parameter-content::after {
content: "";
display: table;
clear: both;
} }
label { label {
display: block; display: block;
@ -247,6 +240,10 @@ button:active {
background: linear-gradient(135deg, #ff416c 0%, #ff4b2b 100%); background: linear-gradient(135deg, #ff416c 0%, #ff4b2b 100%);
color: white; color: white;
} }
.status-manual-stop {
background: linear-gradient(135deg, #4e54c8 0%, #8f94fb 100%);
color: white;
}
.btn-stop { .btn-stop {
background: linear-gradient(135deg, #ff416c 0%, #ff4b2b 100%); background: linear-gradient(135deg, #ff416c 0%, #ff4b2b 100%);
padding: 8px 15px; padding: 8px 15px;

View File

@ -89,16 +89,15 @@ function loadProcesses() {
processItem.className = 'process-item'; processItem.className = 'process-item';
let statusClass = ''; let statusClass = '';
let statusText = ''; // 根据后端返回的status字段设置状态类
if (process.running) { if (process.status === '运行中') {
statusClass = 'status-running'; statusClass = 'status-running';
statusText = '运行中'; } else if (process.status === '手动停止') {
} else if (process.error) { statusClass = 'status-manual-stop';
} else if (process.status === '出错') {
statusClass = 'status-error'; statusClass = 'status-error';
statusText = '出错';
} else { } else {
statusClass = 'status-completed'; statusClass = 'status-completed';
statusText = '已完成';
} }
processItem.innerHTML = ` processItem.innerHTML = `
@ -107,11 +106,12 @@ function loadProcesses() {
<strong>${process.train_type}</strong> - ${process.start_time} <strong>${process.train_type}</strong> - ${process.start_time}
</div> </div>
<div> <div>
<span class="process-status ${statusClass}">${statusText}</span> <span class="process-status ${statusClass}">${process.status}</span>
</div> </div>
</div> </div>
<div> <div>
<button class="btn-logs" onclick="showLogs('${process.id}')">查看日志</button> <button class="btn-logs" onclick="showLogs('${process.id}')">查看日志</button>
<button class="btn-logs" onclick="refreshLog('${process.id}')">刷新日志</button>
${process.running ? `<button class="btn-stop" onclick="stopProcess('${process.id}')">停止训练</button>` : ''} ${process.running ? `<button class="btn-stop" onclick="stopProcess('${process.id}')">停止训练</button>` : ''}
</div> </div>
<div id="logs-${process.id}" class="logs-container hidden"></div> <div id="logs-${process.id}" class="logs-container hidden"></div>
@ -128,6 +128,20 @@ function showLogs(processId) {
logsContainer.classList.toggle('hidden'); logsContainer.classList.toggle('hidden');
if (!logsContainer.classList.contains('hidden')) { if (!logsContainer.classList.contains('hidden')) {
loadLogContent(processId, logsContainer);
}
}
// 刷新日志
function refreshLog(processId) {
const logsContainer = document.getElementById(`logs-${processId}`);
if (!logsContainer.classList.contains('hidden')) {
loadLogContent(processId, logsContainer);
}
}
// 加载日志内容
function loadLogContent(processId, logsContainer) {
fetch(`/logs/${processId}`) fetch(`/logs/${processId}`)
.then(response => response.text()) .then(response => response.text())
.then(logs => { .then(logs => {
@ -135,7 +149,6 @@ function showLogs(processId) {
logsContainer.scrollTop = logsContainer.scrollHeight; logsContainer.scrollTop = logsContainer.scrollHeight;
}); });
} }
}
// 停止进程 // 停止进程
function stopProcess(processId) { function stopProcess(processId) {
@ -206,10 +219,11 @@ function loadLogFiles() {
fileItem.innerHTML = ` fileItem.innerHTML = `
<div class="process-info"> <div class="process-info">
<div> <div>
<strong>${trainType}</strong> - ${logfile.modified_time} <strong>${trainType}</strong> - ${logfile.filename}
</div> </div>
<div> <div>
<span class="process-status status-completed">已保存</span> <span class="process-status status-completed">已保存</span>
<span style="margin-left: 10px; color: #999; font-size: 0.9em;">${logfile.modified_time}</span>
</div> </div>
</div> </div>
<div> <div>

View File

@ -80,17 +80,6 @@
</div> </div>
</div> </div>
<!-- 硬件配置 -->
<div class="parameter-card">
<h3 class="card-title">硬件配置</h3>
<div class="parameter-content">
<div class="form-group">
<label for="device">训练设备:</label>
<input type="text" id="device" name="device" value="cuda:0" required>
</div>
</div>
</div>
<!-- 模型保存与恢复 --> <!-- 模型保存与恢复 -->
<div class="parameter-card"> <div class="parameter-card">
<h3 class="card-title">模型保存与恢复</h3> <h3 class="card-title">模型保存与恢复</h3>
@ -129,10 +118,15 @@
<h3 class="card-title">其他设置</h3> <h3 class="card-title">其他设置</h3>
<div class="parameter-content"> <div class="parameter-content">
<div class="form-group"> <div class="form-group">
<div class="checkbox-group"> <label for="device">训练设备:</label>
<input type="checkbox" id="use_wandb" name="use_wandb" value="1"> <input type="text" id="device" name="device" value="cuda:0" required>
<label for="use_wandb">是否使用wandb</label>
</div> </div>
<div class="form-group">
<label for="train_monitor">训练监控:</label>
<select id="train_monitor" name="train_monitor">
<option value="none">无监控</option>
<option value="wandb">使用WandB/SwanLab</option>
</select>
</div> </div>
</div> </div>
</div> </div>

View File

@ -48,15 +48,23 @@ def start_training_process(train_type, params):
# 添加通用参数 # 添加通用参数
for key, value in params.items(): for key, value in params.items():
if key not in ['train_type', 'save_weight', 'lora_name']: # 跳过特殊参数
if key in ['train_type', 'save_weight', 'lora_name', 'train_monitor']:
continue
# 特殊处理布尔标志参数 # 特殊处理布尔标志参数
if key in ['use_wandb', 'from_resume']: if key == 'from_resume':
if value == '1': # 只有当值为1时才添加这个标志 if value == '1': # 只有当值为1时才添加这个标志
cmd.append(f'--{key}') cmd.append(f'--{key}')
else: else:
# 确保log_interval和save_interval参数正确传递 # 确保log_interval和save_interval参数正确传递
cmd.extend([f'--{key}', str(value)]) cmd.extend([f'--{key}', str(value)])
# 单独处理训练监控参数,确保它不会被错误地添加值
if 'train_monitor' in params:
if params['train_monitor'] == 'wandb' or params['train_monitor'] == 'swanlab':
cmd.append('--use_wandb') # 对于wandb和swanlab,只添加标志,不添加值
# 创建日志文件 # 创建日志文件
with open(log_file, 'w') as f: with open(log_file, 'w') as f:
f.write(f"开始训练 {train_type} 进程\n") f.write(f"开始训练 {train_type} 进程\n")
@ -118,8 +126,6 @@ def train():
# 处理复选框参数 # 处理复选框参数
if 'from_resume' not in params: if 'from_resume' not in params:
params['from_resume'] = '0' params['from_resume'] = '0'
if 'use_wandb' not in params:
params['use_wandb'] = '0'
# 启动训练进程 # 启动训练进程
process_id = start_training_process(train_type, params) process_id = start_training_process(train_type, params)
@ -133,12 +139,18 @@ def train():
def processes(): def processes():
result = [] result = []
for process_id, info in training_processes.items(): for process_id, info in training_processes.items():
# 确定状态
status = '运行中' if info['running'] else \
'手动停止' if 'manually_stopped' in info and info['manually_stopped'] else \
'出错' if info['error'] else '已完成'
result.append({ result.append({
'id': process_id, 'id': process_id,
'train_type': info['train_type'], 'train_type': info['train_type'],
'start_time': info['start_time'], 'start_time': info['start_time'],
'running': info['running'], 'running': info['running'],
'error': info['error'] 'error': info['error'],
'status': status
}) })
return jsonify(result) return jsonify(result)
@ -158,7 +170,10 @@ def logs(process_id):
if os.path.exists(log_file): if os.path.exists(log_file):
try: try:
with open(log_file, 'r', encoding='utf-8') as f: with open(log_file, 'r', encoding='utf-8') as f:
return f.read() # 只读取最后500行
lines = f.readlines()
last_500_lines = lines[-500:] if len(lines) > 500 else lines
return ''.join(last_500_lines)
except Exception as e: except Exception as e:
return f'读取日志失败: {str(e)}' return f'读取日志失败: {str(e)}'
return '日志文件不存在或已被删除' return '日志文件不存在或已被删除'
@ -186,6 +201,8 @@ def get_logfiles():
}) })
except Exception as e: except Exception as e:
continue continue
# 按修改时间倒序排序,最新的在前面
logfiles.sort(key=lambda x: x['modified_time'], reverse=True)
return jsonify(logfiles) return jsonify(logfiles)
@app.route('/logfile-content/<filename>') @app.route('/logfile-content/<filename>')
@ -219,11 +236,15 @@ def stop(process_id):
process.terminate() process.terminate()
# 等待进程结束 # 等待进程结束
process.wait(timeout=5) process.wait(timeout=5)
# 标记为手动停止
training_processes[process_id]['running'] = False
training_processes[process_id]['manually_stopped'] = True
except subprocess.TimeoutExpired: except subprocess.TimeoutExpired:
# 如果超时,强制杀死 # 如果超时,强制杀死
process.kill() process.kill()
# 标记为手动停止
training_processes[process_id]['running'] = False training_processes[process_id]['running'] = False
training_processes[process_id]['manually_stopped'] = True
return jsonify({'success': True}) return jsonify({'success': True})
return jsonify({'success': False}) return jsonify({'success': False})