fix training logs show
yuyu5333
parent
9bfb58ad1e
commit
e9d2ccbb2d
|
|
@ -89,37 +89,30 @@ h1 {
|
|||
border-top: 1px solid #4d4d4d;
|
||||
}
|
||||
|
||||
/* 参数内容容器 */
|
||||
/* 参数内容容器 - 使用flex布局替代float */
|
||||
.parameter-content {
|
||||
width: 100%;
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
justify-content: space-between;
|
||||
gap: 15px;
|
||||
}
|
||||
|
||||
.form-group {
|
||||
width: 40%;
|
||||
float: left;
|
||||
margin-bottom: 15px;
|
||||
margin-right: 10%;
|
||||
width: 40%; /* 考虑gap间距，确保两个元素能并排 */
|
||||
margin-bottom: 0;
|
||||
margin-right: 5%;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
.form-group:nth-child(2n) {
|
||||
margin-right: 0;
|
||||
}
|
||||
|
||||
/* 确保复选框组占满整行 */
|
||||
.form-group.checkbox-group {
|
||||
width: 100%;
|
||||
margin-right: 0;
|
||||
}
|
||||
|
||||
.form-group.pretrain-sft, .form-group.lora {
|
||||
/* 移除100%宽度设置，让这些参数也遵循每行两个的布局 */
|
||||
}
|
||||
|
||||
.parameter-content::after {
|
||||
content: "";
|
||||
display: table;
|
||||
clear: both;
|
||||
/* 保持默认宽度，遵循每行两个的布局 */
|
||||
width: calc(40% - 8px);
|
||||
}
|
||||
label {
|
||||
display: block;
|
||||
|
|
@ -247,6 +240,10 @@ button:active {
|
|||
background: linear-gradient(135deg, #ff416c 0%, #ff4b2b 100%);
|
||||
color: white;
|
||||
}
|
||||
.status-manual-stop {
|
||||
background: linear-gradient(135deg, #4e54c8 0%, #8f94fb 100%);
|
||||
color: white;
|
||||
}
|
||||
.btn-stop {
|
||||
background: linear-gradient(135deg, #ff416c 0%, #ff4b2b 100%);
|
||||
padding: 8px 15px;
|
||||
|
|
|
|||
|
|
@ -89,16 +89,15 @@ function loadProcesses() {
|
|||
processItem.className = 'process-item';
|
||||
|
||||
let statusClass = '';
|
||||
let statusText = '';
|
||||
if (process.running) {
|
||||
// 根据后端返回的status字段设置状态类
|
||||
if (process.status === '运行中') {
|
||||
statusClass = 'status-running';
|
||||
statusText = '运行中';
|
||||
} else if (process.error) {
|
||||
} else if (process.status === '手动停止') {
|
||||
statusClass = 'status-manual-stop';
|
||||
} else if (process.status === '出错') {
|
||||
statusClass = 'status-error';
|
||||
statusText = '出错';
|
||||
} else {
|
||||
statusClass = 'status-completed';
|
||||
statusText = '已完成';
|
||||
}
|
||||
|
||||
processItem.innerHTML = `
|
||||
|
|
@ -107,11 +106,12 @@ function loadProcesses() {
|
|||
<strong>${process.train_type}</strong> - ${process.start_time}
|
||||
</div>
|
||||
<div>
|
||||
<span class="process-status ${statusClass}">${statusText}</span>
|
||||
<span class="process-status ${statusClass}">${process.status}</span>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<button class="btn-logs" onclick="showLogs('${process.id}')">查看日志</button>
|
||||
<button class="btn-logs" onclick="refreshLog('${process.id}')">刷新日志</button>
|
||||
${process.running ? `<button class="btn-stop" onclick="stopProcess('${process.id}')">停止训练</button>` : ''}
|
||||
</div>
|
||||
<div id="logs-${process.id}" class="logs-container hidden"></div>
|
||||
|
|
@ -128,13 +128,26 @@ function showLogs(processId) {
|
|||
logsContainer.classList.toggle('hidden');
|
||||
|
||||
if (!logsContainer.classList.contains('hidden')) {
|
||||
loadLogContent(processId, logsContainer);
|
||||
}
|
||||
}
|
||||
|
||||
// 刷新日志
|
||||
function refreshLog(processId) {
|
||||
const logsContainer = document.getElementById(`logs-${processId}`);
|
||||
if (!logsContainer.classList.contains('hidden')) {
|
||||
loadLogContent(processId, logsContainer);
|
||||
}
|
||||
}
|
||||
|
||||
// 加载日志内容
|
||||
function loadLogContent(processId, logsContainer) {
|
||||
fetch(`/logs/${processId}`)
|
||||
.then(response => response.text())
|
||||
.then(logs => {
|
||||
logsContainer.textContent = logs;
|
||||
logsContainer.scrollTop = logsContainer.scrollHeight;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 停止进程
|
||||
|
|
@ -206,10 +219,11 @@ function loadLogFiles() {
|
|||
fileItem.innerHTML = `
|
||||
<div class="process-info">
|
||||
<div>
|
||||
<strong>${trainType}</strong> - ${logfile.modified_time}
|
||||
<strong>${trainType}</strong> - ${logfile.filename}
|
||||
</div>
|
||||
<div>
|
||||
<span class="process-status status-completed">已保存</span>
|
||||
<span style="margin-left: 10px; color: #999; font-size: 0.9em;">${logfile.modified_time}</span>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
|
|
|
|||
|
|
@ -80,17 +80,6 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 硬件配置 -->
|
||||
<div class="parameter-card">
|
||||
<h3 class="card-title">硬件配置</h3>
|
||||
<div class="parameter-content">
|
||||
<div class="form-group">
|
||||
<label for="device">训练设备：</label>
|
||||
<input type="text" id="device" name="device" value="cuda:0" required>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 模型保存与恢复 -->
|
||||
<div class="parameter-card">
|
||||
<h3 class="card-title">模型保存与恢复</h3>
|
||||
|
|
@ -129,10 +118,15 @@
|
|||
<h3 class="card-title">其他设置</h3>
|
||||
<div class="parameter-content">
|
||||
<div class="form-group">
|
||||
<div class="checkbox-group">
|
||||
<input type="checkbox" id="use_wandb" name="use_wandb" value="1">
|
||||
<label for="use_wandb">是否使用wandb</label>
|
||||
<label for="device">训练设备：</label>
|
||||
<input type="text" id="device" name="device" value="cuda:0" required>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="train_monitor">训练监控：</label>
|
||||
<select id="train_monitor" name="train_monitor">
|
||||
<option value="none">无监控</option>
|
||||
<option value="wandb">使用WandB/SwanLab</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -48,15 +48,23 @@ def start_training_process(train_type, params):
|
|||
|
||||
# 添加通用参数
|
||||
for key, value in params.items():
|
||||
if key not in ['train_type', 'save_weight', 'lora_name']:
|
||||
# 跳过特殊参数
|
||||
if key in ['train_type', 'save_weight', 'lora_name', 'train_monitor']:
|
||||
continue
|
||||
|
||||
# 特殊处理布尔标志参数
|
||||
if key in ['use_wandb', 'from_resume']:
|
||||
if key == 'from_resume':
|
||||
if value == '1': # 只有当值为1时才添加这个标志
|
||||
cmd.append(f'--{key}')
|
||||
else:
|
||||
# 确保log_interval和save_interval参数正确传递
|
||||
cmd.extend([f'--{key}', str(value)])
|
||||
|
||||
# 单独处理训练监控参数，确保它不会被错误地添加值
|
||||
if 'train_monitor' in params:
|
||||
if params['train_monitor'] == 'wandb' or params['train_monitor'] == 'swanlab':
|
||||
cmd.append('--use_wandb') # 对于wandb和swanlab，只添加标志，不添加值
|
||||
|
||||
# 创建日志文件
|
||||
with open(log_file, 'w') as f:
|
||||
f.write(f"开始训练 {train_type} 进程\n")
|
||||
|
|
@ -118,8 +126,6 @@ def train():
|
|||
# 处理复选框参数
|
||||
if 'from_resume' not in params:
|
||||
params['from_resume'] = '0'
|
||||
if 'use_wandb' not in params:
|
||||
params['use_wandb'] = '0'
|
||||
|
||||
# 启动训练进程
|
||||
process_id = start_training_process(train_type, params)
|
||||
|
|
@ -133,12 +139,18 @@ def train():
|
|||
def processes():
|
||||
result = []
|
||||
for process_id, info in training_processes.items():
|
||||
# 确定状态
|
||||
status = '运行中' if info['running'] else \
|
||||
'手动停止' if 'manually_stopped' in info and info['manually_stopped'] else \
|
||||
'出错' if info['error'] else '已完成'
|
||||
|
||||
result.append({
|
||||
'id': process_id,
|
||||
'train_type': info['train_type'],
|
||||
'start_time': info['start_time'],
|
||||
'running': info['running'],
|
||||
'error': info['error']
|
||||
'error': info['error'],
|
||||
'status': status
|
||||
})
|
||||
return jsonify(result)
|
||||
|
||||
|
|
@ -158,7 +170,10 @@ def logs(process_id):
|
|||
if os.path.exists(log_file):
|
||||
try:
|
||||
with open(log_file, 'r', encoding='utf-8') as f:
|
||||
return f.read()
|
||||
# 只读取最后500行
|
||||
lines = f.readlines()
|
||||
last_500_lines = lines[-500:] if len(lines) > 500 else lines
|
||||
return ''.join(last_500_lines)
|
||||
except Exception as e:
|
||||
return f'读取日志失败: {str(e)}'
|
||||
return '日志文件不存在或已被删除'
|
||||
|
|
@ -186,6 +201,8 @@ def get_logfiles():
|
|||
})
|
||||
except Exception as e:
|
||||
continue
|
||||
# 按修改时间倒序排序，最新的在前面
|
||||
logfiles.sort(key=lambda x: x['modified_time'], reverse=True)
|
||||
return jsonify(logfiles)
|
||||
|
||||
@app.route('/logfile-content/<filename>')
|
||||
|
|
@ -219,11 +236,15 @@ def stop(process_id):
|
|||
process.terminate()
|
||||
# 等待进程结束
|
||||
process.wait(timeout=5)
|
||||
# 标记为手动停止
|
||||
training_processes[process_id]['running'] = False
|
||||
training_processes[process_id]['manually_stopped'] = True
|
||||
except subprocess.TimeoutExpired:
|
||||
# 如果超时，强制杀死
|
||||
process.kill()
|
||||
|
||||
# 标记为手动停止
|
||||
training_processes[process_id]['running'] = False
|
||||
training_processes[process_id]['manually_stopped'] = True
|
||||
return jsonify({'success': True})
|
||||
return jsonify({'success': False})
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue