fix training logs show
parent
9bfb58ad1e
commit
e9d2ccbb2d
|
|
@ -89,37 +89,30 @@ h1 {
|
||||||
border-top: 1px solid #4d4d4d;
|
border-top: 1px solid #4d4d4d;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* 参数内容容器 */
|
/* 参数内容容器 - 使用flex布局替代float */
|
||||||
.parameter-content {
|
.parameter-content {
|
||||||
width: 100%;
|
width: 100%;
|
||||||
|
display: flex;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 15px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.form-group {
|
.form-group {
|
||||||
width: 40%;
|
width: 40%; /* 考虑gap间距,确保两个元素能并排 */
|
||||||
float: left;
|
margin-bottom: 0;
|
||||||
margin-bottom: 15px;
|
margin-right: 5%;
|
||||||
margin-right: 10%;
|
|
||||||
box-sizing: border-box;
|
box-sizing: border-box;
|
||||||
}
|
}
|
||||||
|
|
||||||
.form-group:nth-child(2n) {
|
|
||||||
margin-right: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* 确保复选框组占满整行 */
|
/* 确保复选框组占满整行 */
|
||||||
.form-group.checkbox-group {
|
.form-group.checkbox-group {
|
||||||
width: 100%;
|
width: 100%;
|
||||||
margin-right: 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
.form-group.pretrain-sft, .form-group.lora {
|
.form-group.pretrain-sft, .form-group.lora {
|
||||||
/* 移除100%宽度设置,让这些参数也遵循每行两个的布局 */
|
/* 保持默认宽度,遵循每行两个的布局 */
|
||||||
}
|
width: calc(40% - 8px);
|
||||||
|
|
||||||
.parameter-content::after {
|
|
||||||
content: "";
|
|
||||||
display: table;
|
|
||||||
clear: both;
|
|
||||||
}
|
}
|
||||||
label {
|
label {
|
||||||
display: block;
|
display: block;
|
||||||
|
|
@ -247,6 +240,10 @@ button:active {
|
||||||
background: linear-gradient(135deg, #ff416c 0%, #ff4b2b 100%);
|
background: linear-gradient(135deg, #ff416c 0%, #ff4b2b 100%);
|
||||||
color: white;
|
color: white;
|
||||||
}
|
}
|
||||||
|
.status-manual-stop {
|
||||||
|
background: linear-gradient(135deg, #4e54c8 0%, #8f94fb 100%);
|
||||||
|
color: white;
|
||||||
|
}
|
||||||
.btn-stop {
|
.btn-stop {
|
||||||
background: linear-gradient(135deg, #ff416c 0%, #ff4b2b 100%);
|
background: linear-gradient(135deg, #ff416c 0%, #ff4b2b 100%);
|
||||||
padding: 8px 15px;
|
padding: 8px 15px;
|
||||||
|
|
|
||||||
|
|
@ -89,16 +89,15 @@ function loadProcesses() {
|
||||||
processItem.className = 'process-item';
|
processItem.className = 'process-item';
|
||||||
|
|
||||||
let statusClass = '';
|
let statusClass = '';
|
||||||
let statusText = '';
|
// 根据后端返回的status字段设置状态类
|
||||||
if (process.running) {
|
if (process.status === '运行中') {
|
||||||
statusClass = 'status-running';
|
statusClass = 'status-running';
|
||||||
statusText = '运行中';
|
} else if (process.status === '手动停止') {
|
||||||
} else if (process.error) {
|
statusClass = 'status-manual-stop';
|
||||||
|
} else if (process.status === '出错') {
|
||||||
statusClass = 'status-error';
|
statusClass = 'status-error';
|
||||||
statusText = '出错';
|
|
||||||
} else {
|
} else {
|
||||||
statusClass = 'status-completed';
|
statusClass = 'status-completed';
|
||||||
statusText = '已完成';
|
|
||||||
}
|
}
|
||||||
|
|
||||||
processItem.innerHTML = `
|
processItem.innerHTML = `
|
||||||
|
|
@ -107,11 +106,12 @@ function loadProcesses() {
|
||||||
<strong>${process.train_type}</strong> - ${process.start_time}
|
<strong>${process.train_type}</strong> - ${process.start_time}
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<span class="process-status ${statusClass}">${statusText}</span>
|
<span class="process-status ${statusClass}">${process.status}</span>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<button class="btn-logs" onclick="showLogs('${process.id}')">查看日志</button>
|
<button class="btn-logs" onclick="showLogs('${process.id}')">查看日志</button>
|
||||||
|
<button class="btn-logs" onclick="refreshLog('${process.id}')">刷新日志</button>
|
||||||
${process.running ? `<button class="btn-stop" onclick="stopProcess('${process.id}')">停止训练</button>` : ''}
|
${process.running ? `<button class="btn-stop" onclick="stopProcess('${process.id}')">停止训练</button>` : ''}
|
||||||
</div>
|
</div>
|
||||||
<div id="logs-${process.id}" class="logs-container hidden"></div>
|
<div id="logs-${process.id}" class="logs-container hidden"></div>
|
||||||
|
|
@ -128,6 +128,20 @@ function showLogs(processId) {
|
||||||
logsContainer.classList.toggle('hidden');
|
logsContainer.classList.toggle('hidden');
|
||||||
|
|
||||||
if (!logsContainer.classList.contains('hidden')) {
|
if (!logsContainer.classList.contains('hidden')) {
|
||||||
|
loadLogContent(processId, logsContainer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 刷新日志
|
||||||
|
function refreshLog(processId) {
|
||||||
|
const logsContainer = document.getElementById(`logs-${processId}`);
|
||||||
|
if (!logsContainer.classList.contains('hidden')) {
|
||||||
|
loadLogContent(processId, logsContainer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 加载日志内容
|
||||||
|
function loadLogContent(processId, logsContainer) {
|
||||||
fetch(`/logs/${processId}`)
|
fetch(`/logs/${processId}`)
|
||||||
.then(response => response.text())
|
.then(response => response.text())
|
||||||
.then(logs => {
|
.then(logs => {
|
||||||
|
|
@ -135,7 +149,6 @@ function showLogs(processId) {
|
||||||
logsContainer.scrollTop = logsContainer.scrollHeight;
|
logsContainer.scrollTop = logsContainer.scrollHeight;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// 停止进程
|
// 停止进程
|
||||||
function stopProcess(processId) {
|
function stopProcess(processId) {
|
||||||
|
|
@ -206,10 +219,11 @@ function loadLogFiles() {
|
||||||
fileItem.innerHTML = `
|
fileItem.innerHTML = `
|
||||||
<div class="process-info">
|
<div class="process-info">
|
||||||
<div>
|
<div>
|
||||||
<strong>${trainType}</strong> - ${logfile.modified_time}
|
<strong>${trainType}</strong> - ${logfile.filename}
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<span class="process-status status-completed">已保存</span>
|
<span class="process-status status-completed">已保存</span>
|
||||||
|
<span style="margin-left: 10px; color: #999; font-size: 0.9em;">${logfile.modified_time}</span>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
|
|
|
||||||
|
|
@ -80,17 +80,6 @@
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- 硬件配置 -->
|
|
||||||
<div class="parameter-card">
|
|
||||||
<h3 class="card-title">硬件配置</h3>
|
|
||||||
<div class="parameter-content">
|
|
||||||
<div class="form-group">
|
|
||||||
<label for="device">训练设备:</label>
|
|
||||||
<input type="text" id="device" name="device" value="cuda:0" required>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<!-- 模型保存与恢复 -->
|
<!-- 模型保存与恢复 -->
|
||||||
<div class="parameter-card">
|
<div class="parameter-card">
|
||||||
<h3 class="card-title">模型保存与恢复</h3>
|
<h3 class="card-title">模型保存与恢复</h3>
|
||||||
|
|
@ -129,10 +118,15 @@
|
||||||
<h3 class="card-title">其他设置</h3>
|
<h3 class="card-title">其他设置</h3>
|
||||||
<div class="parameter-content">
|
<div class="parameter-content">
|
||||||
<div class="form-group">
|
<div class="form-group">
|
||||||
<div class="checkbox-group">
|
<label for="device">训练设备:</label>
|
||||||
<input type="checkbox" id="use_wandb" name="use_wandb" value="1">
|
<input type="text" id="device" name="device" value="cuda:0" required>
|
||||||
<label for="use_wandb">是否使用wandb</label>
|
|
||||||
</div>
|
</div>
|
||||||
|
<div class="form-group">
|
||||||
|
<label for="train_monitor">训练监控:</label>
|
||||||
|
<select id="train_monitor" name="train_monitor">
|
||||||
|
<option value="none">无监控</option>
|
||||||
|
<option value="wandb">使用WandB/SwanLab</option>
|
||||||
|
</select>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -48,15 +48,23 @@ def start_training_process(train_type, params):
|
||||||
|
|
||||||
# 添加通用参数
|
# 添加通用参数
|
||||||
for key, value in params.items():
|
for key, value in params.items():
|
||||||
if key not in ['train_type', 'save_weight', 'lora_name']:
|
# 跳过特殊参数
|
||||||
|
if key in ['train_type', 'save_weight', 'lora_name', 'train_monitor']:
|
||||||
|
continue
|
||||||
|
|
||||||
# 特殊处理布尔标志参数
|
# 特殊处理布尔标志参数
|
||||||
if key in ['use_wandb', 'from_resume']:
|
if key == 'from_resume':
|
||||||
if value == '1': # 只有当值为1时才添加这个标志
|
if value == '1': # 只有当值为1时才添加这个标志
|
||||||
cmd.append(f'--{key}')
|
cmd.append(f'--{key}')
|
||||||
else:
|
else:
|
||||||
# 确保log_interval和save_interval参数正确传递
|
# 确保log_interval和save_interval参数正确传递
|
||||||
cmd.extend([f'--{key}', str(value)])
|
cmd.extend([f'--{key}', str(value)])
|
||||||
|
|
||||||
|
# 单独处理训练监控参数,确保它不会被错误地添加值
|
||||||
|
if 'train_monitor' in params:
|
||||||
|
if params['train_monitor'] == 'wandb' or params['train_monitor'] == 'swanlab':
|
||||||
|
cmd.append('--use_wandb') # 对于wandb和swanlab,只添加标志,不添加值
|
||||||
|
|
||||||
# 创建日志文件
|
# 创建日志文件
|
||||||
with open(log_file, 'w') as f:
|
with open(log_file, 'w') as f:
|
||||||
f.write(f"开始训练 {train_type} 进程\n")
|
f.write(f"开始训练 {train_type} 进程\n")
|
||||||
|
|
@ -118,8 +126,6 @@ def train():
|
||||||
# 处理复选框参数
|
# 处理复选框参数
|
||||||
if 'from_resume' not in params:
|
if 'from_resume' not in params:
|
||||||
params['from_resume'] = '0'
|
params['from_resume'] = '0'
|
||||||
if 'use_wandb' not in params:
|
|
||||||
params['use_wandb'] = '0'
|
|
||||||
|
|
||||||
# 启动训练进程
|
# 启动训练进程
|
||||||
process_id = start_training_process(train_type, params)
|
process_id = start_training_process(train_type, params)
|
||||||
|
|
@ -133,12 +139,18 @@ def train():
|
||||||
def processes():
|
def processes():
|
||||||
result = []
|
result = []
|
||||||
for process_id, info in training_processes.items():
|
for process_id, info in training_processes.items():
|
||||||
|
# 确定状态
|
||||||
|
status = '运行中' if info['running'] else \
|
||||||
|
'手动停止' if 'manually_stopped' in info and info['manually_stopped'] else \
|
||||||
|
'出错' if info['error'] else '已完成'
|
||||||
|
|
||||||
result.append({
|
result.append({
|
||||||
'id': process_id,
|
'id': process_id,
|
||||||
'train_type': info['train_type'],
|
'train_type': info['train_type'],
|
||||||
'start_time': info['start_time'],
|
'start_time': info['start_time'],
|
||||||
'running': info['running'],
|
'running': info['running'],
|
||||||
'error': info['error']
|
'error': info['error'],
|
||||||
|
'status': status
|
||||||
})
|
})
|
||||||
return jsonify(result)
|
return jsonify(result)
|
||||||
|
|
||||||
|
|
@ -158,7 +170,10 @@ def logs(process_id):
|
||||||
if os.path.exists(log_file):
|
if os.path.exists(log_file):
|
||||||
try:
|
try:
|
||||||
with open(log_file, 'r', encoding='utf-8') as f:
|
with open(log_file, 'r', encoding='utf-8') as f:
|
||||||
return f.read()
|
# 只读取最后500行
|
||||||
|
lines = f.readlines()
|
||||||
|
last_500_lines = lines[-500:] if len(lines) > 500 else lines
|
||||||
|
return ''.join(last_500_lines)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
return f'读取日志失败: {str(e)}'
|
return f'读取日志失败: {str(e)}'
|
||||||
return '日志文件不存在或已被删除'
|
return '日志文件不存在或已被删除'
|
||||||
|
|
@ -186,6 +201,8 @@ def get_logfiles():
|
||||||
})
|
})
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
continue
|
continue
|
||||||
|
# 按修改时间倒序排序,最新的在前面
|
||||||
|
logfiles.sort(key=lambda x: x['modified_time'], reverse=True)
|
||||||
return jsonify(logfiles)
|
return jsonify(logfiles)
|
||||||
|
|
||||||
@app.route('/logfile-content/<filename>')
|
@app.route('/logfile-content/<filename>')
|
||||||
|
|
@ -219,11 +236,15 @@ def stop(process_id):
|
||||||
process.terminate()
|
process.terminate()
|
||||||
# 等待进程结束
|
# 等待进程结束
|
||||||
process.wait(timeout=5)
|
process.wait(timeout=5)
|
||||||
|
# 标记为手动停止
|
||||||
|
training_processes[process_id]['running'] = False
|
||||||
|
training_processes[process_id]['manually_stopped'] = True
|
||||||
except subprocess.TimeoutExpired:
|
except subprocess.TimeoutExpired:
|
||||||
# 如果超时,强制杀死
|
# 如果超时,强制杀死
|
||||||
process.kill()
|
process.kill()
|
||||||
|
# 标记为手动停止
|
||||||
training_processes[process_id]['running'] = False
|
training_processes[process_id]['running'] = False
|
||||||
|
training_processes[process_id]['manually_stopped'] = True
|
||||||
return jsonify({'success': True})
|
return jsonify({'success': True})
|
||||||
return jsonify({'success': False})
|
return jsonify({'success': False})
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue