From 095a656cff2350dd544d1ab46d0b9d8ec8dbe62c Mon Sep 17 00:00:00 2001 From: yuyu5333 <1812107659@qq.com> Date: Mon, 10 Nov 2025 10:00:44 +0000 Subject: [PATCH] support ddp --- scripts/static/js/script.js | 113 +++++++++++++++++-------- scripts/templates/index.html | 72 +++++++++++++++- scripts/train_web_ui.py | 159 +++++++++++++++++++++++++++-------- 3 files changed, 276 insertions(+), 68 deletions(-) diff --git a/scripts/static/js/script.js b/scripts/static/js/script.js index 0d0ceb3..287f7c0 100644 --- a/scripts/static/js/script.js +++ b/scripts/static/js/script.js @@ -533,6 +533,10 @@ function refreshLog(processId) { clearInterval(logTimers[processId]); delete logTimers[processId]; } + } else { + // 如果日志容器被隐藏，清除定时器 + clearInterval(logTimers[processId]); + delete logTimers[processId]; } }, 1000); } @@ -541,11 +545,33 @@ function refreshLog(processId) { // 加载日志内容 function loadLogContent(processId, logsContainer) { + // 显示加载状态 + const oldContent = logsContainer.textContent; + const isScrolledToBottom = logsContainer.scrollHeight - logsContainer.scrollTop <= logsContainer.clientHeight + 10; + fetch(`/logs/${processId}`) - .then(response => response.text()) + .then(response => { + if (!response.ok) { + throw new Error('获取日志失败'); + } + return response.text(); + }) .then(logs => { + // 更新日志内容 logsContainer.textContent = logs; - logsContainer.scrollTop = logsContainer.scrollHeight; + + // 如果用户之前滚动到底部，则继续保持在底部 + // 否则保持用户的滚动位置 + if (isScrolledToBottom || oldContent === logsContainer.textContent) { + logsContainer.scrollTop = logsContainer.scrollHeight; + } + }) + .catch(error => { + console.error('加载日志失败:', error); + // 只在内容为空或之前不是错误信息时才显示错误 + if (!logsContainer.textContent.includes('加载失败')) { + logsContainer.textContent = `加载日志失败: ${error.message}`; + } }); } @@ -815,7 +841,32 @@ function stopProcess(processId) { document.getElementById('train-form').addEventListener('submit', function(e) { e.preventDefault(); const formData = new FormData(this); - const data = Object.fromEntries(formData.entries()); + const data = {}; + const trainingMode = formData.get('training_mode'); + + // 处理表单数据，特别关注训练方式相关参数 + formData.forEach((value, key) => { + // 对于GPU数量，确保只有当选择了多卡训练时才添加到数据中 + if (key === 'gpu_num') { + const multiGpuSelection = document.getElementById('multi-gpu-selection'); + if (multiGpuSelection && multiGpuSelection.style.display !== 'none') { + data[key] = value; + } + } else if (key === 'device') { + // 处理device参数，根据训练方式设置正确格式 + if (trainingMode === 'single_gpu') { + // 单卡训练时，格式为cuda:序号 + data[key] = `cuda:${value}`; + } else if (trainingMode === 'cpu') { + // CPU训练时，固定为cpu + data[key] = 'cpu'; + } + // 多卡训练时不需要device参数，会通过torchrun自动设置 + } else if (key !== 'training_mode') { + // 排除training_mode参数，保留其他参数 + data[key] = value; + } + }); // 立即显示加载通知 showNotification('正在启动训练...', 'info'); @@ -1131,36 +1182,32 @@ function viewLogFile(filename, button) { if (logContainer.classList.contains('hidden')) { logContainer.classList.remove('hidden'); - // 如果是首次加载，获取日志内容 - if (logContainer.textContent.trim() === '') { - logContainer.textContent = '加载中...'; - - fetch(`/logfile-content/${encodeURIComponent(filename)}`) - .then(response => { - if (!response.ok) { - throw new Error('获取日志失败'); - } - return response.text(); - }) - .then(logs => { - // 保留空白格式，使用textContent而不是innerHTML - logContainer.textContent = logs; - logContainer.scrollTop = 0; - - // 更新内容容器的高度 - updateContentContainerHeight(); - }) - .catch(error => { - console.error('获取日志失败:', error); - logContainer.innerHTML = `

获取日志失败: ${error.message}

`; - - // 更新内容容器的高度 - updateContentContainerHeight(); - }); - } else { - // 如果不是首次加载，直接更新内容容器的高度 - updateContentContainerHeight(); - } + // 读取完整日志内容 + logContainer.textContent = '加载中...'; + + fetch(`/logfile-content/${encodeURIComponent(filename)}`) + .then(response => { + if (!response.ok) { + throw new Error('获取日志失败'); + } + return response.text(); + }) + .then(logs => { + // 保留空白格式，使用textContent而不是innerHTML + logContainer.textContent = logs; + // 默认滚动到顶部，显示完整历史 + logContainer.scrollTop = 0; + + // 更新内容容器的高度 + updateContentContainerHeight(); + }) + .catch(error => { + console.error('获取日志失败:', error); + logContainer.textContent = `获取日志失败: ${error.message}`; + + // 更新内容容器的高度 + updateContentContainerHeight(); + }); } else { logContainer.classList.add('hidden'); diff --git a/scripts/templates/index.html b/scripts/templates/index.html index 7d5b342..a49c4de 100644 --- a/scripts/templates/index.html +++ b/scripts/templates/index.html @@ -133,8 +133,21 @@

其他设置

- - + + +
+
+ + +
+
@@ -146,6 +159,61 @@
+ +
diff --git a/scripts/train_web_ui.py b/scripts/train_web_ui.py index d60fdcb..f640e12 100644 --- a/scripts/train_web_ui.py +++ b/scripts/train_web_ui.py @@ -7,6 +7,38 @@ import socket from flask import Flask, render_template, request, jsonify, redirect, url_for import time +# 尝试导入torch来检测GPU +try: + import torch + HAS_TORCH = True + # 检测可用的GPU数量和设备信息 + if torch.cuda.is_available(): + GPU_COUNT = torch.cuda.device_count() + # 获取GPU设备名称 + GPU_NAMES = [torch.cuda.get_device_name(i) for i in range(GPU_COUNT)] + else: + GPU_COUNT = 0 + GPU_NAMES = [] +except ImportError: + HAS_TORCH = False + GPU_COUNT = 0 + GPU_NAMES = [] + +# 训练方式支持检测 +def get_supported_training_methods(): + """获取当前环境支持的训练方法""" + methods = { + 'pretrain': True, # 预训练总是支持 + 'sft': True, # SFT总是支持 + 'lora': True, # LoRA总是支持 + 'dpo': True, # DPO总是支持 + 'multi_gpu': HAS_TORCH and GPU_COUNT > 1 # 多GPU训练需要PyTorch和多个GPU + } + return methods + +# 获取当前环境支持的训练方法 +SUPPORTED_METHODS = get_supported_training_methods() + sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) app = Flask(__name__, template_folder='templates', static_folder='static') @@ -28,25 +60,41 @@ def start_training_process(train_type, params): # 确保日志目录存在 os.makedirs(log_dir, exist_ok=True) + # 获取GPU数量参数，如果存在且大于1，则使用torchrun启动多卡训练 + gpu_num = int(params.get('gpu_num', 0)) if 'gpu_num' in params else 0 + use_torchrun = HAS_TORCH and GPU_COUNT > 0 and gpu_num > 1 + # 构建命令 if train_type == 'pretrain': script_path = '../trainer/train_pretrain.py' - cmd = [sys.executable, script_path] + if use_torchrun: + cmd = ['torchrun', '--nproc_per_node', str(gpu_num), script_path] + else: + cmd = [sys.executable, script_path] if 'save_weight' in params: cmd.extend(['--save_weight', params['save_weight']]) elif train_type == 'sft': script_path = '../trainer/train_full_sft.py' - cmd = [sys.executable, script_path] + if use_torchrun: + cmd = ['torchrun', '--nproc_per_node', str(gpu_num), script_path] + else: + cmd = [sys.executable, script_path] if 'save_weight' in params: cmd.extend(['--save_weight', params['save_weight']]) elif train_type == 'lora': script_path = '../trainer/train_lora.py' - cmd = [sys.executable, script_path] + if use_torchrun: + cmd = ['torchrun', '--nproc_per_node', str(gpu_num), script_path] + else: + cmd = [sys.executable, script_path] if 'lora_name' in params: cmd.extend(['--lora_name', params['lora_name']]) elif train_type == 'dpo': script_path = '../trainer/train_dpo.py' - cmd = [sys.executable, script_path] + if use_torchrun: + cmd = ['torchrun', '--nproc_per_node', str(gpu_num), script_path] + else: + cmd = [sys.executable, script_path] # 添加DPO特定参数 if 'beta' in params and params['beta']: cmd.extend(['--beta', params['beta']]) @@ -59,14 +107,14 @@ def start_training_process(train_type, params): # 添加通用参数 for key, value in params.items(): - # 跳过特殊参数和DPO特有参数 - if key in ['train_type', 'save_weight', 'lora_name', 'train_monitor', 'beta', 'accumulation_steps', 'grad_clip']: + # 跳过特殊参数和DPO特有参数，以及gpu_num参数（因为已经在torchrun命令中使用） + if key in ['train_type', 'save_weight', 'lora_name', 'train_monitor', 'beta', 'accumulation_steps', 'grad_clip', 'gpu_num']: continue - # 特殊处理布尔标志参数 + # 对于from_resume参数，需要正确传递参数值 if key == 'from_resume': - if value == '1': # 只有当值为1时才添加这个标志 - cmd.append(f'--{key}') + # 确保传递参数名和参数值 + cmd.extend([f'--{key}', str(value)]) else: # 确保log_interval和save_interval参数正确传递 cmd.extend([f'--{key}', str(value)]) @@ -126,7 +174,8 @@ def start_training_process(train_type, params): # Flask路由 @app.route('/') def index(): - return render_template('index.html') + # 传递GPU信息到前端 + return render_template('index.html', has_gpu=HAS_TORCH and GPU_COUNT > 0, gpu_count=GPU_COUNT) @app.route('/train', methods=['POST']) def train(): @@ -176,20 +225,62 @@ def logs(process_id): log_dir = os.path.join(script_dir, '../logfile') log_dir = os.path.abspath(log_dir) + # 查找匹配的日志文件 + log_file = None if os.path.exists(log_dir): for filename in os.listdir(log_dir): if filename.endswith(f'{process_id}.log'): log_file = os.path.join(log_dir, filename) - if os.path.exists(log_file): - try: - with open(log_file, 'r', encoding='utf-8') as f: - # 只读取最后200行 - lines = f.readlines() - last_200_lines = lines[-200:] if len(lines) > 200 else lines - return ''.join(last_200_lines) - except Exception as e: - return f'读取日志失败: {str(e)}' - return '日志文件不存在或已被删除' + break + + if not log_file or not os.path.exists(log_file): + return '日志文件不存在或已被删除' + + try: + # 使用高效的方法读取文件的最后200行 + # 这对于大文件特别有用，可以避免读取整个文件 + last_200_lines = [] + block_size = 8192 # 8KB blocks + + with open(log_file, 'r', encoding='utf-8') as f: + # 尝试直接定位到文件末尾，然后向前读取 + f.seek(0, os.SEEK_END) + file_size = f.tell() + + # 计算需要读取的块数 + position = file_size + blocks = [] + while position > 0: + # 后退一个块的位置 + position -= block_size + if position < 0: + position = 0 + + # 移动到计算的位置 + f.seek(position) + + # 读取这个块 + block = f.read(block_size) + blocks.append(block) + + # 如果已经收集了足够的行，就停止 + combined_text = ''.join(reversed(blocks)) + lines = combined_text.splitlines(True) + if len(lines) >= 200: + # 获取最后200行 + last_200_lines = lines[-200:] + break + + # 如果文件内容不足200行，或者上面的方法没有收集到足够的行 + if len(last_200_lines) < 200: + # 重新读取整个文件（对于小文件） + f.seek(0) + all_lines = f.readlines() + last_200_lines = all_lines[-200:] if len(all_lines) > 200 else all_lines + + return ''.join(last_200_lines) + except Exception as e: + return f'读取日志失败: {str(e)}' @app.route('/logfiles') def get_logfiles(): @@ -220,24 +311,26 @@ def get_logfiles(): @app.route('/logfile-content/') def get_logfile_content(filename): + # 安全检查：确保文件名不包含路径遍历字符 + if '..' in filename or '/' in filename or '\\' in filename: + return jsonify({'error': 'Invalid filename'}), 400 + # 获取脚本所在目录的绝对路径 script_dir = os.path.dirname(os.path.abspath(__file__)) - # 构建logfile目录的绝对路径 + # 构建logfile目录的绝对路径，train_web_ui.py在scripts目录下 log_dir = os.path.join(script_dir, '../logfile') log_dir = os.path.abspath(log_dir) - - # 安全检查：防止路径遍历攻击 - if '..' in filename or '/' in filename or '\\' in filename: - return '非法的文件名' - log_file = os.path.join(log_dir, filename) - if os.path.exists(log_file) and os.path.isfile(log_file): - try: - with open(log_file, 'r', encoding='utf-8') as f: - return f.read() - except Exception as e: - return f'读取日志失败: {str(e)}' - return '日志文件不存在' + + try: + # 读取完整的日志文件内容 + with open(log_file, 'r', encoding='utf-8') as f: + content = f.read() + return content + except FileNotFoundError: + return jsonify({'error': 'Log file not found'}), 404 + except Exception as e: + return jsonify({'error': str(e)}), 500 @app.route('/delete-logfile/', methods=['DELETE']) def delete_logfile(filename):