support ddp

scripts/static/js/script.js

@@ -533,6 +533,10 @@ function refreshLog(processId) {
                         clearInterval(logTimers[processId]);
                         delete logTimers[processId];
                     }
+                } else {
+                    // 如果日志容器被隐藏，清除定时器
+                    clearInterval(logTimers[processId]);
+                    delete logTimers[processId];
                 }
             }, 1000);
         }
@@ -541,11 +545,33 @@ function refreshLog(processId) {
 
 // 加载日志内容
 function loadLogContent(processId, logsContainer) {
+    // 显示加载状态
+    const oldContent = logsContainer.textContent;
+    const isScrolledToBottom = logsContainer.scrollHeight - logsContainer.scrollTop <= logsContainer.clientHeight + 10;
+    
     fetch(`/logs/${processId}`)
-        .then(response => response.text())
+        .then(response => {
+            if (!response.ok) {
+                throw new Error('获取日志失败');
+            }
+            return response.text();
+        })
         .then(logs => {
+            // 更新日志内容
             logsContainer.textContent = logs;
-            logsContainer.scrollTop = logsContainer.scrollHeight;
+            
+            // 如果用户之前滚动到底部，则继续保持在底部
+            // 否则保持用户的滚动位置
+            if (isScrolledToBottom || oldContent === logsContainer.textContent) {
+                logsContainer.scrollTop = logsContainer.scrollHeight;
+            }
+        })
+        .catch(error => {
+            console.error('加载日志失败:', error);
+            // 只在内容为空或之前不是错误信息时才显示错误
+            if (!logsContainer.textContent.includes('加载失败')) {
+                logsContainer.textContent = `加载日志失败: ${error.message}`;
+            }
         });
 }
 
@@ -815,7 +841,32 @@ function stopProcess(processId) {
 document.getElementById('train-form').addEventListener('submit', function(e) {
     e.preventDefault();
     const formData = new FormData(this);
-    const data = Object.fromEntries(formData.entries());
+    const data = {};
+    const trainingMode = formData.get('training_mode');
+    
+    // 处理表单数据，特别关注训练方式相关参数
+    formData.forEach((value, key) => {
+        // 对于GPU数量，确保只有当选择了多卡训练时才添加到数据中
+        if (key === 'gpu_num') {
+            const multiGpuSelection = document.getElementById('multi-gpu-selection');
+            if (multiGpuSelection && multiGpuSelection.style.display !== 'none') {
+                data[key] = value;
+            }
+        } else if (key === 'device') {
+            // 处理device参数，根据训练方式设置正确格式
+            if (trainingMode === 'single_gpu') {
+                // 单卡训练时，格式为cuda:序号
+                data[key] = `cuda:${value}`;
+            } else if (trainingMode === 'cpu') {
+                // CPU训练时，固定为cpu
+                data[key] = 'cpu';
+            }
+            // 多卡训练时不需要device参数，会通过torchrun自动设置
+        } else if (key !== 'training_mode') {
+            // 排除training_mode参数，保留其他参数
+            data[key] = value;
+        }
+    });
     
     // 立即显示加载通知
     showNotification('正在启动训练...', 'info');
@@ -1131,36 +1182,32 @@ function viewLogFile(filename, button) {
     if (logContainer.classList.contains('hidden')) {
         logContainer.classList.remove('hidden');
         
-        // 如果是首次加载，获取日志内容
+        // 读取完整日志内容
-        if (logContainer.textContent.trim() === '') {
+        logContainer.textContent = '加载中...';
-            logContainer.textContent = '加载中...';
         
-            fetch(`/logfile-content/${encodeURIComponent(filename)}`)
+        fetch(`/logfile-content/${encodeURIComponent(filename)}`)
-                .then(response => {
+            .then(response => {
-                    if (!response.ok) {
+                if (!response.ok) {
-                        throw new Error('获取日志失败');
+                    throw new Error('获取日志失败');
-                    }
+                }
-                    return response.text();
+                return response.text();
-                })
+            })
-                .then(logs => {
+            .then(logs => {
-                    // 保留空白格式，使用textContent而不是innerHTML
+                // 保留空白格式，使用textContent而不是innerHTML
-                    logContainer.textContent = logs;
+                logContainer.textContent = logs;
-                    logContainer.scrollTop = 0;
+                // 默认滚动到顶部，显示完整历史
+                logContainer.scrollTop = 0;
                 
-                    // 更新内容容器的高度
+                // 更新内容容器的高度
-                    updateContentContainerHeight();
+                updateContentContainerHeight();
-                })
+            })
-                .catch(error => {
+            .catch(error => {
-                    console.error('获取日志失败:', error);
+                console.error('获取日志失败:', error);
-                    logContainer.innerHTML = `<p class="error">获取日志失败: ${error.message}</p>`;
+                logContainer.textContent = `获取日志失败: ${error.message}`;
                 
-                    // 更新内容容器的高度
+                // 更新内容容器的高度
-                    updateContentContainerHeight();
+                updateContentContainerHeight();
-                });
+            });
-        } else {
-            // 如果不是首次加载，直接更新内容容器的高度
-            updateContentContainerHeight();
-        }
     } else {
         logContainer.classList.add('hidden');
         

scripts/templates/index.html

@@ -133,8 +133,21 @@
                     <h3 class="card-title">其他设置</h3>
                     <div class="parameter-content">
                         <div class="form-group">
-                            <label for="device">训练设备：</label>
+                            <label for="training_mode">训练方式：</label>
-                            <input type="text" id="device" name="device" value="cuda:0" required>
+                            <select id="training_mode" name="training_mode" required>
+                                <option value="single_gpu">单卡训练</option>
+                                <option value="multi_gpu">多卡训练</option>
+                                <option value="cpu">CPU训练</option>
+                            </select>
+                        </div>
+                        <div class="form-group" id="single-gpu-selection">
+                            <label for="device">GPU序号：</label>
+                            <input type="number" id="device" name="device" min="0" max="{{ gpu_count|default(1) - 1 }}" value="0" required>
+                        </div>
+                        <div class="form-group" id="multi-gpu-selection" style="display: none;">
+                            <label for="gpu_num">多卡并行数：</label>
+                            <input type="number" id="gpu_num" name="gpu_num" min="1" max="{{ gpu_count|default(1) }}" value="{{ gpu_count|default(1) }}" required>
+                            <span class="hint-text">(可用GPU数量: {{ gpu_count|default(0) }})</span>
                         </div>
                         <div class="form-group">
                             <label for="train_monitor">训练监控：</label>
@@ -146,6 +159,61 @@
                     </div>
                 </div>
                 
+                <script>
+                    // 检查是否支持GPU并显示对应的选择器
+                    // 使用Flask模板变量传递GPU信息
+                    var hasGpu = {{ has_gpu|default(false)|tojson|safe }};
+                    var gpuCount = {{ gpu_count|default(0)|tojson|safe }};
+                    
+                    // 根据是否有GPU设置默认训练方式
+                    var trainingModeSelect = document.getElementById('training_mode');
+                    var singleGpuSelection = document.getElementById('single-gpu-selection');
+                    var multiGpuSelection = document.getElementById('multi-gpu-selection');
+                    
+                    // 设置默认值并初始化UI
+                    function initUI() {
+                        if (!trainingModeSelect) return;
+                        
+                        if (!hasGpu) {
+                            // 没有GPU时，默认选择CPU训练
+                            trainingModeSelect.value = 'cpu';
+                            if (singleGpuSelection) singleGpuSelection.style.display = 'none';
+                            if (multiGpuSelection) multiGpuSelection.style.display = 'none';
+                        } else {
+                            // 设置GPU数量的默认值
+                            var gpuNumInput = document.getElementById('gpu_num');
+                            if (gpuNumInput && gpuCount > 0) {
+                                gpuNumInput.value = gpuCount;
+                            }
+                        }
+                        
+                        // 初始化显示状态
+                        updateSelectionVisibility();
+                    }
+                    
+                    // 更新选择器可见性
+                    function updateSelectionVisibility() {
+                        if (!trainingModeSelect) return;
+                        
+                        var selectedMode = trainingModeSelect.value;
+                        
+                        if (singleGpuSelection) {
+                            singleGpuSelection.style.display = selectedMode === 'single_gpu' ? 'block' : 'none';
+                        }
+                        if (multiGpuSelection) {
+                            multiGpuSelection.style.display = selectedMode === 'multi_gpu' ? 'block' : 'none';
+                        }
+                    }
+                    
+                    // 添加训练方式切换事件监听器
+                    if (trainingModeSelect) {
+                        trainingModeSelect.addEventListener('change', updateSelectionVisibility);
+                    }
+                    
+                    // 初始化UI
+                    window.addEventListener('DOMContentLoaded', initUI);
+                </script>
+                
                 <div class="submit-container">
                     <button type="submit">开始训练</button>
                 </div>

scripts/train_web_ui.py

@@ -7,6 +7,38 @@ import socket
 from flask import Flask, render_template, request, jsonify, redirect, url_for
 import time
 
+# 尝试导入torch来检测GPU
+try:
+    import torch
+    HAS_TORCH = True
+    # 检测可用的GPU数量和设备信息
+    if torch.cuda.is_available():
+        GPU_COUNT = torch.cuda.device_count()
+        # 获取GPU设备名称
+        GPU_NAMES = [torch.cuda.get_device_name(i) for i in range(GPU_COUNT)]
+    else:
+        GPU_COUNT = 0
+        GPU_NAMES = []
+except ImportError:
+    HAS_TORCH = False
+    GPU_COUNT = 0
+    GPU_NAMES = []
+
+# 训练方式支持检测
+def get_supported_training_methods():
+    """获取当前环境支持的训练方法"""
+    methods = {
+        'pretrain': True,  # 预训练总是支持
+        'sft': True,       # SFT总是支持
+        'lora': True,      # LoRA总是支持
+        'dpo': True,       # DPO总是支持
+        'multi_gpu': HAS_TORCH and GPU_COUNT > 1  # 多GPU训练需要PyTorch和多个GPU
+    }
+    return methods
+
+# 获取当前环境支持的训练方法
+SUPPORTED_METHODS = get_supported_training_methods()
+
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
 
 app = Flask(__name__, template_folder='templates', static_folder='static')
@@ -28,25 +60,41 @@ def start_training_process(train_type, params):
     # 确保日志目录存在
     os.makedirs(log_dir, exist_ok=True)
     
+    # 获取GPU数量参数，如果存在且大于1，则使用torchrun启动多卡训练
+    gpu_num = int(params.get('gpu_num', 0)) if 'gpu_num' in params else 0
+    use_torchrun = HAS_TORCH and GPU_COUNT > 0 and gpu_num > 1
+    
     # 构建命令
     if train_type == 'pretrain':
         script_path = '../trainer/train_pretrain.py'
-        cmd = [sys.executable, script_path]
+        if use_torchrun:
+            cmd = ['torchrun', '--nproc_per_node', str(gpu_num), script_path]
+        else:
+            cmd = [sys.executable, script_path]
         if 'save_weight' in params:
             cmd.extend(['--save_weight', params['save_weight']])
     elif train_type == 'sft':
         script_path = '../trainer/train_full_sft.py'
-        cmd = [sys.executable, script_path]
+        if use_torchrun:
+            cmd = ['torchrun', '--nproc_per_node', str(gpu_num), script_path]
+        else:
+            cmd = [sys.executable, script_path]
         if 'save_weight' in params:
             cmd.extend(['--save_weight', params['save_weight']])
     elif train_type == 'lora':
         script_path = '../trainer/train_lora.py'
-        cmd = [sys.executable, script_path]
+        if use_torchrun:
+            cmd = ['torchrun', '--nproc_per_node', str(gpu_num), script_path]
+        else:
+            cmd = [sys.executable, script_path]
         if 'lora_name' in params:
             cmd.extend(['--lora_name', params['lora_name']])
     elif train_type == 'dpo':
         script_path = '../trainer/train_dpo.py'
-        cmd = [sys.executable, script_path]
+        if use_torchrun:
+            cmd = ['torchrun', '--nproc_per_node', str(gpu_num), script_path]
+        else:
+            cmd = [sys.executable, script_path]
         # 添加DPO特定参数
         if 'beta' in params and params['beta']:
             cmd.extend(['--beta', params['beta']])
@@ -59,14 +107,14 @@ def start_training_process(train_type, params):
     
     # 添加通用参数
     for key, value in params.items():
-        # 跳过特殊参数和DPO特有参数
+        # 跳过特殊参数和DPO特有参数，以及gpu_num参数（因为已经在torchrun命令中使用）
-        if key in ['train_type', 'save_weight', 'lora_name', 'train_monitor', 'beta', 'accumulation_steps', 'grad_clip']:
+        if key in ['train_type', 'save_weight', 'lora_name', 'train_monitor', 'beta', 'accumulation_steps', 'grad_clip', 'gpu_num']:
             continue
             
-        # 特殊处理布尔标志参数
+        # 对于from_resume参数，需要正确传递参数值
         if key == 'from_resume':
-            if value == '1':  # 只有当值为1时才添加这个标志
+            # 确保传递参数名和参数值
-                cmd.append(f'--{key}')
+            cmd.extend([f'--{key}', str(value)])
         else:
             # 确保log_interval和save_interval参数正确传递
             cmd.extend([f'--{key}', str(value)])
@@ -126,7 +174,8 @@ def start_training_process(train_type, params):
 # Flask路由
 @app.route('/')
 def index():
-    return render_template('index.html')
+    # 传递GPU信息到前端
+    return render_template('index.html', has_gpu=HAS_TORCH and GPU_COUNT > 0, gpu_count=GPU_COUNT)
 
 @app.route('/train', methods=['POST'])
 def train():
@@ -176,20 +225,62 @@ def logs(process_id):
     log_dir = os.path.join(script_dir, '../logfile')
     log_dir = os.path.abspath(log_dir)
     
+    # 查找匹配的日志文件
+    log_file = None
     if os.path.exists(log_dir):
         for filename in os.listdir(log_dir):
             if filename.endswith(f'{process_id}.log'):
                 log_file = os.path.join(log_dir, filename)
-                if os.path.exists(log_file):
+                break
-                    try:
+    
-                        with open(log_file, 'r', encoding='utf-8') as f:
+    if not log_file or not os.path.exists(log_file):
-                            # 只读取最后200行
+        return '日志文件不存在或已被删除'
-                            lines = f.readlines()
+    
-                            last_200_lines = lines[-200:] if len(lines) > 200 else lines
+    try:
-                            return ''.join(last_200_lines)
+        # 使用高效的方法读取文件的最后200行
-                    except Exception as e:
+        # 这对于大文件特别有用，可以避免读取整个文件
-                        return f'读取日志失败: {str(e)}'
+        last_200_lines = []
-    return '日志文件不存在或已被删除'
+        block_size = 8192  # 8KB blocks
+        
+        with open(log_file, 'r', encoding='utf-8') as f:
+            # 尝试直接定位到文件末尾，然后向前读取
+            f.seek(0, os.SEEK_END)
+            file_size = f.tell()
+            
+            # 计算需要读取的块数
+            position = file_size
+            blocks = []
+            while position > 0:
+                # 后退一个块的位置
+                position -= block_size
+                if position < 0:
+                    position = 0
+                
+                # 移动到计算的位置
+                f.seek(position)
+                
+                # 读取这个块
+                block = f.read(block_size)
+                blocks.append(block)
+                
+                # 如果已经收集了足够的行，就停止
+                combined_text = ''.join(reversed(blocks))
+                lines = combined_text.splitlines(True)
+                if len(lines) >= 200:
+                    # 获取最后200行
+                    last_200_lines = lines[-200:]
+                    break
+            
+            # 如果文件内容不足200行，或者上面的方法没有收集到足够的行
+            if len(last_200_lines) < 200:
+                # 重新读取整个文件（对于小文件）
+                f.seek(0)
+                all_lines = f.readlines()
+                last_200_lines = all_lines[-200:] if len(all_lines) > 200 else all_lines
+        
+        return ''.join(last_200_lines)
+    except Exception as e:
+        return f'读取日志失败: {str(e)}'
 
 @app.route('/logfiles')
 def get_logfiles():
@@ -220,24 +311,26 @@ def get_logfiles():
 
 @app.route('/logfile-content/<filename>')
 def get_logfile_content(filename):
+    # 安全检查：确保文件名不包含路径遍历字符
+    if '..' in filename or '/' in filename or '\\' in filename:
+        return jsonify({'error': 'Invalid filename'}), 400
+    
     # 获取脚本所在目录的绝对路径
     script_dir = os.path.dirname(os.path.abspath(__file__))
-    # 构建logfile目录的绝对路径
+    # 构建logfile目录的绝对路径，train_web_ui.py在scripts目录下
     log_dir = os.path.join(script_dir, '../logfile')
     log_dir = os.path.abspath(log_dir)
-    
-    # 安全检查：防止路径遍历攻击
-    if '..' in filename or '/' in filename or '\\' in filename:
-        return '非法的文件名'
-    
     log_file = os.path.join(log_dir, filename)
-    if os.path.exists(log_file) and os.path.isfile(log_file):
+    
-        try:
+    try:
-            with open(log_file, 'r', encoding='utf-8') as f:
+        # 读取完整的日志文件内容
-                return f.read()
+        with open(log_file, 'r', encoding='utf-8') as f:
-        except Exception as e:
+            content = f.read()
-            return f'读取日志失败: {str(e)}'
+        return content
-    return '日志文件不存在'
+    except FileNotFoundError:
+        return jsonify({'error': 'Log file not found'}), 404
+    except Exception as e:
+        return jsonify({'error': str(e)}), 500
 
 @app.route('/delete-logfile/<filename>', methods=['DELETE'])
 def delete_logfile(filename):