Merge 6b2f4c939a into cdf0a93e02

trainer/train_agent.py

@@ -473,8 +473,7 @@ if __name__ == "__main__":
         Logger('torch.compile enabled')
         rollout_engine.update_policy(model)
     if dist.is_initialized():
-        # 同 train_ppo：RoPE buffer 各 rank 一致，每步广播纯属浪费
+        model = DistributedDataParallel(model, device_ids=[local_rank])
-        model = DistributedDataParallel(model, device_ids=[local_rank], broadcast_buffers=False)
     rollout_engine.update_policy(model)
 
     for epoch in range(start_epoch, args.epochs):

trainer/train_grpo.py

@@ -314,8 +314,7 @@ if __name__ == "__main__":
         Logger('torch.compile enabled')
         rollout_engine.update_policy(model)
     if dist.is_initialized():
-        # 同 train_ppo：RoPE buffer 各 rank 一致，每步广播纯属浪费
+        model = DistributedDataParallel(model, device_ids=[local_rank])
-        model = DistributedDataParallel(model, device_ids=[local_rank], broadcast_buffers=False)
     rollout_engine.update_policy(model)
     
     # ========== 8. 开始训练 ==========

trainer/train_ppo.py

@@ -36,7 +36,7 @@ def rep_penalty(text, n=3, cap=0.5):
 class CriticModel(MiniMindForCausalLM):
     def __init__(self, params):
         super().__init__(params)
-        # lm_head 不参与 forward，仅靠 tie_word_embeddings 与 embed_tokens 共享权重，解绑后 DDP 会报未使用参数
+        # 替换lm_head为输出单一价值的线性层
         self.value_head = nn.Linear(params.hidden_size, 1)
 
     def forward(self, input_ids=None, attention_mask=None, **kwargs):
@@ -429,9 +429,8 @@ if __name__ == "__main__":
         Logger('torch.compile enabled')
         rollout_engine.update_policy(actor_model)
     if dist.is_initialized():
-        # freqs_cos/freqs_sin 各 rank 由 config 确定性算出，默认每步广播一次纯属浪费
+        actor_model = DistributedDataParallel(actor_model, device_ids=[local_rank])
-        actor_model = DistributedDataParallel(actor_model, device_ids=[local_rank], broadcast_buffers=False)
+        critic_model = DistributedDataParallel(critic_model, device_ids=[local_rank])
-        critic_model = DistributedDataParallel(critic_model, device_ids=[local_rank], broadcast_buffers=False)
     rollout_engine.update_policy(actor_model)
     
     # ========== 8. 开始训练 ==========

trainer/trainer_utils.py

@@ -179,31 +179,50 @@ class LMForRewardModel:
         return max(min(score, 3.0), -3.0)
 
 
-# ===== 数学表达式安全求值：替代 eval，只放行算术运算与 math 白名单（长度上限 512） =====
+# ===== 数学表达式安全求值（替代 eval，避免执行模型生成的任意代码） =====
+_MATH_FUNCS = {name: getattr(math, name) for name in (
+    'sqrt', 'pow', 'exp', 'log', 'log2', 'log10', 'sin', 'cos', 'tan', 'asin', 'acos', 'atan', 'atan2',
+    'sinh', 'cosh', 'tanh', 'floor', 'ceil', 'trunc', 'fabs', 'fmod', 'hypot', 'gcd', 'degrees', 'radians'
+)}
+_MATH_CONSTS = {'pi': math.pi, 'e': math.e, 'tau': math.tau}
+_BIN_OPS = {ast.Add: operator.add, ast.Sub: operator.sub, ast.Mult: operator.mul, ast.Div: operator.truediv,
+            ast.FloorDiv: operator.floordiv, ast.Mod: operator.mod, ast.Pow: operator.pow}
+_UNARY_OPS = {ast.UAdd: operator.pos, ast.USub: operator.neg}
+_MAX_EXPR_LEN, _MAX_EXP, _MAX_DIGITS = 512, 1e4, 100  # 表达式长度 / 指数 / 结果位数上限
+
+
+def _check_pow(base, exp):  # 防止 9**9**9 这类表达式把进程算死
+    if abs(exp) > _MAX_EXP or (abs(base) > 1 and abs(exp) * math.log10(abs(base)) > _MAX_DIGITS):
+        raise ValueError('幂运算结果过大')
+
+
+def _resolve_name(node):  # 同时兼容 sqrt(4) 与 math.sqrt(4) 两种写法
+    if isinstance(node, ast.Name): return node.id
+    if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name) and node.value.id == 'math': return node.attr
+    return None
+
+
+def _eval_ast(node):
+    if isinstance(node, ast.Constant) and isinstance(node.value, (int, float)) and not isinstance(node.value, bool):
+        return node.value
+    if isinstance(node, (ast.Name, ast.Attribute)) and _resolve_name(node) in _MATH_CONSTS:
+        return _MATH_CONSTS[_resolve_name(node)]
+    if isinstance(node, ast.UnaryOp) and type(node.op) in _UNARY_OPS:
+        return _UNARY_OPS[type(node.op)](_eval_ast(node.operand))
+    if isinstance(node, ast.BinOp) and type(node.op) in _BIN_OPS:
+        left, right = _eval_ast(node.left), _eval_ast(node.right)
+        if isinstance(node.op, ast.Pow): _check_pow(left, right)
+        return _BIN_OPS[type(node.op)](left, right)
+    if isinstance(node, ast.Call) and _resolve_name(node.func) in _MATH_FUNCS and not node.keywords:
+        name, args = _resolve_name(node.func), [_eval_ast(arg) for arg in node.args]
+        if name == 'pow' and len(args) == 2: _check_pow(*args)
+        return _MATH_FUNCS[name](*args)
+    raise ValueError(f'不支持的表达式语法: {type(node).__name__}')
+
+
 def safe_math_eval(expression):
-    """对模型生成的数学表达式求值：支持 + - * / // % **、math 白名单函数与 pi/e/tau 常量。"""
+    """对模型生成的数学表达式求值：只放行算术运算与 math 白名单函数，不执行任意代码。"""
-    def pow_guard(base, exp):  # 幂运算统一走这里，拦截 9**9**9 / 10**99999 这类把进程算死的输入
+    expr = str(expression).replace('^', '**').replace('×', '*').replace('÷', '/').replace('−', '-')
-        if abs(exp) > 1e4 or (abs(base) > 1 and abs(exp) * math.log10(abs(base)) > 100): raise ValueError('幂运算结果过大')
+    expr = expr.replace('²', '**2').replace('³', '**3').replace('（', '(').replace('）', ')').strip()
-        return base ** exp
+    if not expr or len(expr) > _MAX_EXPR_LEN: raise ValueError('表达式为空或过长')
-    def resolve(node):  # 同时兼容 sqrt(4) 与 math.sqrt(4) 两种写法
+    return _eval_ast(ast.parse(expr, mode='eval').body)
-        if isinstance(node, ast.Name): return node.id
-        if isinstance(node, ast.Attribute) and getattr(node.value, 'id', '') == 'math': return node.attr
-    def walk(node):
-        if isinstance(node, ast.Constant) and type(node.value) in (int, float): return node.value
-        name = resolve(node)
-        if name in consts: return consts[name]
-        if isinstance(node, ast.UnaryOp): return unary_ops[type(node.op)](walk(node.operand))
-        if isinstance(node, ast.BinOp): return bin_ops[type(node.op)](walk(node.left), walk(node.right))
-        if isinstance(node, ast.Call) and not node.keywords: return funcs[resolve(node.func)](*map(walk, node.args))
-        raise ValueError(f'不支持的表达式语法: {type(node).__name__}')
-    consts = {'pi': math.pi, 'e': math.e, 'tau': math.tau}
-    funcs = {'pow': pow_guard, **{n: getattr(math, n) for n in 'sqrt exp log log2 log10 sin cos tan asin acos atan atan2 sinh cosh tanh floor ceil trunc fabs fmod hypot gcd degrees radians'.split()}}
-    bin_ops = {ast.Add: operator.add, ast.Sub: operator.sub, ast.Mult: operator.mul, ast.Div: operator.truediv, ast.FloorDiv: operator.floordiv, ast.Mod: operator.mod, ast.Pow: pow_guard}
-    unary_ops = {ast.UAdd: operator.pos, ast.USub: operator.neg}
-    chars = str.maketrans({'^': '**', '×': '*', '÷': '/', '−': '-', '²': '**2', '³': '**3', '（': '(', '）': ')'})
-    expr = str(expression).translate(chars).strip()
-    if not expr or len(expr) > 512: raise ValueError('表达式为空或过长')
-    try:
-        return walk(ast.parse(expr, mode='eval').body)
-    except (KeyError, SyntaxError, TypeError):
-        raise ValueError('不支持的表达式语法') from None