Update train_grpo.py
Litar
GitHub
parent
16434d79c2
commit
3052baff06
|
|
@ -26,112 +26,175 @@ warnings.filterwarnings('ignore')
|
|||
|
||||
def calculate_rewards(prompts, responses, reward_model, reward_tokenizer):
|
||||
"""整合所有奖励函数计算总奖励"""
|
||||
|
||||
def reasoning_model_reward(rewards):
|
||||
# --- 1. 格式奖励：鼓励模型学会“思考”格式 ---
|
||||
# 使用正则表达式检查回答是否遵循 <think>...</think><answer>...</answer> 的标准格式。
|
||||
# 这种格式化的输出有助于模型学习链式思考（Chain-of-Thought）。
|
||||
pattern = r"^<think>\n.*?\n</think>\n<answer>\n.*?\n</answer>$"
|
||||
pattern2 = r"^<think>\n.*?\n</think>\n\n<answer>\n.*?\n</answer>$"
|
||||
|
||||
matches_pattern = [re.match(pattern, response, re.S) for response in responses]
|
||||
matches_pattern2 = [re.match(pattern2, response, re.S) for response in responses]
|
||||
|
||||
format_rewards = []
|
||||
for match_pattern, match_pattern2 in zip(matches_pattern, matches_pattern2):
|
||||
if match_pattern or match_pattern2:
|
||||
if match_pattern:
|
||||
format_rewards.append(0.5) # 格式正确则给予奖励
|
||||
elif match_pattern2:
|
||||
format_rewards.append(0.5)
|
||||
else:
|
||||
format_rewards.append(0.0)
|
||||
rewards += torch.tensor(format_rewards, device=args.device)
|
||||
|
||||
# --- 2. 标记奖励：提供更密集的引导信号 ---
|
||||
# 检查<think>, </think>, <answer>, </answer>四个标签是否都只出现一次。
|
||||
# 这是一个“稀疏奖励”的补充，即使格式不完全对，只要包含了正确的标签，也能得到一些分数，
|
||||
# 避免了模型在初期因无法生成完整正确格式而得不到任何正反馈的问题。
|
||||
def mark_num(text):
|
||||
reward = 0
|
||||
if text.count("<think>") == 1: reward += 0.25
|
||||
if text.count("</think>") == 1: reward += 0.25
|
||||
if text.count("<answer>") == 1: reward += 0.25
|
||||
if text.count("</answer>") == 1: reward += 0.25
|
||||
if text.count("<think>") == 1:
|
||||
reward += 0.25
|
||||
if text.count("</think>") == 1:
|
||||
reward += 0.25
|
||||
if text.count("<answer>") == 1:
|
||||
reward += 0.25
|
||||
if text.count("</answer>") == 1:
|
||||
reward += 0.25
|
||||
return reward
|
||||
|
||||
mark_rewards = [mark_num(response) for response in responses]
|
||||
rewards += torch.tensor(mark_rewards, device=args.device)
|
||||
return rewards
|
||||
|
||||
# 初始化一个长度为 responses 的张量，用于保存奖励
|
||||
rewards = torch.zeros(len(responses), device=args.device)
|
||||
if args.reasoning == 1:
|
||||
rewards = reasoning_model_reward(rewards)
|
||||
|
||||
# --- 3. 如果是推理模式，则添加上述的格式和标记奖励 ---
|
||||
if args.reasoning == 1:
|
||||
rewards = reasoning_model_reward(rewards) # 训练推理模型时使用
|
||||
|
||||
# --- 4. 核心奖励：使用外部奖励模型打分 ---
|
||||
# with torch.no_grad() 确保这部分计算不产生梯度
|
||||
with torch.no_grad():
|
||||
reward_model_scores = []
|
||||
batch_size = len(prompts)
|
||||
scale = 3.0
|
||||
|
||||
# 遍历每一个Prompt
|
||||
for i in range(batch_size):
|
||||
# 每一个Prompt对应num_generations个生成结果
|
||||
for j in range(args.num_generations):
|
||||
# 取该个Prompt的j个生成结果
|
||||
response_idx = i * args.num_generations + j
|
||||
response = responses[response_idx]
|
||||
prompt = prompts[i]
|
||||
|
||||
# --- 准备奖励模型的输入 ---
|
||||
# 解析prompt，将其转换为奖励模型能理解的对话格式列表。
|
||||
pattern = r"<\|im_start\|>(system|user|assistant)\s+(.*?)<\|im_end\|>"
|
||||
matches = re.findall(pattern, prompt, re.DOTALL)
|
||||
messages = [{"role": role, "content": content.strip()} for role, content in matches]
|
||||
|
||||
# 将当前response作为assistant的最新回复，添加到对话历史中。
|
||||
tmp_chat = messages + [{"role": "assistant", "content": response}]
|
||||
# 调用奖励模型，获取对整个对话（包括当前response）的评分。
|
||||
score = reward_model.get_score(reward_tokenizer, tmp_chat)
|
||||
# 将分数裁剪到[-3.0, 3.0]区间，防止极端值影响训练稳定性。
|
||||
score = max(min(score, scale), -scale)
|
||||
|
||||
# --- 对推理任务的特殊处理 ---
|
||||
# 如果是推理模式...
|
||||
if args.reasoning == 1:
|
||||
#...尝试从response中提取<answer>标签内的最终答案。
|
||||
answer_match = re.search(r'<answer>(.*?)</answer>', response, re.DOTALL)
|
||||
# 如果找到了最终答案...
|
||||
if answer_match:
|
||||
#...提取答案内容。
|
||||
answer_content = answer_match.group(1).strip()
|
||||
tmp_chat = messages + [{"role": "assistant", "content": answer_content}]
|
||||
answer_score = reward_model.get_score(reward_tokenizer, tmp_chat)
|
||||
#...构造一个新的对话历史，其中assistant的回复仅包含这个最终答案。
|
||||
tmp_chat_answer = messages + [{"role": "assistant", "content": answer_content}]
|
||||
#...单独对这个最终答案进行评分。
|
||||
answer_score = reward_model.get_score(reward_tokenizer, tmp_chat_answer)
|
||||
answer_score = max(min(answer_score, scale), -scale)
|
||||
# 最终分数是“完整回答”和“核心答案”的加权平均，核心答案权重更高(0.6)。
|
||||
score = score * 0.4 + answer_score * 0.6
|
||||
|
||||
# 将计算出的最终分数添加到列表中。
|
||||
reward_model_scores.append(score)
|
||||
|
||||
# 4. 将所有分数转换为PyTorch张量，并累加到总奖励上。
|
||||
reward_model_scores = torch.tensor(reward_model_scores, device=args.device)
|
||||
rewards += reward_model_scores
|
||||
|
||||
# 5. 返回包含了所有维度奖励的最终总奖励张量。
|
||||
return rewards
|
||||
|
||||
|
||||
def grpo_train_epoch(epoch, loader, iters, ref_model, reward_model, reward_tokenizer, start_step=0, wandb=None):
|
||||
for step, batch in enumerate(loader, start=start_step + 1):
|
||||
prompts = batch['prompt'] # list[str], length B
|
||||
def grpo_train_epoch(epoch, wandb):
|
||||
for step, batch in enumerate(train_loader):
|
||||
# --- 准备输入 ---
|
||||
prompts = batch['prompt'] # list[str], 一维List 长度 = Batch_size
|
||||
prompt_inputs = tokenizer(prompts, return_tensors="pt", padding=True, return_token_type_ids=False,
|
||||
padding_side="left", add_special_tokens=False).to(args.device) # input_ids: [B, P], attention_mask: [B, P]
|
||||
# 如果超出长度则截断
|
||||
if args.max_seq_len:
|
||||
prompt_inputs["input_ids"] = prompt_inputs["input_ids"][:, -args.max_seq_len:]
|
||||
prompt_inputs["attention_mask"] = prompt_inputs["attention_mask"][:, -args.max_seq_len:]
|
||||
|
||||
# --- 步骤1: 生成一组响应 (Rollout) ---
|
||||
# 对于每个prompt，让当前策略模型生成 num_generations 个不同的回答。
|
||||
# do_sample=True 和 temperature=0.8 保证了生成的多样性。
|
||||
with torch.no_grad():
|
||||
# DDP 模型需要使用 .module 访问 generate 方法
|
||||
model_for_gen = model.module if isinstance(model, DistributedDataParallel) else model
|
||||
outputs = model_for_gen.generate(
|
||||
outputs = (model.module if ddp else model).generate(
|
||||
**prompt_inputs, max_new_tokens=args.max_gen_len, do_sample=True, temperature=0.8,
|
||||
num_return_sequences=args.num_generations, pad_token_id=tokenizer.pad_token_id) # [B*num_gen, P+R]
|
||||
|
||||
# 从生成结果中分离出回答部分 (completion) 的 token IDs
|
||||
completion_ids = outputs[:, prompt_inputs["input_ids"].size(1):] # [B*num_gen, R]
|
||||
|
||||
# --- 步骤2: 计算对数概率 ---
|
||||
# 定义一个辅助函数来计算给定序列中每个token的对数概率
|
||||
def get_per_token_logps(mdl, input_ids, n_keep):
|
||||
# input_ids [B*num_gen, P+R]
|
||||
# 判断是否为推理模式，如果是，则 detaching input_ids，以防止反向传播
|
||||
input_ids = input_ids.detach().clone() if input_ids.is_inference() else input_ids
|
||||
# 计算模型的 logits，logits 的形状是 [B, P+R, vocab_size]
|
||||
logits = mdl(input_ids, logits_to_keep=n_keep + 1).logits[:, :-1, :]
|
||||
per_token_logps = []
|
||||
for logits_row, ids_row in zip(logits, input_ids[:, -n_keep:]):
|
||||
ids_row = ids_row.detach().clone() if ids_row.is_inference() else ids_row
|
||||
per_token_logps.append(torch.gather(logits_row.log_softmax(dim=-1), 1, ids_row.unsqueeze(1)).squeeze(1))
|
||||
return torch.stack(per_token_logps)
|
||||
|
||||
per_token_logps = []
|
||||
# 遍历每一行 logits 和输入的 id，计算每个 token 的 log 概率
|
||||
for logits_row, ids_row in zip(logits, input_ids[:, -n_keep:]): # [B*num_gen , R]
|
||||
# 如果是推理阶段，detach input_ids 防止反向传播
|
||||
ids_row = ids_row.detach().clone() if ids_row.is_inference() else ids_row
|
||||
# 使用 torch.gather 从 logits 中选出 ids_row 对应的列，然后对 logits 做 softmax 转换得到 log 概率
|
||||
per_token_logps.append(torch.gather(logits_row.log_softmax(dim=-1), 1, ids_row.unsqueeze(1)).squeeze(1))
|
||||
return torch.stack(per_token_logps) # [B*num_gen, R]
|
||||
|
||||
# 计算“策略模型”对于这批生成的responses的log_probs。
|
||||
per_token_logps = get_per_token_logps(model, outputs, completion_ids.size(1)) # [B*num_gen, R]
|
||||
# 在不计算梯度的模式下，计算“参考模型”的log_probs。
|
||||
with torch.no_grad():
|
||||
ref_per_token_logps = get_per_token_logps(ref_model, outputs, completion_ids.size(1)) # [B*num_gen, R]
|
||||
|
||||
completions = tokenizer.batch_decode(completion_ids, skip_special_tokens=True)
|
||||
# --- 步骤3: 计算奖励和优势 (Reward & Advantage) ---
|
||||
# 将response的token IDs解码回文本。
|
||||
completions = tokenizer.batch_decode(completion_ids, skip_special_tokens=True) # [B*num_gen, R]
|
||||
rewards = calculate_rewards(prompts, completions, reward_model, reward_tokenizer).to(args.device) # [B*num_gen]
|
||||
|
||||
#!!! GRPO算法的核心!!!
|
||||
# 将扁平的rewards张量重塑为(batch_size, num_generations)的形状。
|
||||
grouped_rewards = rewards.view(-1, args.num_generations) # [B, num_gen]
|
||||
# 沿着num_generations维度计算每组的平均奖励。
|
||||
mean_r = grouped_rewards.mean(dim=1).repeat_interleave(args.num_generations) # [B*num_gen]
|
||||
std_r = grouped_rewards.std(dim=1).repeat_interleave(args.num_generations) # [B*num_gen]
|
||||
# 计算优势值：(当前奖励 - 组内平均奖励) / 组内标准差。
|
||||
# 这衡量了每个response比其“兄弟”们好多少（或差多少）。
|
||||
advantages = torch.clamp((rewards - mean_r) / (std_r + 1e-4), -10, 10)
|
||||
# 对所有优势值进行全局归一化，使训练更加稳定。
|
||||
advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8) # [B*num_gen]
|
||||
|
||||
# --- 步骤4: 计算损失 (Loss) ---
|
||||
# 创建一个mask，确保只计算到句子结束符(EOS)为止的token的损失。
|
||||
is_eos = completion_ids == tokenizer.eos_token_id # [B*num_gen, R]
|
||||
eos_idx = torch.full((is_eos.size(0),), is_eos.size(1), dtype=torch.long, device=args.device)
|
||||
eos_idx[is_eos.any(dim=1)] = is_eos.int().argmax(dim=1)[is_eos.any(dim=1)]
|
||||
|
|
|
|||
Loading…
Reference in New Issue