github
/
minimind
mirror of https://github.com/jingyaogong/minimind.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update train_gated_grpo.py

pull/664/head
vanking 2026-02-03 10:33:27 +08:00 committed by GitHub
parent e84437a8ca
commit 0b37f04f15
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
train_gated_grpo.py
View File

@ -141,7 +141,7 @@ def grpo_train_epoch(epoch, loader, iters, ref_model, reward_model, reward_token
per_token_kl = torch.exp(kl_div) - kl_div - 1 # [B*num_gen, R] per_token_kl = torch.exp(kl_div) - kl_div - 1 # [B*num_gen, R]
# 修改部分：添加门控 # 修改部分：添加门控
per_token_loss = -((torch.exp(per_token_logps - per_token_logps.detach()) * torch.sigmoid(0.5 * torch.exp(per_token_logps - per_token_logps.detach()))) * advantages.unsqueeze(1) - args.beta * per_token_kl) # [B*num_gen, R] per_token_loss = -((torch.exp(per_token_logps - per_token_logps.detach()) * torch.sigmoid(0.1 * torch.exp(per_token_logps - per_token_logps.detach()))) * advantages.unsqueeze(1) - args.beta * per_token_kl) # [B*num_gen, R]
loss = ((per_token_loss * completion_mask).sum(dim=1) / completion_mask.sum(dim=1)).mean() / args.accumulation_steps # scalar loss = ((per_token_loss * completion_mask).sum(dim=1) / completion_mask.sum(dim=1)).mean() / args.accumulation_steps # scalar