github
/
minimind
mirror of https://github.com/jingyaogong/minimind.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix weight initialization for residual block

pull/136/head
CohleM 2025-02-03 15:50:50 +05:45
parent 6e9cd28ef9
commit 5f84ee6370
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
model/model.py
View File

@ -333,7 +333,7 @@ class Transformer(PreTrainedModel):
self.apply(self._init_weights) self.apply(self._init_weights)
for pn, p in self.named_parameters(): for pn, p in self.named_parameters():
if pn.endswith('w3.weight') or pn.endswith('wo.weight'): if pn.endswith('w2.weight') or pn.endswith('wo.weight'):
torch.nn.init.normal_(p, mean=0.0, std=0.02 / math.sqrt(2 * params.n_layers)) torch.nn.init.normal_(p, mean=0.0, std=0.02 / math.sqrt(2 * params.n_layers))
self.last_loss = None self.last_loss = None