[fix] reduce aux_loss_alpha

2025-12-05 23:07:55 +08:00 · 2025-12-05 23:07:55 +08:00 · b7cb98c136
parent 13c3a2c093
commit b7cb98c136
1 changed files with 1 additions and 1 deletions
--- a/model/model_minimind.py
+++ b/model/model_minimind.py
@ -34,7 +34,7 @@ class MiniMindConfig(PretrainedConfig):
            n_routed_experts: int = 4,
            n_shared_experts: int = 1,
            scoring_func: str = 'softmax',
-            aux_loss_alpha: float = 0.1,
+            aux_loss_alpha: float = 0.01,
            seq_aux: bool = True,
            norm_topk_prob: bool = True,
            **kwargs