github
/
minimind
mirror of https://github.com/jingyaogong/minimind.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: switch DeepSpeed config from bf16 to fp16, ZeRO-2 to ZeRO-1 (RTX 3090 compat)

pull/791/head
baihaochen 2026-05-23 16:45:56 +08:00
parent 46dabcfc19
commit 2201060169
1 changed files with 8 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
trainer/ds_config_pretrain.json
View File

@ -4,7 +4,7 @@
"gradient_accumulation_steps": "auto", "gradient_accumulation_steps": "auto",
"gradient_clipping": 1.0, "gradient_clipping": 1.0,
"zero_optimization": { "zero_optimization": {
"stage": 2, "stage": 1,
"allgather_partitions": true, "allgather_partitions": true,
"allgather_bucket_size": 5e6, "allgather_bucket_size": 5e6,
"reduce_scatter": true, "reduce_scatter": true,
@ -12,8 +12,13 @@
"overlap_comm": true, "overlap_comm": true,
"contiguous_gradients": true "contiguous_gradients": true
}, },
"bf16": { "fp16": {
"enabled": true "enabled": true,
"loss_scale": 0,
"loss_scale_window": 1000,
"initial_scale_power": 16,
"hysteresis": 2,
"min_loss_scale": 1
}, },
"optimizer": { "optimizer": {
"type": "AdamW", "type": "AdamW",