From 66ad1c4b7fef65541a33e4636b2c96271866edbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E8=B7=AF=E5=AE=9D?= Date: Wed, 1 Apr 2026 14:20:30 +0800 Subject: [PATCH] fix: add find_unused_parameters=True to DDP for mixed text-image batches When a batch contains only text samples (pixel_values is None), vision_proj parameters do not participate in the forward pass, causing DDP to raise: 'Expected to have finished reduction in the prior iteration' Adding find_unused_parameters=True resolves this. --- trainer/train_sft_vlm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trainer/train_sft_vlm.py b/trainer/train_sft_vlm.py index 8a42bd7..bb609e9 100644 --- a/trainer/train_sft_vlm.py +++ b/trainer/train_sft_vlm.py @@ -150,7 +150,7 @@ if __name__ == "__main__": Logger('torch.compile enabled') if dist.is_initialized(): model._ddp_params_and_buffers_to_ignore = {"freqs_cos", "freqs_sin"} - model = DistributedDataParallel(model, device_ids=[local_rank]) + model = DistributedDataParallel(model, device_ids=[local_rank], find_unused_parameters=True) # ========== 8. 开始训练 ========== for epoch in range(start_epoch, args.epochs):