fix: add find_unused_parameters=True to DDP for mixed text-image batches
When a batch contains only text samples (pixel_values is None), vision_proj parameters do not participate in the forward pass, causing DDP to raise: 'Expected to have finished reduction in the prior iteration' Adding find_unused_parameters=True resolves this.pull/112/head
parent
14cfa88629
commit
66ad1c4b7f
|
|
@ -150,7 +150,7 @@ if __name__ == "__main__":
|
|||
Logger('torch.compile enabled')
|
||||
if dist.is_initialized():
|
||||
model._ddp_params_and_buffers_to_ignore = {"freqs_cos", "freqs_sin"}
|
||||
model = DistributedDataParallel(model, device_ids=[local_rank])
|
||||
model = DistributedDataParallel(model, device_ids=[local_rank], find_unused_parameters=True)
|
||||
|
||||
# ========== 8. 开始训练 ==========
|
||||
for epoch in range(start_epoch, args.epochs):
|
||||
|
|
|
|||
Loading…
Reference in New Issue