From 98e89b7a3dd0277ff0e732736d305ac486d246f6 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 28 Dec 2024 03:24:18 -0800 Subject: [PATCH] Update _utils.py --- unsloth/models/_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 1f2f9018d6..86346d7e2e 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -1023,8 +1023,7 @@ def _unsloth_get_batch_samples(self, epoch_iterator, num_batches): num_items_in_batch = sum( [(x["labels"][..., 1:] != -100).sum() for x in batch_samples] ) - # num_items_in_batch = sum([(batch["labels"].ne(-100)).sum() for batch in batch_samples]) - + if self.args.average_tokens_across_devices: num_items_in_batch = self.accelerator.gather(num_items_in_batch).sum().item()