From 17d8cb2dfe030396e722508470a281e8bc8dd025 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 30 Apr 2025 10:22:01 -0700 Subject: [PATCH] Update synthetic.py --- unsloth/dataprep/synthetic.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/unsloth/dataprep/synthetic.py b/unsloth/dataprep/synthetic.py index 5fc6e6c577..a49a312a9d 100644 --- a/unsloth/dataprep/synthetic.py +++ b/unsloth/dataprep/synthetic.py @@ -205,15 +205,13 @@ class SyntheticDataKit: original_n_tokens = len(input_ids) if len(input_ids) > max_tokens: - # Will fix later, but for now we simply naively truncate by 10% increments - ratio = 0.9 + # Will fix later, but for now we simply naively truncate by 100 in length length = original_length while True: input_ids = self.tokenizer(text[:length]).input_ids if len(input_ids) < max_tokens or length == 0: break - length = int(original_length * ratio) + length -= 100 length = max(length, 0) - ratio -= 0.1 pass print(f"Unsloth: Will truncate your data which has {original_n_tokens} tokens to {len(input_ids)} tokens.")