diff --git a/unsloth/dataprep/synthetic.py b/unsloth/dataprep/synthetic.py index 5fc6e6c577..a49a312a9d 100644 --- a/unsloth/dataprep/synthetic.py +++ b/unsloth/dataprep/synthetic.py @@ -205,15 +205,13 @@ class SyntheticDataKit: original_n_tokens = len(input_ids) if len(input_ids) > max_tokens: - # Will fix later, but for now we simply naively truncate by 10% increments - ratio = 0.9 + # Will fix later, but for now we simply naively truncate by 100 in length length = original_length while True: input_ids = self.tokenizer(text[:length]).input_ids if len(input_ids) < max_tokens or length == 0: break - length = int(original_length * ratio) + length -= 100 length = max(length, 0) - ratio -= 0.1 pass print(f"Unsloth: Will truncate your data which has {original_n_tokens} tokens to {len(input_ids)} tokens.")