Update synthetic.py

unsloth/dataprep/synthetic.py

@@ -205,15 +205,13 @@ class SyntheticDataKit:
         original_n_tokens = len(input_ids)
 
         if len(input_ids) > max_tokens:
-            # Will fix later, but for now we simply naively truncate by 10% increments
-            ratio = 0.9
+            # Will fix later, but for now we simply naively truncate by 100 in length
             length = original_length
             while True:
                 input_ids = self.tokenizer(text[:length]).input_ids
                 if len(input_ids) < max_tokens or length == 0: break
-                length = int(original_length * ratio)
+                length -= 100
                 length = max(length, 0)
-                ratio -= 0.1
             pass
             print(f"Unsloth: Will truncate your data which has {original_n_tokens} tokens to {len(input_ids)} tokens.")