From fc398c440e022031b551bc7bd850bec42d8d9de6 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 1 May 2025 06:55:43 -0700 Subject: [PATCH] Update synthetic.py --- unsloth/dataprep/synthetic.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/unsloth/dataprep/synthetic.py b/unsloth/dataprep/synthetic.py index 4e0b2ba3a3..ec18104f75 100644 --- a/unsloth/dataprep/synthetic.py +++ b/unsloth/dataprep/synthetic.py @@ -206,7 +206,9 @@ class SyntheticDataKit: with open(filename, "r") as f: text = f.read() - max_tokens = self.max_seq_length - self.max_generation_tokens*2 - 2 + max_tokens = self.max_seq_length - self.max_generation_tokens*3 # * 3 to reduce errors + if max_tokens <= 5: + raise RuntimeError("Generation length is way too long!") input_ids = self.tokenizer(text, add_special_tokens = False).input_ids # Get left and right boundaries