ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update synthetic.py

Browse source
This commit is contained in:
Daniel Han 2025-05-01 06:55:43 -07:00
commit fc398c440e
1 changed files with 3 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

4
unsloth/dataprep/synthetic.py
View file

@ -206,7 +206,9 @@ class SyntheticDataKit:
with open(filename, "r") as f: text = f.read()
max_tokens = self.max_seq_length - self.max_generation_tokens*2 - 2
max_tokens = self.max_seq_length - self.max_generation_tokens*3 # * 3 to reduce errors
if max_tokens <= 5:
raise RuntimeError("Generation length is way too long!")
input_ids = self.tokenizer(text, add_special_tokens = False).input_ids
# Get left and right boundaries