ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update synthetic.py

Browse source
This commit is contained in:
Daniel Han 2025-05-01 06:42:49 -07:00
commit b606fb438f
1 changed files with 3 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

6
unsloth/dataprep/synthetic.py
View file

@ -211,9 +211,9 @@ class SyntheticDataKit:
# Get left and right boundaries
length = len(input_ids)
n_chunks = int(np.ceil(length / (max_tokens - overlap)))
boundaries = np.ceil(np.linspace(0, length - overlap, n_chunks)).astype(int)
boundaries = np.stack((boundaries[:-1], (boundaries + overlap)[1:])).T
n_chunks = int(np.ceil(length / (max_tokens - self.overlap)))
boundaries = np.ceil(np.linspace(0, length - self.overlap, n_chunks)).astype(int)
boundaries = np.stack((boundaries[:-1], (boundaries + self.overlap)[1:])).T
boundaries = np.minimum(boundaries, length).tolist()
# Get extension of filename like .txt