From b606fb438f02e30cbbe6a121ca175b9820155c99 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 1 May 2025 06:42:49 -0700 Subject: [PATCH] Update synthetic.py --- unsloth/dataprep/synthetic.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/unsloth/dataprep/synthetic.py b/unsloth/dataprep/synthetic.py index 0916b61397..360f701713 100644 --- a/unsloth/dataprep/synthetic.py +++ b/unsloth/dataprep/synthetic.py @@ -211,9 +211,9 @@ class SyntheticDataKit: # Get left and right boundaries length = len(input_ids) - n_chunks = int(np.ceil(length / (max_tokens - overlap))) - boundaries = np.ceil(np.linspace(0, length - overlap, n_chunks)).astype(int) - boundaries = np.stack((boundaries[:-1], (boundaries + overlap)[1:])).T + n_chunks = int(np.ceil(length / (max_tokens - self.overlap))) + boundaries = np.ceil(np.linspace(0, length - self.overlap, n_chunks)).astype(int) + boundaries = np.stack((boundaries[:-1], (boundaries + self.overlap)[1:])).T boundaries = np.minimum(boundaries, length).tolist() # Get extension of filename like .txt