From 96eba88c90ac46bfe60bf7c07a20051408eb39be Mon Sep 17 00:00:00 2001 From: vangmay Date: Wed, 10 Dec 2025 10:46:29 +0530 Subject: [PATCH] =?UTF-8?q?Fix=20Chunking=20loop=20can=20hang=20when=20str?= =?UTF-8?q?ide=20=E2=89=A5=20chunk=5Fsize?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- unsloth/dataprep/raw_text.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py index b880e97338..f7c1bf7856 100644 --- a/unsloth/dataprep/raw_text.py +++ b/unsloth/dataprep/raw_text.py @@ -101,6 +101,13 @@ class RawTextDataLoader: 3. Maintains context with stride overlap 4. Returns tokenized chunks directly (more efficient) or text chunks """ + if chunk_size <= 0: + raise ValueError(f"chunk_size must be positive, got {chunk_size}") + if stride >= chunk_size: + raise ValueError( + f"stride ({stride}) must be smaller than chunk_size ({chunk_size}) to progress the chunking loop" + ) + # First pass: tokenize the entire text to get accurate token counts tokenized = self.tokenizer(text, return_tensors = "pt", add_special_tokens = False) tokens = tokenized["input_ids"]