From 082da69cc4f39637567ac69ac78d482bf1666f60 Mon Sep 17 00:00:00 2001 From: vangmay Date: Thu, 20 Nov 2025 21:40:45 +0800 Subject: [PATCH] remove old function --- unsloth/dataprep/raw_text.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py index d6a9a6a047..b880e97338 100644 --- a/unsloth/dataprep/raw_text.py +++ b/unsloth/dataprep/raw_text.py @@ -184,14 +184,6 @@ class RawTextDataLoader: return chunks - def tokenize_and_chunk(self, text): - """ - Tokenize first, then chunk by token count: - 1. More precise length control - 2. Avoids mid-token splits - 3. Handles different languages better - """ - def _read_file_by_format(self, file_path, file_format): """Read file content based on detected format.""" with open(file_path, "r", encoding = "utf-8") as f: