diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py index eba4577315..8c4606f418 100644 --- a/unsloth/chat_templates.py +++ b/unsloth/chat_templates.py @@ -2276,28 +2276,28 @@ def get_ollama_eos_tokens(tokenizer, extra_eos_tokens = []): if getattr(tokenizer, "bos_token", None) is not None: added_tokens_decoder = [x for x in added_tokens_decoder if x != tokenizer.bos_token] - repeatted_tokens = [] + repeated_tokens = [] # Join all vocab joined_text = "\x01\x00".join(added_tokens_decoder) for token in added_tokens_decoder: n = len(token) - repeatted_counts = joined_text.count(token[:n//2]) + repeated_counts = joined_text.count(token[:n//2]) # Try finding longer than 1/2 of the token in the rest # For eg <|reserved_special_token_0|>, <|reserved_special_token_1|> - if repeatted_counts > 2: + if repeated_counts > 2: for j in range(n//2+1, n): - if joined_text.count(token[:j]) < repeatted_counts: + if joined_text.count(token[:j]) < repeated_counts: j -= 1 - # Remove repeatted tokens to reduce search space + # Remove repeated tokens to reduce search space joined_text = joined_text.replace(token[:j], "") - repeatted_tokens.append(token[:j]) + repeated_tokens.append(token[:j]) break # Remove duplicates - splitted = joined_text.split("\x01\x00") - final_eos_tokens = [old for old, new in zip(added_tokens_decoder, splitted) if old == new] + split = joined_text.split("\x01\x00") + final_eos_tokens = [old for old, new in zip(added_tokens_decoder, split) if old == new] final_eos_tokens += extra_eos_tokens - final_eos_tokens += repeatted_tokens + final_eos_tokens += repeated_tokens # Remove new lines, spaces and HTML tags filtered_eos_tokens = [] diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py index 076a781d4b..7d18b1ff29 100644 --- a/unsloth/dataprep/raw_text.py +++ b/unsloth/dataprep/raw_text.py @@ -223,48 +223,63 @@ class RawTextDataLoader: return "\n\n".join(texts) return "" + # Cache text fields/columns for better performance + _TEXT_FIELDS = ("text", "content", "message", "body", "description", "prompt") + _TEXT_COLUMNS = _TEXT_FIELDS + def _extract_text_from_json(self, data): """Extract text from JSON object using common field names.""" - text_fields = ["text", "content", "message", "body", "description", "prompt"] - for field in text_fields: + for field in self._TEXT_FIELDS: if field in data and isinstance(data[field], str): return data[field] return "" def _extract_text_from_csv_row(self, row): """Extract text from CSV row using common column names.""" - text_columns = ["text", "content", "message", "body", "description", "prompt"] - for column in text_columns: + for column in self._TEXT_COLUMNS: if column in row and row[column]: return row[column] return "" class TextPreprocessor: + # Compile regex patterns once for better performance + _WHITESPACE_PATTERN = re.compile(r"[^\S\n]+") + _INVALID_CHARS_PATTERN = re.compile(r"[^\x20-\x7E\n]") + _MULTIPLE_SPACES_PATTERN = re.compile(r"[ ]{2,}") + _NEWLINE_SPACES_PATTERN = re.compile(r" *\n *") + _MULTIPLE_NEWLINES_PATTERN = re.compile(r"\n{3,}") + _CHAPTER_PATTERN = re.compile(r"^# (.+)$", re.MULTILINE) + _SECTION_PATTERN = re.compile(r"^## (.+)$", re.MULTILINE) + _SUBSECTION_PATTERN = re.compile(r"^### (.+)$", re.MULTILINE) + _CODE_BLOCK_PATTERN = re.compile(r"```(\w*)\n(.*?)\n```", re.DOTALL) + def clean_text(self, text): """Remove unwanted characters, normalize whitespace""" text = text.replace("\r\n", "\n").replace("\r", "\n") - text = re.sub(r"[^\S\n]+", " ", text) - text = re.sub(r"[^\x20-\x7E\n]", "", text) - text = re.sub(r"[ ]{2,}", " ", text) - text = re.sub(r" *\n *", "\n", text) - text = re.sub(r"\n{3,}", "\n\n", text) + text = self._WHITESPACE_PATTERN.sub(" ", text) + text = self._INVALID_CHARS_PATTERN.sub("", text) + text = self._MULTIPLE_SPACES_PATTERN.sub(" ", text) + text = self._NEWLINE_SPACES_PATTERN.sub("\n", text) + text = self._MULTIPLE_NEWLINES_PATTERN.sub("\n\n", text) return text.strip() def extract_sections(self, text, patterns): """Extract specific sections (e.g., code blocks, quotes)""" sections = [] for pattern in patterns: + # Compile pattern on first use and cache? Well, patterns are user-provided, + # so just use re.findall with compiled flags matches = re.findall(pattern, text, re.MULTILINE | re.DOTALL) sections.extend(matches) return sections def add_structure_tokens(self, text): """Add special tokens for structure (chapters, sections)""" - text = re.sub(r"^# (.+)$", r"<|chapter|>\1<|/chapter|>", text, flags = re.MULTILINE) - text = re.sub(r"^## (.+)$", r"<|section|>\1<|/section|>", text, flags = re.MULTILINE) - text = re.sub(r"^### (.+)$", r"<|subsection|>\1<|/subsection|>", text, flags = re.MULTILINE) - text = re.sub(r"```(\w*)\n(.*?)\n```", r"<|code|\1|>\2<|/code|>", text, flags = re.DOTALL) + text = self._CHAPTER_PATTERN.sub(r"<|chapter|>\1<|/chapter|>", text) + text = self._SECTION_PATTERN.sub(r"<|section|>\1<|/section|>", text) + text = self._SUBSECTION_PATTERN.sub(r"<|subsection|>\1<|/subsection|>", text) + text = self._CODE_BLOCK_PATTERN.sub(r"<|code|\1|>\2<|/code|>", text) return text def validate_dataset(self, dataset): diff --git a/unsloth/kernels/geglu.py b/unsloth/kernels/geglu.py index 3a628f105c..0ed325afa0 100644 --- a/unsloth/kernels/geglu.py +++ b/unsloth/kernels/geglu.py @@ -97,7 +97,7 @@ def _exact_backward_kernel( e_row = tl.load(e + offsets, mask = mask, other = 0).to(tl.float32) g_row = tl.load(g + offsets, mask = mask, other = 0) # .to(tl.float32) - # Break e_row away for re-use + # Break e_row away for reuse # f = 1/2 * e * (1 + erf(1/sqrt(2) * e)) f_partial_row = 0.5 * (tl.math.erf(tl.math.rsqrt(2.0) * e_row) + 1.0) f_row = f_partial_row * e_row diff --git a/unsloth/kernels/moe/grouped_gemm/kernels/forward.py b/unsloth/kernels/moe/grouped_gemm/kernels/forward.py index e95179dd9f..cb1f22d0b8 100644 --- a/unsloth/kernels/moe/grouped_gemm/kernels/forward.py +++ b/unsloth/kernels/moe/grouped_gemm/kernels/forward.py @@ -111,7 +111,7 @@ def _grouped_gemm_forward_kernel( while tidx >= processed_tiles and tidx < processed_tiles + num_tiles_per_expert: tile_idx = tidx - processed_tiles - # Check if L2 cache re-use for this order is optimal + # Check if L2 cache reuse for this order is optimal tile_m_idx = tile_idx % num_m_tiles tile_n_idx = tile_idx // num_m_tiles diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 7ad6e8ea33..599a5c0262 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -3078,7 +3078,7 @@ class TorchAOConfig: def _untie_input_output_embeddings(model: torch.nn.Module) -> None: """ Utility to untie input/output embeddings in a HuggingFace model. - This is useful if we want to quantize the input/ouput embeddings differently. + This is useful if we want to quantize the input/output embeddings differently. Model is modified in-place. """