From eb4dd11c493a1a7ecf366395ce29d6aa8a786ed1 Mon Sep 17 00:00:00 2001 From: vangmay Date: Tue, 18 Nov 2025 21:46:41 +0800 Subject: [PATCH 001/114] Write file and template for raw_text dataprep --- unsloth/dataprep/raw_text.py | 76 ++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 unsloth/dataprep/raw_text.py diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py new file mode 100644 index 0000000000..93c8c6855b --- /dev/null +++ b/unsloth/dataprep/raw_text.py @@ -0,0 +1,76 @@ +# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import json +import csv +from typing import List, Dict, Any, Union, Optional +from datasets import Dataset +from pathlib import Path + +class RawTextDataLoader: + def __init__(self, tokenizer, chunk_size=2048, stride=512): + self.tokenizer = tokenizer + self.chunk_size = chunk_size + self.stride = stride + + def load_from_file(self, file_path): + """Load raw text and convert to dataset""" + + def load_from_files(self, file_paths): + """Load multiple text files""" + + def chunk_text(self, text): + """Split text into overlapping chunks""" + + def create_causal_dataset(self, chunks): + """Create dataset for causal language modeling""" + + def smart_chunk_text(self, text, chunk_size, stride): + """ + Intelligent chunking that: + 1. Respects sentence/paragraph boundaries + 2. Handles various text formats (.txt, .md, .json, etc.) + 3. Maintains context with stride overlap + 4. Adds proper EOS tokens + """ + + def tokenize_and_chunk(self, text): + """ + Tokenize first, then chunk by token count: + 1. More precise length control + 2. Avoids mid-token splits + 3. Handles different languages better + """ + +class TextPreprocessor: + def clean_text(self, text): + """Remove unwanted characters, normalize whitespace""" + + def extract_sections(self, text, patterns): + """Extract specific sections (e.g., code blocks, quotes)""" + + def add_structure_tokens(self, text): + """Add special tokens for structure (chapters, sections)""" + +def validate_dataset(self, dataset): + """ + Check for: + - Minimum/maximum sequence lengths + - Character encoding issues + - Repeated content + - Empty chunks + """ + From 14985c4011519dfc9f5827d8d2a837d891abed0e Mon Sep 17 00:00:00 2001 From: vangmay Date: Tue, 18 Nov 2025 21:53:20 +0800 Subject: [PATCH 002/114] Add implementation to cli --- unsloth-cli.py | 48 ++++++++++++++++++++++++++++++++++++ unsloth/dataprep/raw_text.py | 25 +++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/unsloth-cli.py b/unsloth-cli.py index fb6e392662..aac0e7f7e1 100644 --- a/unsloth-cli.py +++ b/unsloth-cli.py @@ -42,6 +42,7 @@ def run(args): from transformers import TrainingArguments from unsloth import is_bfloat16_supported import logging + from unsloth import RawTextDataLoader logging.getLogger("hf-to-gguf").setLevel(logging.WARNING) @@ -98,6 +99,21 @@ def run(args): texts.append(text) return {"text": texts} + def load_dataset_smart(args): + if args.raw_text_file: + # Use raw text loader + loader = RawTextDataLoader(tokenizer, args.chunk_size, args.stride) + dataset = loader.load_from_file(args.raw_text_file) + elif args.dataset.endswith(('.txt', '.md', '.json', '.jsonl')): + # Auto-detect local raw text files + loader = RawTextDataLoader(tokenizer) + dataset = loader.load_from_file(args.dataset) + else: + # Existing HuggingFace dataset logic + dataset = load_dataset(args.dataset, split="train") + dataset = dataset.map(formatting_prompts_func, batched=True) + return dataset + use_modelscope = strtobool(os.environ.get("UNSLOTH_USE_MODELSCOPE", "False")) if use_modelscope: from modelscope import MsDataset @@ -389,5 +405,37 @@ if __name__ == "__main__": "--hub_token", type = str, help = "Token for pushing the model to Hugging Face hub" ) + parser.add_argument( + "--raw_text_file", + type=str, + help="Path to raw text file for training" + ) + parser.add_argument( + "--chunk_size", + type=int, + default=2048, + help="Size of text chunks for training" + ) + parser.add_argument( + "--stride", + type=int, + default=512, + help="Overlap between chunks" + ) + + TRAINING_MODES = { + 'instruction': 'Standard instruction-following', + 'causal': 'Causal language modeling (raw text)', + 'completion': 'Text completion tasks' + } + + parser.add_argument( + "--training_mode", + type=str, + default="instruction", + choices=list(TRAINING_MODES.keys()), + help="Training mode for the model" + ) + args = parser.parse_args() run(args) diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py index 93c8c6855b..e36978bf1c 100644 --- a/unsloth/dataprep/raw_text.py +++ b/unsloth/dataprep/raw_text.py @@ -20,12 +20,28 @@ from typing import List, Dict, Any, Union, Optional from datasets import Dataset from pathlib import Path +__all__ = [ + "RawTextDataLoader", + "TextPreprocessor", +] + +SUPPORTED_FORMATS = { + '.txt': 'plain_text', + '.md': 'markdown', + '.json': 'json_lines', + '.jsonl': 'json_lines', + '.csv': 'csv_text_column' +} + class RawTextDataLoader: def __init__(self, tokenizer, chunk_size=2048, stride=512): self.tokenizer = tokenizer self.chunk_size = chunk_size self.stride = stride + def detect_format(self, file_path): + """Auto-detect file format and parse accordingly""" + def load_from_file(self, file_path): """Load raw text and convert to dataset""" @@ -64,6 +80,15 @@ class TextPreprocessor: def add_structure_tokens(self, text): """Add special tokens for structure (chapters, sections)""" + + def validate_dataset(self, dataset): + """ + Check for: + - Minimum/maximum sequence lengths + - Character encoding issues + - Repeated content + - Empty chunks + """ def validate_dataset(self, dataset): """ From 84b12be161080315251700d9bab4f5587efa1558 Mon Sep 17 00:00:00 2001 From: vangmay Date: Tue, 18 Nov 2025 21:59:01 +0800 Subject: [PATCH 003/114] Add support for multiple files --- unsloth/dataprep/raw_text.py | 56 ++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py index e36978bf1c..8b705b3a96 100644 --- a/unsloth/dataprep/raw_text.py +++ b/unsloth/dataprep/raw_text.py @@ -41,12 +41,26 @@ class RawTextDataLoader: def detect_format(self, file_path): """Auto-detect file format and parse accordingly""" + extension = Path(file_path).suffix.lower() + return SUPPORTED_FORMATS.get(extension, 'plain_text') def load_from_file(self, file_path): """Load raw text and convert to dataset""" + file_format = self.detect_format(file_path) + text_content = self._read_file_by_format(file_path, file_format) + chunks = self.smart_chunk_text(text_content, self.chunk_size, self.stride) + return self.create_causal_dataset(chunks) def load_from_files(self, file_paths): """Load multiple text files""" + all_chunks = [] + for file_path in file_paths: + file_format = self.detect_format(file_path) + text_content = self._read_file_by_format(file_path, file_format) + chunks = self.smart_chunk_text(text_content, self.chunk_size, self.stride) + all_chunks.extend(chunks) + return self.create_causal_dataset(all_chunks) + def chunk_text(self, text): """Split text into overlapping chunks""" @@ -71,6 +85,48 @@ class RawTextDataLoader: 3. Handles different languages better """ + def _read_file_by_format(self, file_path, file_format): + """Read file content based on detected format.""" + with open(file_path, 'r', encoding='utf-8') as f: + if file_format == 'plain_text' or file_format == 'markdown': + return f.read() + elif file_format == 'json_lines': + lines = [] + for line in f: + try: + data = json.loads(line.strip()) + text = self._extract_text_from_json(data) + if text: + lines.append(text) + except json.JSONDecodeError: + continue + return '\n\n'.join(lines) + elif file_format == 'csv_text_column': + reader = csv.DictReader(f) + texts = [] + for row in reader: + text = self._extract_text_from_csv_row(row) + if text: + texts.append(text) + return '\n\n'.join(texts) + return "" + + def _extract_text_from_json(self, data): + """Extract text from JSON object using common field names.""" + text_fields = ['text', 'content', 'message', 'body', 'description', 'prompt'] + for field in text_fields: + if field in data and isinstance(data[field], str): + return data[field] + return "" + + def _extract_text_from_csv_row(self, row): + """Extract text from CSV row using common column names.""" + text_columns = ['text', 'content', 'message', 'body', 'description', 'prompt'] + for column in text_columns: + if column in row and row[column]: + return row[column] + return "" + class TextPreprocessor: def clean_text(self, text): """Remove unwanted characters, normalize whitespace""" From d4060b627a68b5c82ed825899be826f0679534dc Mon Sep 17 00:00:00 2001 From: vangmay Date: Tue, 18 Nov 2025 22:00:07 +0800 Subject: [PATCH 004/114] Write chunking logic --- unsloth/dataprep/raw_text.py | 47 ++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py index 8b705b3a96..44582c76a6 100644 --- a/unsloth/dataprep/raw_text.py +++ b/unsloth/dataprep/raw_text.py @@ -64,9 +64,11 @@ class RawTextDataLoader: def chunk_text(self, text): """Split text into overlapping chunks""" + return self.smart_chunk_text(text, self.chunk_size, self.stride) def create_causal_dataset(self, chunks): """Create dataset for causal language modeling""" + return Dataset.from_dict({"text": chunks}) def smart_chunk_text(self, text, chunk_size, stride): """ @@ -76,6 +78,51 @@ class RawTextDataLoader: 3. Maintains context with stride overlap 4. Adds proper EOS tokens """ + # First pass: tokenize the entire text to get accurate token counts + tokenized = self.tokenizer(text, return_tensors="pt", add_special_tokens=False) + tokens = tokenized["input_ids"] + + # Handle different tokenizer return formats + if hasattr(tokens, '__len__') and len(tokens) > 0: + # If it's a nested structure, get the first element + if hasattr(tokens[0], '__len__'): + tokens = tokens[0] + elif isinstance(tokens, int): + # If tokenizer returns just a count, create a simple range + tokens = list(range(tokens)) + + if len(tokens) <= chunk_size: + # Text is small enough to fit in one chunk + eos_token = self.tokenizer.eos_token if self.tokenizer.eos_token else "" + return [text + eos_token] + + chunks = [] + start_idx = 0 + + while start_idx < len(tokens): + # Calculate end index for this chunk + end_idx = min(start_idx + chunk_size, len(tokens)) + + # Extract tokens for this chunk + chunk_tokens = tokens[start_idx:end_idx] + + # Decode back to text + chunk_text = self.tokenizer.decode(chunk_tokens, skip_special_tokens=True) + + # Add EOS token if it's the last chunk or chunk is complete + if end_idx == len(tokens) or len(chunk_tokens) == chunk_size: + eos_token = self.tokenizer.eos_token if self.tokenizer.eos_token else "" + chunk_text += eos_token + + chunks.append(chunk_text) + + # Move to next chunk with stride overlap + if end_idx == len(tokens): + break + start_idx += chunk_size - stride + + return chunks + def tokenize_and_chunk(self, text): """ From d404ff653c2d24e68d3c23f8b3ee5e30eca33e42 Mon Sep 17 00:00:00 2001 From: vangmay Date: Tue, 18 Nov 2025 22:01:36 +0800 Subject: [PATCH 005/114] Add logic to clean and extract text sections --- unsloth/dataprep/raw_text.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py index 44582c76a6..b31120cc59 100644 --- a/unsloth/dataprep/raw_text.py +++ b/unsloth/dataprep/raw_text.py @@ -177,12 +177,27 @@ class RawTextDataLoader: class TextPreprocessor: def clean_text(self, text): """Remove unwanted characters, normalize whitespace""" + text = re.sub(r'\s+', ' ', text) + text = re.sub(r'[^\x20-\x7E\n\t]', '', text) + text = text.replace('\r\n', '\n').replace('\r', '\n') + text = re.sub(r'\n{3,}', '\n\n', text) + return text.strip() def extract_sections(self, text, patterns): """Extract specific sections (e.g., code blocks, quotes)""" - + sections = [] + for pattern in patterns: + matches = re.findall(pattern, text, re.MULTILINE | re.DOTALL) + sections.extend(matches) + return sections + def add_structure_tokens(self, text): """Add special tokens for structure (chapters, sections)""" + text = re.sub(r'^# (.+)$', r'<|chapter|>\1<|/chapter|>', text, flags=re.MULTILINE) + text = re.sub(r'^## (.+)$', r'<|section|>\1<|/section|>', text, flags=re.MULTILINE) + text = re.sub(r'^### (.+)$', r'<|subsection|>\1<|/subsection|>', text, flags=re.MULTILINE) + text = re.sub(r'```(\w*)\n(.*?)\n```', r'<|code|\1|>\2<|/code|>', text, flags=re.DOTALL) + return text def validate_dataset(self, dataset): """ From afdbeff8e957e1f61dd6a5fd9617b4f9c42d623c Mon Sep 17 00:00:00 2001 From: vangmay Date: Tue, 18 Nov 2025 22:02:35 +0800 Subject: [PATCH 006/114] Add validation code --- unsloth/dataprep/raw_text.py | 67 +++++++++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 9 deletions(-) diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py index b31120cc59..dd6d6b96d0 100644 --- a/unsloth/dataprep/raw_text.py +++ b/unsloth/dataprep/raw_text.py @@ -207,13 +207,62 @@ class TextPreprocessor: - Repeated content - Empty chunks """ - -def validate_dataset(self, dataset): - """ - Check for: - - Minimum/maximum sequence lengths - - Character encoding issues - - Repeated content - - Empty chunks - """ + stats = { + 'total_samples': len(dataset), + 'empty_samples': 0, + 'min_length': float('inf'), + 'max_length': 0, + 'avg_length': 0, + 'repeated_content': 0, + 'encoding_issues': 0, + 'warnings': [] + } + + texts = dataset['text'] + text_lengths = [] + seen_texts = set() + + for i, text in enumerate(texts): + if not text or len(text.strip()) == 0: + stats['empty_samples'] += 1 + continue + + # Check for encoding issues + try: + text.encode('utf-8') + except UnicodeEncodeError: + stats['encoding_issues'] += 1 + + # Calculate lengths + length = len(text) + text_lengths.append(length) + stats['min_length'] = min(stats['min_length'], length) + stats['max_length'] = max(stats['max_length'], length) + + # Check for repeated content + text_hash = hash(text.strip()) + if text_hash in seen_texts: + stats['repeated_content'] += 1 + else: + seen_texts.add(text_hash) + + # Calculate average length + if text_lengths: + stats['avg_length'] = sum(text_lengths) / len(text_lengths) + stats['min_length'] = stats['min_length'] if stats['min_length'] != float('inf') else 0 + + # Generate warnings + if stats['empty_samples'] > 0: + stats['warnings'].append(f"Found {stats['empty_samples']} empty samples") + + if stats['repeated_content'] > 0: + stats['warnings'].append(f"Found {stats['repeated_content']} repeated samples") + + if stats['encoding_issues'] > 0: + stats['warnings'].append(f"Found {stats['encoding_issues']} encoding issues") + + if stats['min_length'] < 10: + stats['warnings'].append("Some samples are very short (< 10 characters)") + + return stats From e473c0dbdf968bdf22065ad4f5ba0bdc8a6caff4 Mon Sep 17 00:00:00 2001 From: vangmay Date: Tue, 18 Nov 2025 22:36:38 +0800 Subject: [PATCH 007/114] Write simple test --- tests/test_raw_text.py | 121 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 tests/test_raw_text.py diff --git a/tests/test_raw_text.py b/tests/test_raw_text.py new file mode 100644 index 0000000000..503fbeb4c3 --- /dev/null +++ b/tests/test_raw_text.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +""" +Minimal test for raw text training implementation. +Tests basic functionality without heavy dependencies. +""" + +import sys +import os +import tempfile +from pathlib import Path +import importlib.util + +# Mock the datasets module since it's not installed +class MockDataset: + def __init__(self, data_dict): + self.data = data_dict + self.column_names = list(data_dict.keys()) + + def __len__(self): + return len(next(iter(self.data.values()))) + + def __getitem__(self, idx): + if isinstance(idx, str): + # Allow accessing columns by name like dataset['text'] + return self.data[idx] + elif isinstance(idx, int): + # Allow accessing individual rows by index + return {key: values[idx] for key, values in self.data.items()} + else: + raise TypeError(f"Invalid index type: {type(idx)}") + + @classmethod + def from_dict(cls, data_dict): + return cls(data_dict) + +# Mock datasets module +datasets_mock = type(sys)('datasets') +datasets_mock.Dataset = MockDataset +sys.modules['datasets'] = datasets_mock + +# Import the raw_text module directly to avoid unsloth/__init__.py dependencies +current_dir = os.path.dirname(__file__) +raw_text_path = os.path.join(os.path.dirname(current_dir), 'unsloth', 'dataprep', 'raw_text.py') + +spec = importlib.util.spec_from_file_location("raw_text", raw_text_path) +raw_text_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(raw_text_module) + +RawTextDataLoader = raw_text_module.RawTextDataLoader +TextPreprocessor = raw_text_module.TextPreprocessor + +def test_raw_text_loader(): + """Test basic RawTextDataLoader functionality.""" + + # Mock tokenizer for testing + class MockTokenizer: + def __init__(self): + self.eos_token = "" + + def __call__(self, text, return_tensors=None, add_special_tokens=False): + words = text.split() + token_ids = list(range(len(words))) + + if return_tensors == "pt": + # Mock tensor-like object + class MockTensor: + def __init__(self, data): + self.data = data + def __getitem__(self, idx): + return self.data + def __len__(self): + return len(self.data) + return {"input_ids": [MockTensor(token_ids)]} + return {"input_ids": token_ids} + + def decode(self, token_ids, skip_special_tokens=False): + return " ".join([f"word_{i}" for i in token_ids]) + + + + + # Create test file + test_content = "This is a test file for raw text training. " * 10 + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + f.write(test_content) + test_file = f.name + + try: + # Test loader + tokenizer = MockTokenizer() + loader = RawTextDataLoader(tokenizer, chunk_size=5, stride=2) + + # Test loading + dataset = loader.load_from_file(test_file) + assert len(dataset) > 0, "Should create at least one chunk" + assert 'text' in dataset.column_names, "Dataset should have 'text' column" + + # Test preprocessor + preprocessor = TextPreprocessor() + clean_text = preprocessor.clean_text(" messy text \n\n\n ") + assert "messy text" in clean_text, "Should clean text properly" + + # Test validation + stats = preprocessor.validate_dataset(dataset) + assert stats['total_samples'] > 0, "Should count samples" + assert 'warnings' in stats, "Should include warnings" + + print("✅ All tests passed!") + return True + + except Exception as e: + print(f"❌ Test failed: {e}") + return False + + finally: + # Cleanup + os.unlink(test_file) + +if __name__ == "__main__": + success = test_raw_text_loader() + sys.exit(0 if success else 1) \ No newline at end of file From d738a087f928611a60ed3098f95a185c187266f7 Mon Sep 17 00:00:00 2001 From: vangmay Date: Tue, 18 Nov 2025 22:44:48 +0800 Subject: [PATCH 008/114] Add module to init --- unsloth/dataprep/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unsloth/dataprep/__init__.py b/unsloth/dataprep/__init__.py index b36122eb74..b6840f247f 100644 --- a/unsloth/dataprep/__init__.py +++ b/unsloth/dataprep/__init__.py @@ -13,3 +13,4 @@ # limitations under the License. from .synthetic import * +from raw_text import * From 69e89677423f2ad062728d47f9aa318946bb6558 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 20 Nov 2025 12:51:17 +0000 Subject: [PATCH 009/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_raw_text.py | 64 ++++++------ unsloth-cli.py | 34 +++---- unsloth/dataprep/raw_text.py | 183 +++++++++++++++++++---------------- 3 files changed, 146 insertions(+), 135 deletions(-) diff --git a/tests/test_raw_text.py b/tests/test_raw_text.py index 503fbeb4c3..9bbfee92ac 100644 --- a/tests/test_raw_text.py +++ b/tests/test_raw_text.py @@ -10,15 +10,16 @@ import tempfile from pathlib import Path import importlib.util + # Mock the datasets module since it's not installed class MockDataset: def __init__(self, data_dict): self.data = data_dict self.column_names = list(data_dict.keys()) - + def __len__(self): return len(next(iter(self.data.values()))) - + def __getitem__(self, idx): if isinstance(idx, str): # Allow accessing columns by name like dataset['text'] @@ -28,19 +29,22 @@ class MockDataset: return {key: values[idx] for key, values in self.data.items()} else: raise TypeError(f"Invalid index type: {type(idx)}") - + @classmethod def from_dict(cls, data_dict): return cls(data_dict) + # Mock datasets module -datasets_mock = type(sys)('datasets') +datasets_mock = type(sys)("datasets") datasets_mock.Dataset = MockDataset -sys.modules['datasets'] = datasets_mock +sys.modules["datasets"] = datasets_mock # Import the raw_text module directly to avoid unsloth/__init__.py dependencies current_dir = os.path.dirname(__file__) -raw_text_path = os.path.join(os.path.dirname(current_dir), 'unsloth', 'dataprep', 'raw_text.py') +raw_text_path = os.path.join( + os.path.dirname(current_dir), "unsloth", "dataprep", "raw_text.py" +) spec = importlib.util.spec_from_file_location("raw_text", raw_text_path) raw_text_module = importlib.util.module_from_spec(spec) @@ -49,73 +53,75 @@ spec.loader.exec_module(raw_text_module) RawTextDataLoader = raw_text_module.RawTextDataLoader TextPreprocessor = raw_text_module.TextPreprocessor + def test_raw_text_loader(): """Test basic RawTextDataLoader functionality.""" - + # Mock tokenizer for testing class MockTokenizer: def __init__(self): self.eos_token = "" - - def __call__(self, text, return_tensors=None, add_special_tokens=False): + + def __call__(self, text, return_tensors = None, add_special_tokens = False): words = text.split() token_ids = list(range(len(words))) - + if return_tensors == "pt": # Mock tensor-like object class MockTensor: def __init__(self, data): self.data = data + def __getitem__(self, idx): return self.data + def __len__(self): return len(self.data) + return {"input_ids": [MockTensor(token_ids)]} return {"input_ids": token_ids} - - def decode(self, token_ids, skip_special_tokens=False): + + def decode(self, token_ids, skip_special_tokens = False): return " ".join([f"word_{i}" for i in token_ids]) - - - - + # Create test file test_content = "This is a test file for raw text training. " * 10 - with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + with tempfile.NamedTemporaryFile(mode = "w", suffix = ".txt", delete = False) as f: f.write(test_content) test_file = f.name - + try: # Test loader tokenizer = MockTokenizer() - loader = RawTextDataLoader(tokenizer, chunk_size=5, stride=2) - + loader = RawTextDataLoader(tokenizer, chunk_size = 5, stride = 2) + # Test loading dataset = loader.load_from_file(test_file) assert len(dataset) > 0, "Should create at least one chunk" - assert 'text' in dataset.column_names, "Dataset should have 'text' column" - + assert "text" in dataset.column_names, "Dataset should have 'text' column" + # Test preprocessor preprocessor = TextPreprocessor() clean_text = preprocessor.clean_text(" messy text \n\n\n ") assert "messy text" in clean_text, "Should clean text properly" - + # Test validation stats = preprocessor.validate_dataset(dataset) - assert stats['total_samples'] > 0, "Should count samples" - assert 'warnings' in stats, "Should include warnings" - + assert stats["total_samples"] > 0, "Should count samples" + assert "warnings" in stats, "Should include warnings" + print("✅ All tests passed!") return True - + except Exception as e: print(f"❌ Test failed: {e}") return False - + finally: # Cleanup os.unlink(test_file) + if __name__ == "__main__": success = test_raw_text_loader() - sys.exit(0 if success else 1) \ No newline at end of file + sys.exit(0 if success else 1) diff --git a/unsloth-cli.py b/unsloth-cli.py index aac0e7f7e1..e454a47c44 100644 --- a/unsloth-cli.py +++ b/unsloth-cli.py @@ -104,14 +104,14 @@ def run(args): # Use raw text loader loader = RawTextDataLoader(tokenizer, args.chunk_size, args.stride) dataset = loader.load_from_file(args.raw_text_file) - elif args.dataset.endswith(('.txt', '.md', '.json', '.jsonl')): + elif args.dataset.endswith((".txt", ".md", ".json", ".jsonl")): # Auto-detect local raw text files loader = RawTextDataLoader(tokenizer) dataset = loader.load_from_file(args.dataset) else: # Existing HuggingFace dataset logic - dataset = load_dataset(args.dataset, split="train") - dataset = dataset.map(formatting_prompts_func, batched=True) + dataset = load_dataset(args.dataset, split = "train") + dataset = dataset.map(formatting_prompts_func, batched = True) return dataset use_modelscope = strtobool(os.environ.get("UNSLOTH_USE_MODELSCOPE", "False")) @@ -406,35 +406,27 @@ if __name__ == "__main__": ) parser.add_argument( - "--raw_text_file", - type=str, - help="Path to raw text file for training" + "--raw_text_file", type = str, help = "Path to raw text file for training" ) parser.add_argument( - "--chunk_size", - type=int, - default=2048, - help="Size of text chunks for training" + "--chunk_size", type = int, default = 2048, help = "Size of text chunks for training" ) parser.add_argument( - "--stride", - type=int, - default=512, - help="Overlap between chunks" + "--stride", type = int, default = 512, help = "Overlap between chunks" ) TRAINING_MODES = { - 'instruction': 'Standard instruction-following', - 'causal': 'Causal language modeling (raw text)', - 'completion': 'Text completion tasks' + "instruction": "Standard instruction-following", + "causal": "Causal language modeling (raw text)", + "completion": "Text completion tasks", } parser.add_argument( "--training_mode", - type=str, - default="instruction", - choices=list(TRAINING_MODES.keys()), - help="Training mode for the model" + type = str, + default = "instruction", + choices = list(TRAINING_MODES.keys()), + help = "Training mode for the model", ) args = parser.parse_args() diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py index dd6d6b96d0..d809eb312a 100644 --- a/unsloth/dataprep/raw_text.py +++ b/unsloth/dataprep/raw_text.py @@ -26,23 +26,24 @@ __all__ = [ ] SUPPORTED_FORMATS = { - '.txt': 'plain_text', - '.md': 'markdown', - '.json': 'json_lines', - '.jsonl': 'json_lines', - '.csv': 'csv_text_column' + ".txt": "plain_text", + ".md": "markdown", + ".json": "json_lines", + ".jsonl": "json_lines", + ".csv": "csv_text_column", } + class RawTextDataLoader: - def __init__(self, tokenizer, chunk_size=2048, stride=512): + def __init__(self, tokenizer, chunk_size = 2048, stride = 512): self.tokenizer = tokenizer - self.chunk_size = chunk_size + self.chunk_size = chunk_size self.stride = stride - + def detect_format(self, file_path): """Auto-detect file format and parse accordingly""" extension = Path(file_path).suffix.lower() - return SUPPORTED_FORMATS.get(extension, 'plain_text') + return SUPPORTED_FORMATS.get(extension, "plain_text") def load_from_file(self, file_path): """Load raw text and convert to dataset""" @@ -50,7 +51,7 @@ class RawTextDataLoader: text_content = self._read_file_by_format(file_path, file_format) chunks = self.smart_chunk_text(text_content, self.chunk_size, self.stride) return self.create_causal_dataset(chunks) - + def load_from_files(self, file_paths): """Load multiple text files""" all_chunks = [] @@ -61,11 +62,10 @@ class RawTextDataLoader: all_chunks.extend(chunks) return self.create_causal_dataset(all_chunks) - def chunk_text(self, text): """Split text into overlapping chunks""" return self.smart_chunk_text(text, self.chunk_size, self.stride) - + def create_causal_dataset(self, chunks): """Create dataset for causal language modeling""" return Dataset.from_dict({"text": chunks}) @@ -79,51 +79,50 @@ class RawTextDataLoader: 4. Adds proper EOS tokens """ # First pass: tokenize the entire text to get accurate token counts - tokenized = self.tokenizer(text, return_tensors="pt", add_special_tokens=False) + tokenized = self.tokenizer(text, return_tensors = "pt", add_special_tokens = False) tokens = tokenized["input_ids"] - + # Handle different tokenizer return formats - if hasattr(tokens, '__len__') and len(tokens) > 0: + if hasattr(tokens, "__len__") and len(tokens) > 0: # If it's a nested structure, get the first element - if hasattr(tokens[0], '__len__'): + if hasattr(tokens[0], "__len__"): tokens = tokens[0] elif isinstance(tokens, int): # If tokenizer returns just a count, create a simple range tokens = list(range(tokens)) - + if len(tokens) <= chunk_size: # Text is small enough to fit in one chunk eos_token = self.tokenizer.eos_token if self.tokenizer.eos_token else "" return [text + eos_token] - + chunks = [] start_idx = 0 - + while start_idx < len(tokens): # Calculate end index for this chunk end_idx = min(start_idx + chunk_size, len(tokens)) - + # Extract tokens for this chunk chunk_tokens = tokens[start_idx:end_idx] - + # Decode back to text - chunk_text = self.tokenizer.decode(chunk_tokens, skip_special_tokens=True) - + chunk_text = self.tokenizer.decode(chunk_tokens, skip_special_tokens = True) + # Add EOS token if it's the last chunk or chunk is complete if end_idx == len(tokens) or len(chunk_tokens) == chunk_size: eos_token = self.tokenizer.eos_token if self.tokenizer.eos_token else "" chunk_text += eos_token - + chunks.append(chunk_text) - + # Move to next chunk with stride overlap if end_idx == len(tokens): break start_idx += chunk_size - stride - + return chunks - def tokenize_and_chunk(self, text): """ Tokenize first, then chunk by token count: @@ -134,10 +133,10 @@ class RawTextDataLoader: def _read_file_by_format(self, file_path, file_format): """Read file content based on detected format.""" - with open(file_path, 'r', encoding='utf-8') as f: - if file_format == 'plain_text' or file_format == 'markdown': + with open(file_path, "r", encoding = "utf-8") as f: + if file_format == "plain_text" or file_format == "markdown": return f.read() - elif file_format == 'json_lines': + elif file_format == "json_lines": lines = [] for line in f: try: @@ -147,42 +146,43 @@ class RawTextDataLoader: lines.append(text) except json.JSONDecodeError: continue - return '\n\n'.join(lines) - elif file_format == 'csv_text_column': + return "\n\n".join(lines) + elif file_format == "csv_text_column": reader = csv.DictReader(f) texts = [] for row in reader: text = self._extract_text_from_csv_row(row) if text: texts.append(text) - return '\n\n'.join(texts) + return "\n\n".join(texts) return "" - + def _extract_text_from_json(self, data): """Extract text from JSON object using common field names.""" - text_fields = ['text', 'content', 'message', 'body', 'description', 'prompt'] + text_fields = ["text", "content", "message", "body", "description", "prompt"] for field in text_fields: if field in data and isinstance(data[field], str): return data[field] return "" - + def _extract_text_from_csv_row(self, row): """Extract text from CSV row using common column names.""" - text_columns = ['text', 'content', 'message', 'body', 'description', 'prompt'] + text_columns = ["text", "content", "message", "body", "description", "prompt"] for column in text_columns: if column in row and row[column]: return row[column] return "" + class TextPreprocessor: def clean_text(self, text): """Remove unwanted characters, normalize whitespace""" - text = re.sub(r'\s+', ' ', text) - text = re.sub(r'[^\x20-\x7E\n\t]', '', text) - text = text.replace('\r\n', '\n').replace('\r', '\n') - text = re.sub(r'\n{3,}', '\n\n', text) + text = re.sub(r"\s+", " ", text) + text = re.sub(r"[^\x20-\x7E\n\t]", "", text) + text = text.replace("\r\n", "\n").replace("\r", "\n") + text = re.sub(r"\n{3,}", "\n\n", text) return text.strip() - + def extract_sections(self, text, patterns): """Extract specific sections (e.g., code blocks, quotes)""" sections = [] @@ -193,12 +193,20 @@ class TextPreprocessor: def add_structure_tokens(self, text): """Add special tokens for structure (chapters, sections)""" - text = re.sub(r'^# (.+)$', r'<|chapter|>\1<|/chapter|>', text, flags=re.MULTILINE) - text = re.sub(r'^## (.+)$', r'<|section|>\1<|/section|>', text, flags=re.MULTILINE) - text = re.sub(r'^### (.+)$', r'<|subsection|>\1<|/subsection|>', text, flags=re.MULTILINE) - text = re.sub(r'```(\w*)\n(.*?)\n```', r'<|code|\1|>\2<|/code|>', text, flags=re.DOTALL) + text = re.sub( + r"^# (.+)$", r"<|chapter|>\1<|/chapter|>", text, flags = re.MULTILINE + ) + text = re.sub( + r"^## (.+)$", r"<|section|>\1<|/section|>", text, flags = re.MULTILINE + ) + text = re.sub( + r"^### (.+)$", r"<|subsection|>\1<|/subsection|>", text, flags = re.MULTILINE + ) + text = re.sub( + r"```(\w*)\n(.*?)\n```", r"<|code|\1|>\2<|/code|>", text, flags = re.DOTALL + ) return text - + def validate_dataset(self, dataset): """ Check for: @@ -208,61 +216,66 @@ class TextPreprocessor: - Empty chunks """ stats = { - 'total_samples': len(dataset), - 'empty_samples': 0, - 'min_length': float('inf'), - 'max_length': 0, - 'avg_length': 0, - 'repeated_content': 0, - 'encoding_issues': 0, - 'warnings': [] + "total_samples": len(dataset), + "empty_samples": 0, + "min_length": float("inf"), + "max_length": 0, + "avg_length": 0, + "repeated_content": 0, + "encoding_issues": 0, + "warnings": [], } - - texts = dataset['text'] + + texts = dataset["text"] text_lengths = [] seen_texts = set() - + for i, text in enumerate(texts): if not text or len(text.strip()) == 0: - stats['empty_samples'] += 1 + stats["empty_samples"] += 1 continue - + # Check for encoding issues try: - text.encode('utf-8') + text.encode("utf-8") except UnicodeEncodeError: - stats['encoding_issues'] += 1 - + stats["encoding_issues"] += 1 + # Calculate lengths length = len(text) text_lengths.append(length) - stats['min_length'] = min(stats['min_length'], length) - stats['max_length'] = max(stats['max_length'], length) - + stats["min_length"] = min(stats["min_length"], length) + stats["max_length"] = max(stats["max_length"], length) + # Check for repeated content text_hash = hash(text.strip()) if text_hash in seen_texts: - stats['repeated_content'] += 1 + stats["repeated_content"] += 1 else: seen_texts.add(text_hash) - + # Calculate average length if text_lengths: - stats['avg_length'] = sum(text_lengths) / len(text_lengths) - stats['min_length'] = stats['min_length'] if stats['min_length'] != float('inf') else 0 - - # Generate warnings - if stats['empty_samples'] > 0: - stats['warnings'].append(f"Found {stats['empty_samples']} empty samples") - - if stats['repeated_content'] > 0: - stats['warnings'].append(f"Found {stats['repeated_content']} repeated samples") - - if stats['encoding_issues'] > 0: - stats['warnings'].append(f"Found {stats['encoding_issues']} encoding issues") - - if stats['min_length'] < 10: - stats['warnings'].append("Some samples are very short (< 10 characters)") - - return stats + stats["avg_length"] = sum(text_lengths) / len(text_lengths) + stats["min_length"] = ( + stats["min_length"] if stats["min_length"] != float("inf") else 0 + ) + # Generate warnings + if stats["empty_samples"] > 0: + stats["warnings"].append(f"Found {stats['empty_samples']} empty samples") + + if stats["repeated_content"] > 0: + stats["warnings"].append( + f"Found {stats['repeated_content']} repeated samples" + ) + + if stats["encoding_issues"] > 0: + stats["warnings"].append( + f"Found {stats['encoding_issues']} encoding issues" + ) + + if stats["min_length"] < 10: + stats["warnings"].append("Some samples are very short (< 10 characters)") + + return stats From f80fe573e6a44e1337cfd2bae7022fba53437d44 Mon Sep 17 00:00:00 2001 From: vangmay Date: Thu, 20 Nov 2025 20:53:22 +0800 Subject: [PATCH 010/114] Integrate smart dataset loader --- unsloth-cli.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/unsloth-cli.py b/unsloth-cli.py index aac0e7f7e1..044ad93f31 100644 --- a/unsloth-cli.py +++ b/unsloth-cli.py @@ -100,6 +100,8 @@ def run(args): return {"text": texts} def load_dataset_smart(args): + from transformers.utils import strtobool + if args.raw_text_file: # Use raw text loader loader = RawTextDataLoader(tokenizer, args.chunk_size, args.stride) @@ -109,20 +111,21 @@ def run(args): loader = RawTextDataLoader(tokenizer) dataset = loader.load_from_file(args.dataset) else: - # Existing HuggingFace dataset logic - dataset = load_dataset(args.dataset, split="train") + # Check for modelscope usage + use_modelscope = strtobool(os.environ.get("UNSLOTH_USE_MODELSCOPE", "False")) + if use_modelscope: + from modelscope import MsDataset + dataset = MsDataset.load(args.dataset, split="train") + else: + # Existing HuggingFace dataset logic + dataset = load_dataset(args.dataset, split="train") + + # Apply formatting for structured datasets dataset = dataset.map(formatting_prompts_func, batched=True) return dataset - use_modelscope = strtobool(os.environ.get("UNSLOTH_USE_MODELSCOPE", "False")) - if use_modelscope: - from modelscope import MsDataset - - dataset = MsDataset.load(args.dataset, split = "train") - else: - # Load and format dataset - dataset = load_dataset(args.dataset, split = "train") - dataset = dataset.map(formatting_prompts_func, batched = True) + # Load dataset using smart loader + dataset = load_dataset_smart(args) print("Data is formatted and ready!") # Configure training arguments From 12474b14d9234f42815ad47c758f979d6680b491 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 20 Nov 2025 12:57:53 +0000 Subject: [PATCH 011/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth-cli.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/unsloth-cli.py b/unsloth-cli.py index 44232e19d7..79b3ef5296 100644 --- a/unsloth-cli.py +++ b/unsloth-cli.py @@ -101,7 +101,7 @@ def run(args): def load_dataset_smart(args): from transformers.utils import strtobool - + if args.raw_text_file: # Use raw text loader loader = RawTextDataLoader(tokenizer, args.chunk_size, args.stride) @@ -112,16 +112,19 @@ def run(args): dataset = loader.load_from_file(args.dataset) else: # Check for modelscope usage - use_modelscope = strtobool(os.environ.get("UNSLOTH_USE_MODELSCOPE", "False")) + use_modelscope = strtobool( + os.environ.get("UNSLOTH_USE_MODELSCOPE", "False") + ) if use_modelscope: from modelscope import MsDataset - dataset = MsDataset.load(args.dataset, split="train") + + dataset = MsDataset.load(args.dataset, split = "train") else: # Existing HuggingFace dataset logic - dataset = load_dataset(args.dataset, split="train") - + dataset = load_dataset(args.dataset, split = "train") + # Apply formatting for structured datasets - dataset = dataset.map(formatting_prompts_func, batched=True) + dataset = dataset.map(formatting_prompts_func, batched = True) return dataset # Load dataset using smart loader From e57b1c73fde87f382f3123d02a84a164f934d6bb Mon Sep 17 00:00:00 2001 From: vangmay Date: Thu, 20 Nov 2025 21:08:33 +0800 Subject: [PATCH 012/114] Make the chunk function efficient --- tests/test_raw_text.py | 24 ++++++++-- unsloth-cli.py | 20 ++++++--- unsloth/dataprep/raw_text.py | 85 ++++++++++++++++++++++++++++-------- 3 files changed, 99 insertions(+), 30 deletions(-) diff --git a/tests/test_raw_text.py b/tests/test_raw_text.py index 9bbfee92ac..88dac2604d 100644 --- a/tests/test_raw_text.py +++ b/tests/test_raw_text.py @@ -61,6 +61,7 @@ def test_raw_text_loader(): class MockTokenizer: def __init__(self): self.eos_token = "" + self.eos_token_id = 2 # Mock EOS token ID def __call__(self, text, return_tensors = None, add_special_tokens = False): words = text.split() @@ -77,6 +78,9 @@ def test_raw_text_loader(): def __len__(self): return len(self.data) + + def tolist(self): + return self.data return {"input_ids": [MockTensor(token_ids)]} return {"input_ids": token_ids} @@ -95,10 +99,22 @@ def test_raw_text_loader(): tokenizer = MockTokenizer() loader = RawTextDataLoader(tokenizer, chunk_size = 5, stride = 2) - # Test loading - dataset = loader.load_from_file(test_file) - assert len(dataset) > 0, "Should create at least one chunk" - assert "text" in dataset.column_names, "Dataset should have 'text' column" + # Test loading with text output (legacy mode) + text_dataset = loader.load_from_file(test_file, return_tensors=False) + assert len(text_dataset) > 0, "Should create at least one chunk" + assert "text" in text_dataset.column_names, "Dataset should have 'text' column" + + # Test loading with tokenized output (new efficient mode) + tokenized_dataset = loader.load_from_file(test_file, return_tensors=True) + assert len(tokenized_dataset) > 0, "Should create at least one tokenized chunk" + assert "input_ids" in tokenized_dataset.column_names, "Dataset should have 'input_ids' column" + assert "attention_mask" in tokenized_dataset.column_names, "Dataset should have 'attention_mask' column" + + # Verify tokenized data structure + first_sample = tokenized_dataset[0] + assert isinstance(first_sample["input_ids"], list), "input_ids should be a list" + assert isinstance(first_sample["attention_mask"], list), "attention_mask should be a list" + assert len(first_sample["input_ids"]) == len(first_sample["attention_mask"]), "input_ids and attention_mask should have same length" # Test preprocessor preprocessor = TextPreprocessor() diff --git a/unsloth-cli.py b/unsloth-cli.py index 79b3ef5296..efdc0b3f4d 100644 --- a/unsloth-cli.py +++ b/unsloth-cli.py @@ -103,13 +103,19 @@ def run(args): from transformers.utils import strtobool if args.raw_text_file: - # Use raw text loader + # Use raw text loader - returns pre-tokenized data loader = RawTextDataLoader(tokenizer, args.chunk_size, args.stride) - dataset = loader.load_from_file(args.raw_text_file) + dataset = loader.load_from_file(args.raw_text_file, return_tensors=True) + # Mark dataset as pre-tokenized to skip text formatting + dataset._is_pretokenized = True + return dataset elif args.dataset.endswith((".txt", ".md", ".json", ".jsonl")): - # Auto-detect local raw text files - loader = RawTextDataLoader(tokenizer) - dataset = loader.load_from_file(args.dataset) + # Auto-detect local raw text files - returns pre-tokenized data + loader = RawTextDataLoader(tokenizer, args.chunk_size, args.stride) + dataset = loader.load_from_file(args.dataset, return_tensors=True) + # Mark dataset as pre-tokenized to skip text formatting + dataset._is_pretokenized = True + return dataset else: # Check for modelscope usage use_modelscope = strtobool( @@ -123,9 +129,9 @@ def run(args): # Existing HuggingFace dataset logic dataset = load_dataset(args.dataset, split = "train") - # Apply formatting for structured datasets + # Apply formatting for structured datasets (text-based) dataset = dataset.map(formatting_prompts_func, batched = True) - return dataset + return dataset # Load dataset using smart loader dataset = load_dataset_smart(args) diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py index d809eb312a..50612c1b86 100644 --- a/unsloth/dataprep/raw_text.py +++ b/unsloth/dataprep/raw_text.py @@ -35,48 +35,66 @@ SUPPORTED_FORMATS = { class RawTextDataLoader: - def __init__(self, tokenizer, chunk_size = 2048, stride = 512): + def __init__(self, tokenizer, chunk_size = 2048, stride = 512, return_tokenized = True): self.tokenizer = tokenizer self.chunk_size = chunk_size self.stride = stride + self.return_tokenized = return_tokenized def detect_format(self, file_path): """Auto-detect file format and parse accordingly""" extension = Path(file_path).suffix.lower() return SUPPORTED_FORMATS.get(extension, "plain_text") - def load_from_file(self, file_path): + def load_from_file(self, file_path, return_tokenized=None): """Load raw text and convert to dataset""" + if return_tokenized is None: + return_tokenized = self.return_tokenized file_format = self.detect_format(file_path) text_content = self._read_file_by_format(file_path, file_format) - chunks = self.smart_chunk_text(text_content, self.chunk_size, self.stride) + chunks = self.smart_chunk_text(text_content, self.chunk_size, self.stride, return_tokenized) return self.create_causal_dataset(chunks) - def load_from_files(self, file_paths): + def load_from_files(self, file_paths, return_tokenized=None): """Load multiple text files""" + if return_tokenized is None: + return_tokenized = self.return_tokenized all_chunks = [] for file_path in file_paths: file_format = self.detect_format(file_path) text_content = self._read_file_by_format(file_path, file_format) - chunks = self.smart_chunk_text(text_content, self.chunk_size, self.stride) + chunks = self.smart_chunk_text(text_content, self.chunk_size, self.stride, return_tokenized) all_chunks.extend(chunks) return self.create_causal_dataset(all_chunks) - def chunk_text(self, text): + def chunk_text(self, text, return_tokenized=None): """Split text into overlapping chunks""" - return self.smart_chunk_text(text, self.chunk_size, self.stride) + if return_tokenized is None: + return_tokenized = self.return_tokenized + return self.smart_chunk_text(text, self.chunk_size, self.stride, return_tokenized) def create_causal_dataset(self, chunks): """Create dataset for causal language modeling""" - return Dataset.from_dict({"text": chunks}) + if chunks and isinstance(chunks[0], dict): + # If chunks are already tokenized (dict with input_ids, attention_mask) + # Reorganize the data structure for Dataset.from_dict + input_ids = [chunk["input_ids"] for chunk in chunks] + attention_mask = [chunk["attention_mask"] for chunk in chunks] + return Dataset.from_dict({ + "input_ids": input_ids, + "attention_mask": attention_mask + }) + else: + # If chunks are text strings (backward compatibility) + return Dataset.from_dict({"text": chunks}) - def smart_chunk_text(self, text, chunk_size, stride): + def smart_chunk_text(self, text, chunk_size, stride, return_tokenized=True): """ Intelligent chunking that: 1. Respects sentence/paragraph boundaries 2. Handles various text formats (.txt, .md, .json, etc.) 3. Maintains context with stride overlap - 4. Adds proper EOS tokens + 4. Returns tokenized chunks directly (more efficient) or text chunks """ # First pass: tokenize the entire text to get accurate token counts tokenized = self.tokenizer(text, return_tensors = "pt", add_special_tokens = False) @@ -93,8 +111,19 @@ class RawTextDataLoader: if len(tokens) <= chunk_size: # Text is small enough to fit in one chunk - eos_token = self.tokenizer.eos_token if self.tokenizer.eos_token else "" - return [text + eos_token] + if return_tokenized: + # Add EOS token to the tokens if available + eos_token_id = getattr(self.tokenizer, 'eos_token_id', None) + if eos_token_id is not None: + tokens = tokens.tolist() if hasattr(tokens, 'tolist') else list(tokens) + tokens.append(eos_token_id) + + # Create attention mask + attention_mask = [1] * len(tokens) + return [{"input_ids": tokens, "attention_mask": attention_mask}] + else: + eos_token = self.tokenizer.eos_token if self.tokenizer.eos_token else "" + return [text + eos_token] chunks = [] start_idx = 0 @@ -106,15 +135,33 @@ class RawTextDataLoader: # Extract tokens for this chunk chunk_tokens = tokens[start_idx:end_idx] - # Decode back to text - chunk_text = self.tokenizer.decode(chunk_tokens, skip_special_tokens = True) + if return_tokenized: + # Convert to list if it's a tensor + chunk_tokens_list = chunk_tokens.tolist() if hasattr(chunk_tokens, 'tolist') else list(chunk_tokens) + + # Add EOS token if it's the last chunk or chunk is complete + if end_idx == len(tokens) or len(chunk_tokens_list) == chunk_size: + eos_token_id = getattr(self.tokenizer, 'eos_token_id', None) + if eos_token_id is not None: + chunk_tokens_list.append(eos_token_id) - # Add EOS token if it's the last chunk or chunk is complete - if end_idx == len(tokens) or len(chunk_tokens) == chunk_size: - eos_token = self.tokenizer.eos_token if self.tokenizer.eos_token else "" - chunk_text += eos_token + # Create attention mask (all tokens are attended to) + attention_mask = [1] * len(chunk_tokens_list) + + chunks.append({ + "input_ids": chunk_tokens_list, + "attention_mask": attention_mask + }) + else: + # Decode back to text (backward compatibility) + chunk_text = self.tokenizer.decode(chunk_tokens, skip_special_tokens = True) - chunks.append(chunk_text) + # Add EOS token if it's the last chunk or chunk is complete + if end_idx == len(tokens) or len(chunk_tokens) == chunk_size: + eos_token = self.tokenizer.eos_token if self.tokenizer.eos_token else "" + chunk_text += eos_token + + chunks.append(chunk_text) # Move to next chunk with stride overlap if end_idx == len(tokens): From 2ee16e7ee25be4313c2ba573e93fcdf889502376 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 20 Nov 2025 13:09:07 +0000 Subject: [PATCH 013/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_raw_text.py | 24 +++++++++----- unsloth-cli.py | 4 +-- unsloth/dataprep/raw_text.py | 62 ++++++++++++++++++++++-------------- 3 files changed, 56 insertions(+), 34 deletions(-) diff --git a/tests/test_raw_text.py b/tests/test_raw_text.py index 88dac2604d..5306c68fa6 100644 --- a/tests/test_raw_text.py +++ b/tests/test_raw_text.py @@ -78,7 +78,7 @@ def test_raw_text_loader(): def __len__(self): return len(self.data) - + def tolist(self): return self.data @@ -100,21 +100,29 @@ def test_raw_text_loader(): loader = RawTextDataLoader(tokenizer, chunk_size = 5, stride = 2) # Test loading with text output (legacy mode) - text_dataset = loader.load_from_file(test_file, return_tensors=False) + text_dataset = loader.load_from_file(test_file, return_tensors = False) assert len(text_dataset) > 0, "Should create at least one chunk" assert "text" in text_dataset.column_names, "Dataset should have 'text' column" # Test loading with tokenized output (new efficient mode) - tokenized_dataset = loader.load_from_file(test_file, return_tensors=True) + tokenized_dataset = loader.load_from_file(test_file, return_tensors = True) assert len(tokenized_dataset) > 0, "Should create at least one tokenized chunk" - assert "input_ids" in tokenized_dataset.column_names, "Dataset should have 'input_ids' column" - assert "attention_mask" in tokenized_dataset.column_names, "Dataset should have 'attention_mask' column" - + assert ( + "input_ids" in tokenized_dataset.column_names + ), "Dataset should have 'input_ids' column" + assert ( + "attention_mask" in tokenized_dataset.column_names + ), "Dataset should have 'attention_mask' column" + # Verify tokenized data structure first_sample = tokenized_dataset[0] assert isinstance(first_sample["input_ids"], list), "input_ids should be a list" - assert isinstance(first_sample["attention_mask"], list), "attention_mask should be a list" - assert len(first_sample["input_ids"]) == len(first_sample["attention_mask"]), "input_ids and attention_mask should have same length" + assert isinstance( + first_sample["attention_mask"], list + ), "attention_mask should be a list" + assert len(first_sample["input_ids"]) == len( + first_sample["attention_mask"] + ), "input_ids and attention_mask should have same length" # Test preprocessor preprocessor = TextPreprocessor() diff --git a/unsloth-cli.py b/unsloth-cli.py index efdc0b3f4d..5135d00493 100644 --- a/unsloth-cli.py +++ b/unsloth-cli.py @@ -105,14 +105,14 @@ def run(args): if args.raw_text_file: # Use raw text loader - returns pre-tokenized data loader = RawTextDataLoader(tokenizer, args.chunk_size, args.stride) - dataset = loader.load_from_file(args.raw_text_file, return_tensors=True) + dataset = loader.load_from_file(args.raw_text_file, return_tensors = True) # Mark dataset as pre-tokenized to skip text formatting dataset._is_pretokenized = True return dataset elif args.dataset.endswith((".txt", ".md", ".json", ".jsonl")): # Auto-detect local raw text files - returns pre-tokenized data loader = RawTextDataLoader(tokenizer, args.chunk_size, args.stride) - dataset = loader.load_from_file(args.dataset, return_tensors=True) + dataset = loader.load_from_file(args.dataset, return_tensors = True) # Mark dataset as pre-tokenized to skip text formatting dataset._is_pretokenized = True return dataset diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py index 50612c1b86..d6a9a6a047 100644 --- a/unsloth/dataprep/raw_text.py +++ b/unsloth/dataprep/raw_text.py @@ -46,16 +46,18 @@ class RawTextDataLoader: extension = Path(file_path).suffix.lower() return SUPPORTED_FORMATS.get(extension, "plain_text") - def load_from_file(self, file_path, return_tokenized=None): + def load_from_file(self, file_path, return_tokenized = None): """Load raw text and convert to dataset""" if return_tokenized is None: return_tokenized = self.return_tokenized file_format = self.detect_format(file_path) text_content = self._read_file_by_format(file_path, file_format) - chunks = self.smart_chunk_text(text_content, self.chunk_size, self.stride, return_tokenized) + chunks = self.smart_chunk_text( + text_content, self.chunk_size, self.stride, return_tokenized + ) return self.create_causal_dataset(chunks) - def load_from_files(self, file_paths, return_tokenized=None): + def load_from_files(self, file_paths, return_tokenized = None): """Load multiple text files""" if return_tokenized is None: return_tokenized = self.return_tokenized @@ -63,15 +65,19 @@ class RawTextDataLoader: for file_path in file_paths: file_format = self.detect_format(file_path) text_content = self._read_file_by_format(file_path, file_format) - chunks = self.smart_chunk_text(text_content, self.chunk_size, self.stride, return_tokenized) + chunks = self.smart_chunk_text( + text_content, self.chunk_size, self.stride, return_tokenized + ) all_chunks.extend(chunks) return self.create_causal_dataset(all_chunks) - def chunk_text(self, text, return_tokenized=None): + def chunk_text(self, text, return_tokenized = None): """Split text into overlapping chunks""" if return_tokenized is None: return_tokenized = self.return_tokenized - return self.smart_chunk_text(text, self.chunk_size, self.stride, return_tokenized) + return self.smart_chunk_text( + text, self.chunk_size, self.stride, return_tokenized + ) def create_causal_dataset(self, chunks): """Create dataset for causal language modeling""" @@ -80,15 +86,14 @@ class RawTextDataLoader: # Reorganize the data structure for Dataset.from_dict input_ids = [chunk["input_ids"] for chunk in chunks] attention_mask = [chunk["attention_mask"] for chunk in chunks] - return Dataset.from_dict({ - "input_ids": input_ids, - "attention_mask": attention_mask - }) + return Dataset.from_dict( + {"input_ids": input_ids, "attention_mask": attention_mask} + ) else: # If chunks are text strings (backward compatibility) return Dataset.from_dict({"text": chunks}) - def smart_chunk_text(self, text, chunk_size, stride, return_tokenized=True): + def smart_chunk_text(self, text, chunk_size, stride, return_tokenized = True): """ Intelligent chunking that: 1. Respects sentence/paragraph boundaries @@ -113,11 +118,13 @@ class RawTextDataLoader: # Text is small enough to fit in one chunk if return_tokenized: # Add EOS token to the tokens if available - eos_token_id = getattr(self.tokenizer, 'eos_token_id', None) + eos_token_id = getattr(self.tokenizer, "eos_token_id", None) if eos_token_id is not None: - tokens = tokens.tolist() if hasattr(tokens, 'tolist') else list(tokens) + tokens = ( + tokens.tolist() if hasattr(tokens, "tolist") else list(tokens) + ) tokens.append(eos_token_id) - + # Create attention mask attention_mask = [1] * len(tokens) return [{"input_ids": tokens, "attention_mask": attention_mask}] @@ -137,28 +144,35 @@ class RawTextDataLoader: if return_tokenized: # Convert to list if it's a tensor - chunk_tokens_list = chunk_tokens.tolist() if hasattr(chunk_tokens, 'tolist') else list(chunk_tokens) - + chunk_tokens_list = ( + chunk_tokens.tolist() + if hasattr(chunk_tokens, "tolist") + else list(chunk_tokens) + ) + # Add EOS token if it's the last chunk or chunk is complete if end_idx == len(tokens) or len(chunk_tokens_list) == chunk_size: - eos_token_id = getattr(self.tokenizer, 'eos_token_id', None) + eos_token_id = getattr(self.tokenizer, "eos_token_id", None) if eos_token_id is not None: chunk_tokens_list.append(eos_token_id) # Create attention mask (all tokens are attended to) attention_mask = [1] * len(chunk_tokens_list) - - chunks.append({ - "input_ids": chunk_tokens_list, - "attention_mask": attention_mask - }) + + chunks.append( + {"input_ids": chunk_tokens_list, "attention_mask": attention_mask} + ) else: # Decode back to text (backward compatibility) - chunk_text = self.tokenizer.decode(chunk_tokens, skip_special_tokens = True) + chunk_text = self.tokenizer.decode( + chunk_tokens, skip_special_tokens = True + ) # Add EOS token if it's the last chunk or chunk is complete if end_idx == len(tokens) or len(chunk_tokens) == chunk_size: - eos_token = self.tokenizer.eos_token if self.tokenizer.eos_token else "" + eos_token = ( + self.tokenizer.eos_token if self.tokenizer.eos_token else "" + ) chunk_text += eos_token chunks.append(chunk_text) From 4596b67dc9c12745f6342e5a262b6eab321d4a90 Mon Sep 17 00:00:00 2001 From: vangmay Date: Thu, 20 Nov 2025 21:40:45 +0800 Subject: [PATCH 014/114] remove old function --- unsloth/dataprep/raw_text.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py index d6a9a6a047..b880e97338 100644 --- a/unsloth/dataprep/raw_text.py +++ b/unsloth/dataprep/raw_text.py @@ -184,14 +184,6 @@ class RawTextDataLoader: return chunks - def tokenize_and_chunk(self, text): - """ - Tokenize first, then chunk by token count: - 1. More precise length control - 2. Avoids mid-token splits - 3. Handles different languages better - """ - def _read_file_by_format(self, file_path, file_format): """Read file content based on detected format.""" with open(file_path, "r", encoding = "utf-8") as f: From 979b506069e43495b3d61a65820f41a3b433c2ea Mon Sep 17 00:00:00 2001 From: vangmay Date: Tue, 25 Nov 2025 21:01:43 +0800 Subject: [PATCH 015/114] Remove training mode arg --- unsloth-cli.py | 34 +++++++--------------------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/unsloth-cli.py b/unsloth-cli.py index 5135d00493..ef8167ad39 100644 --- a/unsloth-cli.py +++ b/unsloth-cli.py @@ -103,19 +103,13 @@ def run(args): from transformers.utils import strtobool if args.raw_text_file: - # Use raw text loader - returns pre-tokenized data + # Use raw text loader loader = RawTextDataLoader(tokenizer, args.chunk_size, args.stride) - dataset = loader.load_from_file(args.raw_text_file, return_tensors = True) - # Mark dataset as pre-tokenized to skip text formatting - dataset._is_pretokenized = True - return dataset + dataset = loader.load_from_file(args.raw_text_file) elif args.dataset.endswith((".txt", ".md", ".json", ".jsonl")): - # Auto-detect local raw text files - returns pre-tokenized data - loader = RawTextDataLoader(tokenizer, args.chunk_size, args.stride) - dataset = loader.load_from_file(args.dataset, return_tensors = True) - # Mark dataset as pre-tokenized to skip text formatting - dataset._is_pretokenized = True - return dataset + # Auto-detect local raw text files + loader = RawTextDataLoader(tokenizer) + dataset = loader.load_from_file(args.dataset) else: # Check for modelscope usage use_modelscope = strtobool( @@ -129,9 +123,9 @@ def run(args): # Existing HuggingFace dataset logic dataset = load_dataset(args.dataset, split = "train") - # Apply formatting for structured datasets (text-based) + # Apply formatting for structured datasets dataset = dataset.map(formatting_prompts_func, batched = True) - return dataset + return dataset # Load dataset using smart loader dataset = load_dataset_smart(args) @@ -427,19 +421,5 @@ if __name__ == "__main__": "--stride", type = int, default = 512, help = "Overlap between chunks" ) - TRAINING_MODES = { - "instruction": "Standard instruction-following", - "causal": "Causal language modeling (raw text)", - "completion": "Text completion tasks", - } - - parser.add_argument( - "--training_mode", - type = str, - default = "instruction", - choices = list(TRAINING_MODES.keys()), - help = "Training mode for the model", - ) - args = parser.parse_args() run(args) From aa36dabd81dcb6abac9620eaaee8e9bc672ce81c Mon Sep 17 00:00:00 2001 From: vangmay Date: Wed, 10 Dec 2025 10:15:56 +0530 Subject: [PATCH 016/114] Fix RawTextDataLoader import issue --- unsloth/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unsloth/__init__.py b/unsloth/__init__.py index 8b48ce3ba0..b77af7c242 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -247,6 +247,8 @@ from .save import * from .chat_templates import * from .tokenizer_utils import * from .trainer import * +# Export dataprep utilities for CLI and downstream users +from .dataprep.raw_text import RawTextDataLoader, TextPreprocessor from unsloth_zoo.rl_environments import ( check_python_modules, create_locked_down_function, From 87b924b28ea0190195f2617ec7c500694cad1255 Mon Sep 17 00:00:00 2001 From: vangmay Date: Wed, 10 Dec 2025 10:17:23 +0530 Subject: [PATCH 017/114] Fix Incorrect non-relative import in dataprep package --- unsloth/dataprep/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/dataprep/__init__.py b/unsloth/dataprep/__init__.py index b6840f247f..048f9b8010 100644 --- a/unsloth/dataprep/__init__.py +++ b/unsloth/dataprep/__init__.py @@ -13,4 +13,4 @@ # limitations under the License. from .synthetic import * -from raw_text import * +from .raw_text import * From 3fb6335f0259fa846330f0a25262bca4a6943422 Mon Sep 17 00:00:00 2001 From: vangmay Date: Wed, 10 Dec 2025 10:46:29 +0530 Subject: [PATCH 018/114] =?UTF-8?q?Fix=20Chunking=20loop=20can=20hang=20wh?= =?UTF-8?q?en=20stride=20=E2=89=A5=20chunk=5Fsize?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- unsloth/dataprep/raw_text.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py index b880e97338..f7c1bf7856 100644 --- a/unsloth/dataprep/raw_text.py +++ b/unsloth/dataprep/raw_text.py @@ -101,6 +101,13 @@ class RawTextDataLoader: 3. Maintains context with stride overlap 4. Returns tokenized chunks directly (more efficient) or text chunks """ + if chunk_size <= 0: + raise ValueError(f"chunk_size must be positive, got {chunk_size}") + if stride >= chunk_size: + raise ValueError( + f"stride ({stride}) must be smaller than chunk_size ({chunk_size}) to progress the chunking loop" + ) + # First pass: tokenize the entire text to get accurate token counts tokenized = self.tokenizer(text, return_tensors = "pt", add_special_tokens = False) tokens = tokenized["input_ids"] From c1086e3ed63a2519b3181b01985a3b02b5659a32 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Dec 2025 05:17:01 +0000 Subject: [PATCH 019/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unsloth/__init__.py b/unsloth/__init__.py index b77af7c242..26e43eec80 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -247,6 +247,7 @@ from .save import * from .chat_templates import * from .tokenizer_utils import * from .trainer import * + # Export dataprep utilities for CLI and downstream users from .dataprep.raw_text import RawTextDataLoader, TextPreprocessor from unsloth_zoo.rl_environments import ( From a86363eca972118e2c6c4bb91c42810851fc72d6 Mon Sep 17 00:00:00 2001 From: oKatanaaa Date: Thu, 11 Dec 2025 03:21:02 +0000 Subject: [PATCH 020/114] fix: weights tying --- unsloth/models/llama.py | 48 ++++++++++++++++++++++++++++++++++++++++ unsloth/models/vision.py | 1 + 2 files changed, 49 insertions(+) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 4c9337ccf9..d38018ee1b 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -2601,6 +2601,7 @@ class FastLlamaModel: loftq_config = {}, temporary_location = "_unsloth_temporary_saved_buffers", qat_scheme = None, + ensure_weight_tying = False, **kwargs, ): if os.environ.get("UNSLOTH_USE_NEW_MODEL", "0") == "1": @@ -2630,6 +2631,7 @@ class FastLlamaModel: init_lora_weights = init_lora_weights, loftq_config = loftq_config, temporary_location = temporary_location, + ensure_weight_tying = ensure_weight_tying, **kwargs, ) if os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1": @@ -2953,6 +2955,7 @@ class FastLlamaModel: loftq_config = loftq_config, use_rslora = use_rslora, modules_to_save = modules_to_save, + ensure_weight_tying = ensure_weight_tying, **kwargs, ) if not SUPPORTS_LOFTQ: @@ -3002,6 +3005,51 @@ class FastLlamaModel: model = FastLlamaModel.patch_peft_model(model, use_gradient_checkpointing) + if ensure_weight_tying: + try: + input_embeddings = model.get_input_embeddings() + output_embeddings = model.get_output_embeddings() + + if input_embeddings is not None and output_embeddings is not None: + def _retie_parameter(target_module, source_module): + if not hasattr(source_module, "weight"): + return + weight = source_module.weight + # Remove existing registration to avoid "attribute already exists" + if "weight" in getattr(target_module, "_parameters", {}): + target_module._parameters.pop("weight") + if hasattr(target_module, "weight"): + try: + delattr(target_module, "weight") + except Exception: + pass + target_module.register_parameter("weight", weight) + + # Tie trainable copies created by ModulesToSaveWrapper first (these are used in forward) + if hasattr(input_embeddings, "modules_to_save") and hasattr( + output_embeddings, "modules_to_save" + ): + if hasattr(input_embeddings.modules_to_save, "default") and hasattr( + output_embeddings.modules_to_save, "default" + ): + _retie_parameter( + output_embeddings.modules_to_save.default, + input_embeddings.modules_to_save.default, + ) + + # Tie original_module references as well if present + if hasattr(input_embeddings, "original_module") and hasattr( + output_embeddings, "original_module" + ): + _retie_parameter( + output_embeddings.original_module, + input_embeddings.original_module, + ) + except Exception as e: + logger.warning_once( + f"Unsloth: Failed to ensure weight tying between embeddings and lm_head: {e}" + ) + if train_embed_tokens: print("Unsloth: Training embed_tokens in mixed precision to save VRAM") assert hasattr(model.get_input_embeddings(), "modules_to_save") diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index ed19f587cf..9f847f2837 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -930,6 +930,7 @@ class FastBaseModel: task_type = TaskType.CAUSAL_LM, temporary_location = "_unsloth_temporary_saved_buffers", qat_scheme = None, + ensure_weight_tying = False, **kwargs, ): if os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1": From 1837de275165b5307b057036c420f2778c6d1343 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Dec 2025 03:31:41 +0000 Subject: [PATCH 021/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/llama.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index d38018ee1b..e0d8cbcf25 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -3011,6 +3011,7 @@ class FastLlamaModel: output_embeddings = model.get_output_embeddings() if input_embeddings is not None and output_embeddings is not None: + def _retie_parameter(target_module, source_module): if not hasattr(source_module, "weight"): return @@ -3029,9 +3030,9 @@ class FastLlamaModel: if hasattr(input_embeddings, "modules_to_save") and hasattr( output_embeddings, "modules_to_save" ): - if hasattr(input_embeddings.modules_to_save, "default") and hasattr( - output_embeddings.modules_to_save, "default" - ): + if hasattr( + input_embeddings.modules_to_save, "default" + ) and hasattr(output_embeddings.modules_to_save, "default"): _retie_parameter( output_embeddings.modules_to_save.default, input_embeddings.modules_to_save.default, From 7403104b0c05c0794bd8f74342624a22c930a535 Mon Sep 17 00:00:00 2001 From: oKatanaaa Date: Sat, 13 Dec 2025 00:02:48 +0000 Subject: [PATCH 022/114] fix: add a log instead of silent exception --- unsloth/models/llama.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index e0d8cbcf25..6e47907166 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -3022,8 +3022,11 @@ class FastLlamaModel: if hasattr(target_module, "weight"): try: delattr(target_module, "weight") - except Exception: - pass + except Exception as exc: + logger.warning_once( + f"Unsloth: Could not delete existing weight attr during retie on " + f"{type(target_module).__name__}: {exc}" + ) target_module.register_parameter("weight", weight) # Tie trainable copies created by ModulesToSaveWrapper first (these are used in forward) From a94391d9660727a8c8875193b856390c7bdd8bd4 Mon Sep 17 00:00:00 2001 From: "abhishek.sharma" Date: Sat, 20 Dec 2025 11:47:03 +0530 Subject: [PATCH 023/114] Fix model training state restoration in GRPO trainer Store the model's training state before generation and restore inference mode after completion if the model wasn't originally in training mode. This ensures the model returns to the correct state after generate and score operations. --- unsloth/models/rl_replacements.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 7d4d520c1f..dd139ffd25 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -259,6 +259,7 @@ def grpo_trainer__generate_and_score_completions(function_name, function): # The new multi-line string that will replace the line above replacement_lines = """ batch_size = self.args.per_device_train_batch_size if mode == "train" else self.args.per_device_eval_batch_size + _was_training = self.model.training try: # TRL 0.23.1 and below path if not has_images: @@ -387,6 +388,13 @@ def grpo_trainer__generate_and_score_completions(function_name, function): patched = patched[: match.start()] + wrapped + patched[match.end() :] function = patched + + function = function.replace( + " return output", # 8 spaces before 'return' + """ if not _was_training: + self.model.for_inference() + return output""" + ) return function From bef0371cc668f48c0cbc8dd3a629b05dae6754d5 Mon Sep 17 00:00:00 2001 From: numb3r33 Date: Sat, 20 Dec 2025 12:30:33 +0530 Subject: [PATCH 024/114] Remove the comment. --- unsloth/models/rl_replacements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index dd139ffd25..248c5aab85 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -390,7 +390,7 @@ def grpo_trainer__generate_and_score_completions(function_name, function): function = patched function = function.replace( - " return output", # 8 spaces before 'return' + " return output", """ if not _was_training: self.model.for_inference() return output""" From 7620e75c3964ca60a7a2174bc9b42314d61f9b42 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 20 Dec 2025 07:02:30 +0000 Subject: [PATCH 025/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/rl_replacements.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 248c5aab85..e13e5e6d78 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -388,12 +388,12 @@ def grpo_trainer__generate_and_score_completions(function_name, function): patched = patched[: match.start()] + wrapped + patched[match.end() :] function = patched - + function = function.replace( - " return output", - """ if not _was_training: + " return output", + """ if not _was_training: self.model.for_inference() - return output""" + return output""", ) return function From 89f2d3a28b0744edc5476eb2341ee91bd4621a81 Mon Sep 17 00:00:00 2001 From: numb3r33 Date: Wed, 24 Dec 2025 00:14:50 +0530 Subject: [PATCH 026/114] Fix indentation handling in grpo_trainer return statement replacement Use regex to dynamically detect and preserve the original indentation when replacing the 'return output' statement, instead of hardcoding spaces. This ensures the patched code maintains consistent indentation regardless of the original formatting. --- unsloth/models/rl_replacements.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index e13e5e6d78..bcac699b3f 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -388,13 +388,17 @@ def grpo_trainer__generate_and_score_completions(function_name, function): patched = patched[: match.start()] + wrapped + patched[match.end() :] function = patched + + match = re.search(r'^(\s*)return output', function, re.MULTILINE) - function = function.replace( - " return output", - """ if not _was_training: - self.model.for_inference() - return output""", - ) + if match: + indent = match.group(1) + function = function.replace( + f"{indent}return output", + f"""{indent}if not _was_training: + {indent} self.model.for_inference() + {indent}return output""" + ) return function From ce7251458ecb1c966b3ce1ac9f3b4a3bba0ce7b7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 23 Dec 2025 19:20:08 +0000 Subject: [PATCH 027/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/rl_replacements.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index bcac699b3f..5158019132 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -388,8 +388,8 @@ def grpo_trainer__generate_and_score_completions(function_name, function): patched = patched[: match.start()] + wrapped + patched[match.end() :] function = patched - - match = re.search(r'^(\s*)return output', function, re.MULTILINE) + + match = re.search(r"^(\s*)return output", function, re.MULTILINE) if match: indent = match.group(1) @@ -397,7 +397,7 @@ def grpo_trainer__generate_and_score_completions(function_name, function): f"{indent}return output", f"""{indent}if not _was_training: {indent} self.model.for_inference() - {indent}return output""" + {indent}return output""", ) return function From 880fc3d1756a6c98db19232142e6e087febed599 Mon Sep 17 00:00:00 2001 From: numb3r33 Date: Wed, 24 Dec 2025 01:02:03 +0530 Subject: [PATCH 028/114] Refactor return statement replacement to use explicit newlines Replace f-string triple-quoted approach with explicit newline characters for clearer string construction in the grpo_trainer patch. --- unsloth/models/rl_replacements.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 5158019132..8436ca0dc9 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -393,12 +393,8 @@ def grpo_trainer__generate_and_score_completions(function_name, function): if match: indent = match.group(1) - function = function.replace( - f"{indent}return output", - f"""{indent}if not _was_training: - {indent} self.model.for_inference() - {indent}return output""", - ) + new_code = indent + "if not _was_training:\n" + indent + " self.model.for_inference()\n" + indent + "return output" + function = function.replace(f"{indent}return output", new_code) return function From e7d04737dd3732c7c545147fddce6ed025bfa5a3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 23 Dec 2025 20:06:36 +0000 Subject: [PATCH 029/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/rl_replacements.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 8436ca0dc9..f0f0386bd1 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -393,7 +393,14 @@ def grpo_trainer__generate_and_score_completions(function_name, function): if match: indent = match.group(1) - new_code = indent + "if not _was_training:\n" + indent + " self.model.for_inference()\n" + indent + "return output" + new_code = ( + indent + + "if not _was_training:\n" + + indent + + " self.model.for_inference()\n" + + indent + + "return output" + ) function = function.replace(f"{indent}return output", new_code) return function From 08f1716a70ae932f8299421d020fa44fa2de6f2e Mon Sep 17 00:00:00 2001 From: Strahinja Stamenkovic Date: Fri, 26 Dec 2025 03:43:59 +0100 Subject: [PATCH 030/114] Add missing import of inspect (#3778) * Add missing import of inspect * Update device_type.py --- unsloth/device_type.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unsloth/device_type.py b/unsloth/device_type.py index 68038de679..0f924bfdfd 100644 --- a/unsloth/device_type.py +++ b/unsloth/device_type.py @@ -24,6 +24,7 @@ __all__ = [ import torch import functools +import inspect from unsloth_zoo.utils import Version From 181b76420efde3a0a0e0a4e5f6dd598523027a2e Mon Sep 17 00:00:00 2001 From: Fizza Mukhtar Date: Thu, 25 Dec 2025 18:46:13 -0800 Subject: [PATCH 031/114] Clarify NotImplementedError for fast_inference with full_finetuning (#3768) * Improve error message for fast_inference and full_finetuning * Refine error message string formatting * Update unsloth/models/vision.py --------- Co-authored-by: Daniel Han --- unsloth/models/vision.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index b78b190bcb..e1cf8f6f82 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -718,9 +718,13 @@ class FastBaseModel: if full_finetuning: max_lora_rank = max(get_lora_supported_ranks()) raise NotImplementedError( - f"Unsloth: `fast_inference = True` does not yet support `full_finetuning = True`.\n" - f"Use LoRA rank `r = {max_lora_rank}` as the closest replacement for full finetuning with Unsloth for RL." + "Unsloth: `fast_inference=True` cannot be used together with `full_finetuning=True`.\n" + "Reason: fast_inference is optimized for inference-only workflows and " + "does not currently support full fine-tuning.\n" + "Workaround: disable fast_inference, or use parameter-efficient fine-tuning " + f"(e.g. LoRA with rank r={max_lora_rank})." ) + model_config.model_name = model_name if fast_inference: From b314dca22dc5bedc80235d2c712d2cbfed2add89 Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Sat, 27 Dec 2025 00:49:19 -0800 Subject: [PATCH 032/114] Update README for new unsloth.ai/docs.md --- README.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 43c09381fc..7cd9d0bba4 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ - + ### Train gpt-oss, DeepSeek, Gemma, Qwen & Llama 2x faster with 70% less VRAM! @@ -18,7 +18,7 @@ ## ✨ Train for Free -Notebooks are beginner friendly. Read our [guide](https://docs.unsloth.ai/get-started/fine-tuning-guide). Add dataset, run, then export your trained model to GGUF, llama.cpp, Ollama, vLLM, SGLang or Hugging Face. +Notebooks are beginner friendly. Read our [guide](https://docs.unsloth.ai/get-started/fine-tuning-guide). Add dataset, run, then deploy your trained model. | Model | Free Notebooks | Performance | Memory use | |-----------|---------|--------|----------| @@ -34,9 +34,9 @@ Notebooks are beginner friendly. Read our [guide](https://docs.unsloth.ai/get-st | **Llama 3.2 Conversational** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3.2_(1B_and_3B)-Conversational.ipynb) | 2x faster | 70% less | | **Orpheus-TTS (3B)** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Orpheus_(3B)-TTS.ipynb) | 1.5x faster | 50% less | -- See all our notebooks for: [Kaggle](https://github.com/unslothai/notebooks?tab=readme-ov-file#-kaggle-notebooks), [GRPO](https://docs.unsloth.ai/get-started/unsloth-notebooks#grpo-reasoning-rl-notebooks), **[TTS](https://docs.unsloth.ai/get-started/unsloth-notebooks#text-to-speech-tts-notebooks)** & [Vision](https://docs.unsloth.ai/get-started/unsloth-notebooks#vision-multimodal-notebooks) -- See [all our models](https://docs.unsloth.ai/get-started/all-our-models) and [all our notebooks](https://docs.unsloth.ai/get-started/unsloth-notebooks) -- See detailed documentation for Unsloth [here](https://docs.unsloth.ai/) +- See all our notebooks for: [Kaggle](https://github.com/unslothai/notebooks?tab=readme-ov-file#-kaggle-notebooks), [GRPO](https://unsloth.ai/docs/get-started/unsloth-notebooks#grpo-reasoning-rl-notebooks), [TTS](https://unsloth.ai/docs/get-started/unsloth-notebooks#text-to-speech-tts-notebooks) & [Vision](https://unsloth.ai/docs/get-started/unsloth-notebooks#vision-multimodal-notebooks) +- See [all our models](https://unsloth.ai/docs/get-started/unsloth-model-catalog) and [all our notebooks](https://unsloth.ai/docs/get-started/unsloth-notebooks) +- See detailed documentation for Unsloth [here](https://unsloth.ai/docs) ## ⚡ Quickstart ### Linux or WSL @@ -46,9 +46,9 @@ pip install unsloth ### Windows For Windows, `pip install unsloth` works only if you have Pytorch installed. Read our [Windows Guide](https://docs.unsloth.ai/get-started/installing-+-updating/windows-installation). ### Docker -Use our official [Unsloth Docker image](https://hub.docker.com/r/unsloth/unsloth) ```unsloth/unsloth``` container. Read our [Docker Guide](https://docs.unsloth.ai/get-started/install-and-update/docker). +Use our official [Unsloth Docker image](https://hub.docker.com/r/unsloth/unsloth) ```unsloth/unsloth``` container. Read our [Docker Guide](https://unsloth.ai/docs/get-started/install-and-update/docker). ### Blackwell & DGX Spark -For RTX 50x, B200, 6000 GPUs: `pip install unsloth`. Read our [Blackwell Guide](https://docs.unsloth.ai/basics/training-llms-with-blackwell-rtx-50-series-and-unsloth) and [DGX Spark Guide](https://docs.unsloth.ai/new/fine-tuning-llms-with-nvidia-dgx-spark-and-unsloth) for more details. +For RTX 50x, B200, 6000 GPUs: `pip install unsloth`. Read our [Blackwell Guide](https://unsloth.ai/docs/basics/fine-tuning-llms-with-blackwell-rtx-50-series-and-unsloth) and [DGX Spark Guide](https://unsloth.ai/docs/basics/fine-tuning-llms-with-nvidia-dgx-spark-and-unsloth) for more details. ## 🦥 Unsloth News - New RoPE & MLP **Triton Kernels** & **Padding Free + Packing**: 3x faster training & 30% less VRAM. [Blog](https://docs.unsloth.ai/new/3x-faster-training-packing) @@ -98,6 +98,7 @@ For RTX 50x, B200, 6000 GPUs: `pip install unsloth`. Read our [Blackwell Guide]( - Supports **all models** including [TTS](https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning), multimodal, [BERT](https://docs.unsloth.ai/get-started/unsloth-notebooks#other-important-notebooks) and more! Any model that works in transformers, works in Unsloth. - The most efficient library for [Reinforcement Learning (RL)](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide), using 80% less VRAM. Supports GRPO, GSPO, DrGRPO, DAPO etc. - **0% loss in accuracy** - no approximation methods - all exact. +- Export and [deploy your model](https://unsloth.ai/docs/basics/inference-and-deployment) to GGUF, llama.cpp, vLLM, SGLang and Hugging Face. - Supports NVIDIA (since 2018), [AMD](https://docs.unsloth.ai/get-started/install-and-update/amd) and Intel GPUs. Minimum CUDA Capability 7.0 (V100, T4, Titan V, RTX 20, 30, 40x, A100, H100, L40 etc) - Works on **Linux**, WSL and **Windows** - All kernels written in [OpenAI's Triton](https://openai.com/index/triton/) language. Manual backprop engine. @@ -283,7 +284,7 @@ docker run -d -e JUPYTER_PASSWORD="mypassword" \ Access Jupyter Lab at `http://localhost:8888` and start fine-tuning! ## 📜 Documentation -- Go to our official [Documentation](https://docs.unsloth.ai) for [running models](https://docs.unsloth.ai/basics/running-and-saving-models), [saving to GGUF](https://docs.unsloth.ai/basics/running-and-saving-models/saving-to-gguf), [checkpointing](https://docs.unsloth.ai/basics/finetuning-from-last-checkpoint), [evaluation](https://docs.unsloth.ai/get-started/fine-tuning-llms-guide#evaluation) and more! +- Go to our official [Documentation](https://unsloth.ai/docs) for [running models](https://docs.unsloth.ai/basics/running-and-saving-models), [saving to GGUF](https://docs.unsloth.ai/basics/running-and-saving-models/saving-to-gguf), [checkpointing](https://docs.unsloth.ai/basics/finetuning-from-last-checkpoint), [evaluation](https://docs.unsloth.ai/get-started/fine-tuning-llms-guide#evaluation) and more! - Read our Guides for: [Fine-tuning](https://docs.unsloth.ai/get-started/fine-tuning-llms-guide), [Reinforcement Learning](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide), [Text-to-Speech (TTS)](https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning), [Vision](https://docs.unsloth.ai/basics/vision-fine-tuning) and [any model](https://docs.unsloth.ai/models/tutorials-how-to-fine-tune-and-run-llms). - We support Huggingface's transformers, TRL, Trainer, Seq2SeqTrainer and Pytorch code. From 58235ee1927d2dd448762ff9fd1010e991435370 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 28 Dec 2025 19:57:43 -0800 Subject: [PATCH 033/114] Update FUNDING.yml (#3792) --- .github/FUNDING.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 4ebb6df3d0..ae5dade42d 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -3,7 +3,7 @@ github: unslothai patreon: # Replace with a single Patreon username open_collective: # Replace with a single Open Collective username -ko_fi: unsloth +ko_fi: # unsloth tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry liberapay: # Replace with a single Liberapay username From c0c21a1e227823dac76e1cfc08856ac3def1015b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alk=C4=B1n=20=C3=9Cnl=C3=BC?= Date: Mon, 29 Dec 2025 08:18:02 +0300 Subject: [PATCH 034/114] fix(trainer): import psutil to prevent NameError in _prepare_dataset (#3780) * fix(trainer): import psutil to prevent NameError in _prepare_dataset Fixes #3777 * Update rl.py --------- Co-authored-by: Daniel Han --- unsloth/models/rl.py | 1 + unsloth/tokenizer_utils.py | 1 + unsloth/trainer.py | 1 + 3 files changed, 3 insertions(+) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 4ea36519d9..003a0e7f1b 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -227,6 +227,7 @@ import numpy as np from contextlib import nullcontext from torch.nn import functional as F import inspect +import psutil from transformers import DataCollatorForSeq2Seq, DataCollatorForLanguageModeling as TransformersDataCollatorForLanguageModeling from transformers.training_args import ParallelMode diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 99651643a8..0136e3498e 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -25,6 +25,7 @@ import collections import numpy as np import gc import subprocess +import psutil from unsloth_zoo.tokenizer_utils import ( mean_of_trained_tokens, diff --git a/unsloth/trainer.py b/unsloth/trainer.py index c0b2dd03b6..0d98cff305 100644 --- a/unsloth/trainer.py +++ b/unsloth/trainer.py @@ -14,6 +14,7 @@ import logging import os +import psutil import warnings from dataclasses import dataclass, field from typing import Optional From ab815692a9c80d9737e3b0d67927363e6da3b527 Mon Sep 17 00:00:00 2001 From: Francesco Bertolotti Date: Mon, 29 Dec 2025 06:21:48 +0100 Subject: [PATCH 035/114] fastrope fix for zero strided tensors (#3782) Co-authored-by: Francesco Bertolotti --- unsloth/kernels/rope_embedding.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.py index a032e0f7fc..fcc9cb923b 100644 --- a/unsloth/kernels/rope_embedding.py +++ b/unsloth/kernels/rope_embedding.py @@ -312,8 +312,8 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function): _, n_heads_K, _, _ = K.shape # Inplace rotary embedding is generally fine - Q_out = Q.clone() if not Q.is_contiguous else Q - K_out = K.clone() if not K.is_contiguous else K + Q_out = Q.clone() if not Q.is_contiguous() else Q + K_out = K.clone() if not K.is_contiguous() else K if has_indices: # TRL's rotary indices are always in int32, so casting is just for safety @@ -383,21 +383,21 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function): else ctx.cos.new_empty(1, dtype = torch.int32) ) + # Inplace rotary embedding is generally fine + dQ_out = dQ.clone() if not dQ.is_contiguous() else dQ + dK_out = dK.clone() if not dK.is_contiguous() else dK + Q_batch_stride, Q_head_stride, Q_seq_stride = ( - dQ.stride(0), - dQ.stride(1), - dQ.stride(2), + dQ_out.stride(0), + dQ_out.stride(1), + dQ_out.stride(2), ) K_batch_stride, K_head_stride, K_seq_stride = ( - dK.stride(0), - dK.stride(1), - dK.stride(2), + dK_out.stride(0), + dK_out.stride(1), + dK_out.stride(2), ) - # Inplace rotary embedding is generally fine - dQ_out = dQ.clone() if not dQ.is_contiguous else dQ - dK_out = dK.clone() if not dK.is_contiguous else dK - with torch_gpu_device(dQ.device): _rope_embedding_QK[(batch * ctx.seq_len, ctx.n_heads_Q)]( dQ_out, From c8b0bada94f55ab93848dbff28dcdec22b7cec31 Mon Sep 17 00:00:00 2001 From: Fizza Mukhtar Date: Sun, 28 Dec 2025 21:23:51 -0800 Subject: [PATCH 036/114] Fix crash when trl.experimental.openenv is unavailable (#3787) * Guard optional trl.experimental.openenv usage in RL patches * Simplify optional trl.openenv import handling * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- unsloth/models/rl_replacements.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 7d4d520c1f..3dfeea6871 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -949,11 +949,15 @@ def openenv_vllm_reload_weights(): return if Version(importlib_version("trl")) < Version("0.26.0"): return + try: import trl.experimental.openenv.utils as openenv_utils import trl.experimental.openenv as openenv except ImportError as e: logger.info(f"Unsloth: Failed to import trl openenv: {e}") + logger.info( + "Unsloth: trl.experimental.openenv not available — skipping RL openenv patches." + ) return src = inspect.getsource(openenv_utils.generate_rollout_completions) From fe82f5f3663eb75e106fa17f6bc65141265fd5cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=82=86=E3=82=8A?= Date: Mon, 29 Dec 2025 13:30:55 +0800 Subject: [PATCH 037/114] Fix Boolean value of Tensor ambiguity error in mistral.py (#3790) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix is_contiguous() method call and remove duplicate imports - Fix bug in rope_embedding.py where is_contiguous was used without parentheses, causing the method object (always truthy) to be evaluated instead of calling the method. This fixes issue #3781 where fast rope backpropagation was broken for zero strided/non-contiguous tensors. - Remove duplicate `import torch` in rl.py (lines 20 and 25) - Remove duplicate `import functools` and `import types` in vision.py 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 * Fix Boolean value of Tensor ambiguity error in mistral.py Replace `or` operator with explicit `is None` check when getting n_items from kwargs. The `or` operator fails when the value is a Tensor because Python cannot determine the boolean value of a multi-element tensor. Fixes #3766 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 * Update rope_embedding.py --------- Co-authored-by: yurekami Co-authored-by: Claude Opus 4.5 Co-authored-by: Daniel Han --- unsloth/models/mistral.py | 12 +++++++----- unsloth/models/rl.py | 1 - unsloth/models/vision.py | 2 -- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py index 0eed45c5cd..5e893d2b6f 100644 --- a/unsloth/models/mistral.py +++ b/unsloth/models/mistral.py @@ -307,9 +307,9 @@ def MistralForCausalLM_fast_forward( RETURN_LOGITS = False if not RETURN_LOGITS and labels is not None: - n_items = kwargs.get("num_items_in_batch", None) or kwargs.get( - "n_items", None - ) + n_items = kwargs.get("num_items_in_batch", None) + if n_items is None: + n_items = kwargs.get("n_items", None) logit_softcapping = getattr(self.config, "final_logit_softcapping", 0) # loss = fused_linear_cross_entropy( @@ -363,11 +363,13 @@ def MistralForCausalLM_fast_forward( shift_labels, kwargs.get("packed_seq_lengths"), ) + n_items = kwargs.get("num_items_in_batch", None) + if n_items is None: + n_items = kwargs.get("n_items", None) loss = fast_cross_entropy_loss( logits = shift_logits, labels = shift_labels, - n_items = kwargs.get("num_items_in_batch", None) - or kwargs.get("n_items", None), + n_items = n_items, ) if not return_dict: diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 003a0e7f1b..03f2c44701 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -22,7 +22,6 @@ from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union import inspect import os import re -import torch from unsloth_zoo.compiler import create_new_function from unsloth_zoo.log import logger from unsloth_zoo.logging_utils import PatchRLStatistics diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index e1cf8f6f82..36cfbf0b17 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -68,11 +68,9 @@ import functools import os import gc import math -import functools from typing import Optional, Tuple, List, Union import re, inspect, sys import contextlib -import types try: from huggingface_hub.utils import get_token From 9fedb1c11df2c4b7d1096962d2cf16d74372c80a Mon Sep 17 00:00:00 2001 From: lif <1835304752@qq.com> Date: Mon, 29 Dec 2025 15:17:58 +0800 Subject: [PATCH 038/114] fix: add support for init_lora_weights="corda" in get_peft_model (#3794) Add "corda" as an allowed value for the init_lora_weights parameter in FastLanguageModel.get_peft_model() and FastBaseModel.get_peft_model(). This enables users to use CorDA (Correlation-aware Decomposed Adaptation) initialization from PEFT, which provides an alternative LoRA initialization strategy for improved finetuning performance. Fixes #3693 Signed-off-by: majiayu000 <1835304752@qq.com> --- unsloth/models/_utils.py | 3 ++- unsloth/models/llama.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index abc8380562..ccb547f58e 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -1981,9 +1981,10 @@ def validate_loftq_config(loftq_config, lora_dropout, bias, init_lora_weights, m type(init_lora_weights) is bool or init_lora_weights == "gaussian" or init_lora_weights == "loftq" + or init_lora_weights == "corda" ): raise ValueError( - 'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq"].' + 'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq", "corda"].' ) if init_lora_weights == "loftq": diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 1d7695b9aa..762445b5e8 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -2779,9 +2779,10 @@ class FastLlamaModel: type(init_lora_weights) is bool or init_lora_weights == "gaussian" or init_lora_weights == "loftq" + or init_lora_weights == "corda" ): raise ValueError( - 'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq"].' + 'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq", "corda"].' ) if init_lora_weights == "loftq": From c452eb13f54dc572bb0b12c63ae47746543c4654 Mon Sep 17 00:00:00 2001 From: Fizza-Mukhtar Date: Tue, 30 Dec 2025 07:08:10 -0800 Subject: [PATCH 039/114] Fix 3D tensor support for bitsandbytes 8-bit matmul in forward pass --- unsloth/kernels/fast_lora.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index 60d0c318c3..16b679d005 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -379,9 +379,22 @@ class LoRA_QKV(torch.autograd.Function): ): dtype = X.dtype + # bitsandbytes 8-bit matmul expects 2D inputs. + # TorchInductor/AOTAutograd fails on 3D tensors during backward, + # so we explicitly flatten the sequence dimension. + orig_shape = X.shape + if X.dim() == 3: + X = X.view(-1, X.shape[-1]) + Q = matmul_lora(X, QW, QW_quant, QA, QB, QS) K = matmul_lora(X, KW, KW_quant, KA, KB, KS) V = matmul_lora(X, VW, VW_quant, VA, VB, VS) + + # Restore original shape after matmul + if len(orig_shape) == 3: + Q = Q.view(orig_shape[0], orig_shape[1], -1) + K = K.view(orig_shape[0], orig_shape[1], -1) + V = V.view(orig_shape[0], orig_shape[1], -1) ctx.custom_saved_tensors = ( QW, From f2e87251c721482d05bf8dd21452e4eb5c20ba02 Mon Sep 17 00:00:00 2001 From: Fizza-Mukhtar Date: Tue, 30 Dec 2025 07:56:01 -0800 Subject: [PATCH 040/114] Fix 3D tensor support for bitsandbytes 8-bit matmul in forward pass --- unsloth/kernels/fast_lora.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index 16b679d005..fbb18c3a15 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -383,12 +383,12 @@ class LoRA_QKV(torch.autograd.Function): # TorchInductor/AOTAutograd fails on 3D tensors during backward, # so we explicitly flatten the sequence dimension. orig_shape = X.shape + X_for_matmul = X if X.dim() == 3: - X = X.view(-1, X.shape[-1]) - - Q = matmul_lora(X, QW, QW_quant, QA, QB, QS) - K = matmul_lora(X, KW, KW_quant, KA, KB, KS) - V = matmul_lora(X, VW, VW_quant, VA, VB, VS) + X_for_matmul = X.view(-1, X.shape[-1]) + Q = matmul_lora(X_for_matmul, QW, QW_quant, QA, QB, QS) + K = matmul_lora(X_for_matmul, KW, KW_quant, KA, KB, KS) + V = matmul_lora(X_for_matmul, VW, VW_quant, VA, VB, VS) # Restore original shape after matmul if len(orig_shape) == 3: From e43e67cb18e3f9842ca34416db8f42b21f7154f2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 30 Dec 2025 15:58:40 +0000 Subject: [PATCH 041/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/kernels/fast_lora.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index fbb18c3a15..f1c0e298d9 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -389,7 +389,7 @@ class LoRA_QKV(torch.autograd.Function): Q = matmul_lora(X_for_matmul, QW, QW_quant, QA, QB, QS) K = matmul_lora(X_for_matmul, KW, KW_quant, KA, KB, KS) V = matmul_lora(X_for_matmul, VW, VW_quant, VA, VB, VS) - + # Restore original shape after matmul if len(orig_shape) == 3: Q = Q.view(orig_shape[0], orig_shape[1], -1) From b21b4e6252a8ee2381952d26d21fe023ad14c0d9 Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Tue, 30 Dec 2025 15:14:27 -0800 Subject: [PATCH 042/114] Refresh of Unsloth README.md with https://unsloth.ai/docs --- README.md | 115 +++++++++++++++++++++++++----------------------------- 1 file changed, 53 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index 7cd9d0bba4..ae1fccfbba 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@
- + unsloth logo @@ -18,7 +18,7 @@ ## ✨ Train for Free -Notebooks are beginner friendly. Read our [guide](https://docs.unsloth.ai/get-started/fine-tuning-guide). Add dataset, run, then deploy your trained model. +Notebooks are beginner friendly. Read our [guide](https://unsloth.ai/docs/get-started/fine-tuning-llms-guide). Add dataset, run, then deploy your trained model. | Model | Free Notebooks | Performance | Memory use | |-----------|---------|--------|----------| @@ -44,33 +44,35 @@ Notebooks are beginner friendly. Read our [guide](https://docs.unsloth.ai/get-st pip install unsloth ``` ### Windows -For Windows, `pip install unsloth` works only if you have Pytorch installed. Read our [Windows Guide](https://docs.unsloth.ai/get-started/installing-+-updating/windows-installation). +For Windows, `pip install unsloth` works only if you have Pytorch installed. Read our [Windows Guide](https://unsloth.ai/docs/get-started/install-and-update/windows-installation). + ### Docker Use our official [Unsloth Docker image](https://hub.docker.com/r/unsloth/unsloth) ```unsloth/unsloth``` container. Read our [Docker Guide](https://unsloth.ai/docs/get-started/install-and-update/docker). + ### Blackwell & DGX Spark For RTX 50x, B200, 6000 GPUs: `pip install unsloth`. Read our [Blackwell Guide](https://unsloth.ai/docs/basics/fine-tuning-llms-with-blackwell-rtx-50-series-and-unsloth) and [DGX Spark Guide](https://unsloth.ai/docs/basics/fine-tuning-llms-with-nvidia-dgx-spark-and-unsloth) for more details. ## 🦥 Unsloth News -- New RoPE & MLP **Triton Kernels** & **Padding Free + Packing**: 3x faster training & 30% less VRAM. [Blog](https://docs.unsloth.ai/new/3x-faster-training-packing) -- **Ministral 3** by Mistral: Run Ministral 3 or fine-tune with vision/RL sodoku notebooks. [Guide](https://docs.unsloth.ai/new/ministral-3) • [Notebooks](https://docs.unsloth.ai/new/ministral-3#fine-tuningb) -- **500K Context**: Training a 20B model with >500K context is now possible on an 80GB GPU. [Blog](https://docs.unsloth.ai/new/500k-context-length-fine-tuning) -- **FP8 Reinforcement Learning**: You can now do FP8 GRPO on consumer GPUs. [Blog](https://docs.unsloth.ai/new/fp8-reinforcement-learning) • [Notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_8B_FP8_GRPO.ipynb) -- **DeepSeek-OCR**: Fine-tune to improve language understanding by 89%. [Guide](https://docs.unsloth.ai/new/deepseek-ocr-run-and-fine-tune) • [Notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Deepseek_OCR_(3B).ipynb) -- **Docker**: Use Unsloth with no setup & environment issues with our new image. [Guide](https://docs.unsloth.ai/new/how-to-train-llms-with-unsloth-and-docker) • [Docker image](https://hub.docker.com/r/unsloth/unsloth) -- **gpt-oss RL**: Introducing the fastest possible inference for gpt-oss RL! [Read blog](https://docs.unsloth.ai/new/gpt-oss-reinforcement-learning) -- **Vision RL**: You can now train VLMs with GRPO or GSPO in Unsloth! [Read guide](https://docs.unsloth.ai/new/vision-reinforcement-learning-vlm-rl) -- **gpt-oss** by OpenAI: Read our [Unsloth Flex Attention](https://docs.unsloth.ai/new/long-context-gpt-oss-training) blog and [gpt-oss Guide](https://docs.unsloth.ai/basics/gpt-oss). 20B works on 14GB VRAM. 120B on 65GB. +- New RoPE & MLP **Triton Kernels** & **Padding Free + Packing**: 3x faster training & 30% less VRAM. [Blog](https://unsloth.ai/docs/new/3x-faster-training-packing) +- **New Mistral**: Run Ministral 3 or Devstral 2 and fine-tune with vision/RL sodoku notebooks. [Guide](https://unsloth.ai/docs/models/ministral-3) • [Notebooks](https://unsloth.ai/docs/models/ministral-3#fine-tuning-ministral-3) +- **500K Context**: Training a 20B model with >500K context is now possible on an 80GB GPU. [Blog](https://unsloth.ai/docs/new/500k-context-length-fine-tuning) +- **FP8 Reinforcement Learning**: You can now do FP8 GRPO on consumer GPUs. [Blog](https://unsloth.ai/docs/new/fp8-reinforcement-learning) • [Notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_8B_FP8_GRPO.ipynb) +- **DeepSeek-OCR**: Fine-tune to improve language understanding by 89%. [Guide](https://unsloth.ai/docs/models/deepseek-ocr-how-to-run-and-fine-tune) • [Notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Deepseek_OCR_(3B).ipynb) +- **Docker**: Use Unsloth with no setup & environment issues with our new image. [Guide](https://unsloth.ai/docs/new/how-to-fine-tune-llms-with-unsloth-and-docker) • [Docker image](https://hub.docker.com/r/unsloth/unsloth) +- **gpt-oss RL**: Introducing the fastest possible inference for gpt-oss RL! [Read blog](https://unsloth.ai/docs/models/gpt-oss-how-to-run-and-fine-tune/gpt-oss-reinforcement-learning) +- **Vision RL**: You can now train VLMs with GRPO or GSPO in Unsloth! [Read guide](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide/vision-reinforcement-learning-vlm-rl) +- **gpt-oss** by OpenAI: Read our [Unsloth Flex Attention](https://unsloth.ai/docs/models/gpt-oss-how-to-run-and-fine-tune/long-context-gpt-oss-training) blog and [gpt-oss Guide](https://unsloth.ai/docs/models/gpt-oss-how-to-run-and-fine-tune). 20B works on 14GB VRAM. 120B on 65GB.
Click for more news -- **Quantization-Aware Training**: We collabed with Pytorch, recovering ~70% accuracy. [Read blog](https://docs.unsloth.ai/new/quantization-aware-training-qat) -- **Memory-efficient RL**: We're introducing even better RL. Our new kernels & algos allows faster RL with 50% less VRAM & 10× more context. [Read blog](https://docs.unsloth.ai/new/memory-efficient-rl) -- **Gemma 3n** by Google: [Read Blog](https://docs.unsloth.ai/basics/gemma-3n-how-to-run-and-fine-tune). We [uploaded GGUFs, 4-bit models](https://huggingface.co/collections/unsloth/gemma-3n-685d3874830e49e1c93f9339). -- **[Text-to-Speech (TTS)](https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning)** is now supported, including `sesame/csm-1b` and STT `openai/whisper-large-v3`. -- **[Qwen3](https://docs.unsloth.ai/basics/qwen3-how-to-run-and-fine-tune)** is now supported. Qwen3-30B-A3B fits on 17.5GB VRAM. -- Introducing **[Dynamic 2.0](https://docs.unsloth.ai/basics/unsloth-dynamic-2.0-ggufs)** quants that set new benchmarks on 5-shot MMLU & Aider Polyglot. -- [**EVERYTHING** is now supported](https://unsloth.ai/blog/gemma3#everything) - all models (TTS, BERT, Mamba), FFT, etc. [MultiGPU](https://docs.unsloth.ai/basics/multi-gpu-training-with-unsloth) coming soon. Enable FFT with `full_finetuning = True`, 8-bit with `load_in_8bit = True`. +- **Quantization-Aware Training**: We collabed with Pytorch, recovering ~70% accuracy. [Read blog](https://unsloth.ai/docs/basics/quantization-aware-training-qat) +- **Memory-efficient RL**: We're introducing even better RL. Our new kernels & algos allows faster RL with 50% less VRAM & 10× more context. [Read blog](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide/memory-efficient-rl) +- **Gemma 3n** by Google: [Read Blog](https://unsloth.ai/docs/models/gemma-3-how-to-run-and-fine-tune/gemma-3n-how-to-run-and-fine-tune). We [uploaded GGUFs, 4-bit models](https://huggingface.co/collections/unsloth/gemma-3n-685d3874830e49e1c93f9339). +- **[Text-to-Speech (TTS)](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning)** is now supported, including `sesame/csm-1b` and STT `openai/whisper-large-v3`. +- **[Qwen3](https://unsloth.ai/docs/models/qwen3-how-to-run-and-fine-tune)** is now supported. Qwen3-30B-A3B fits on 17.5GB VRAM. +- Introducing **[Dynamic 2.0](https://unsloth.ai/docs/basics/unsloth-dynamic-2.0-ggufs)** quants that set new benchmarks on 5-shot MMLU & Aider Polyglot. +- [**EVERYTHING** is now supported](https://unsloth.ai/blog/gemma3#everything) - all models (TTS, BERT, Mamba), FFT, etc. [MultiGPU](https://unsloth.ai/docs/basics/multi-gpu-training-with-unsloth) coming soon. Enable FFT with `full_finetuning = True`, 8-bit with `load_in_8bit = True`. - 📣 [DeepSeek-R1](https://unsloth.ai/blog/deepseek-r1) - run or fine-tune them [with our guide](https://unsloth.ai/blog/deepseek-r1). All model uploads: [here](https://huggingface.co/collections/unsloth/deepseek-r1-all-versions-678e1c48f5d2fce87892ace5). - 📣 Introducing Long-context [Reasoning (GRPO)](https://unsloth.ai/blog/grpo) in Unsloth. Train your own reasoning model with just 5GB VRAM. Transform Llama, Phi, Mistral etc. into reasoning LLMs! - 📣 Introducing Unsloth [Dynamic 4-bit Quantization](https://unsloth.ai/blog/dynamic-4bit)! We dynamically opt not to quantize certain parameters and this greatly increases accuracy while only using <10% more VRAM than BnB 4-bit. See our collection on [Hugging Face here.](https://huggingface.co/collections/unsloth/unsloth-4-bit-dynamic-quants-67503bb873f89e15276c44e7) @@ -84,28 +86,29 @@ For RTX 50x, B200, 6000 GPUs: `pip install unsloth`. Read our [Blackwell Guide](
## 🔗 Links and Resources -| Type | Links | -| ------------------------------- | --------------------------------------- | -|   **r/unsloth Reddit** | [Join Reddit community](https://reddit.com/r/unsloth)| -| 📚 **Documentation & Wiki** | [Read Our Docs](https://docs.unsloth.ai) | -|   **Twitter (aka X)** | [Follow us on X](https://twitter.com/unslothai)| -| 💾 **Installation** | [Pip & Docker Install](https://docs.unsloth.ai/get-started/installing-+-updating)| -| 🔮 **Our Models** | [Unsloth Catalog](https://docs.unsloth.ai/get-started/all-our-models)| -| ✍️ **Blog** | [Read our Blogs](https://unsloth.ai/blog)| +| Type | Links | +| ----------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ | +|   **r/unsloth Reddit** | [Join Reddit community](https://reddit.com/r/unsloth) | +| 📚 **Documentation & Wiki** | [Read Our Docs](https://unsloth.ai/docs) | +|   **Twitter (aka X)** | [Follow us on X](https://twitter.com/unslothai) | +| 💾 **Installation** | [Pip & Docker Install](https://unsloth.ai/docs/get-started/install-and-update) | +| 🔮 **Our Models** | [Unsloth Catalog](https://unsloth.ai/docs/get-started/unsloth-model-catalog) | +| ✍️ **Blog** | [Read our Blogs](https://unsloth.ai/blog) | ## ⭐ Key Features -- Supports **full-finetuning**, pretraining, 4b-bit, 16-bit and **FP8** training -- Supports **all models** including [TTS](https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning), multimodal, [BERT](https://docs.unsloth.ai/get-started/unsloth-notebooks#other-important-notebooks) and more! Any model that works in transformers, works in Unsloth. -- The most efficient library for [Reinforcement Learning (RL)](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide), using 80% less VRAM. Supports GRPO, GSPO, DrGRPO, DAPO etc. -- **0% loss in accuracy** - no approximation methods - all exact. -- Export and [deploy your model](https://unsloth.ai/docs/basics/inference-and-deployment) to GGUF, llama.cpp, vLLM, SGLang and Hugging Face. -- Supports NVIDIA (since 2018), [AMD](https://docs.unsloth.ai/get-started/install-and-update/amd) and Intel GPUs. Minimum CUDA Capability 7.0 (V100, T4, Titan V, RTX 20, 30, 40x, A100, H100, L40 etc) -- Works on **Linux**, WSL and **Windows** -- All kernels written in [OpenAI's Triton](https://openai.com/index/triton/) language. Manual backprop engine. -- If you trained a model with 🦥Unsloth, you can use this cool sticker!   + +* Supports **full-finetuning**, pretraining, 4b-bit, 16-bit and **FP8** training +* Supports **all models** including [TTS](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning), multimodal, [BERT](https://unsloth.ai/docs/get-started/unsloth-notebooks#other-important-notebooks) and more! Any model that works in transformers, works in Unsloth. +* The most efficient library for [Reinforcement Learning (RL)](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide), using 80% less VRAM. Supports GRPO, GSPO, DrGRPO, DAPO etc. +* **0% loss in accuracy** - no approximation methods - all exact. +* Export and [deploy your model](https://unsloth.ai/docs/basics/inference-and-deployment) to GGUF, llama.cpp, vLLM, SGLang and Hugging Face. +* Supports NVIDIA (since 2018), [AMD](https://unsloth.ai/docs/get-started/install-and-update/amd) and Intel GPUs. Minimum CUDA Capability 7.0 (V100, T4, Titan V, RTX 20, 30, 40x, A100, H100, L40 etc) +* Works on **Linux**, WSL and **Windows** +* All kernels written in OpenAI's Triton language. Manual backprop engine. +* If you trained a model with 🦥Unsloth, you can use this cool sticker!   ## 💾 Install Unsloth -You can also see our docs for more detailed installation and updating instructions [here](https://docs.unsloth.ai/get-started/installing-+-updating). +You can also see our docs for more detailed installation and updating instructions [here](https://unsloth.ai/docs/get-started/install-and-update). Unsloth supports Python 3.13 or lower. @@ -125,7 +128,7 @@ See [here](#advanced-pip-installation) for advanced pip install instructions. You should install the latest driver for your GPU. Download drivers here: [NVIDIA GPU Driver](https://www.nvidia.com/Download/index.aspx). 3. **Install Visual Studio C++:** - You will need Visual Studio, with C++ installed. By default, C++ is not installed with [Visual Studio](https://visualstudio.microsoft.com/vs/community/), so make sure you select all of the C++ options. Also select options for Windows 10/11 SDK. For detailed instructions with options, see [here](https://docs.unsloth.ai/get-started/installing-+-updating). + You will need Visual Studio, with C++ installed. By default, C++ is not installed with [Visual Studio](https://visualstudio.microsoft.com/vs/community/), so make sure you select all of the C++ options. Also select options for Windows 10/11 SDK. For detailed instructions with options, see [here](https://unsloth.ai/docs/get-started/install-and-update/windows-installation#method-3-windows-directly). 5. **Install CUDA Toolkit:** Follow the instructions to install [CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit-archive). @@ -140,19 +143,7 @@ See [here](#advanced-pip-installation) for advanced pip install instructions. pip install unsloth ``` -#### Notes -To run Unsloth directly on Windows: -- Install Triton from this Windows fork and follow the instructions [here](https://github.com/woct0rdho/triton-windows) (be aware that the Windows fork requires PyTorch >= 2.4 and CUDA 12) -- In the `SFTConfig`, set `dataset_num_proc=1` to avoid a crashing issue: -```python -SFTConfig( - dataset_num_proc=1, - ... -) -``` - #### Advanced/Troubleshooting - For **advanced installation instructions** or if you see weird errors during installations: First try using an isolated environment via then `pip install unsloth` @@ -269,7 +260,7 @@ print(f'pip install --upgrade pip && pip install --no-deps git+https://github.co ``` ### Docker Installation You can use our pre-built Docker container with all dependencies to use Unsloth instantly with no setup required. -[Read our guide](https://docs.unsloth.ai/get-started/install-and-update/docker). +[Read our guide](https://unsloth.ai/docs/get-started/install-and-update/docker). This container requires installing [NVIDIA's Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). @@ -284,9 +275,9 @@ docker run -d -e JUPYTER_PASSWORD="mypassword" \ Access Jupyter Lab at `http://localhost:8888` and start fine-tuning! ## 📜 Documentation -- Go to our official [Documentation](https://unsloth.ai/docs) for [running models](https://docs.unsloth.ai/basics/running-and-saving-models), [saving to GGUF](https://docs.unsloth.ai/basics/running-and-saving-models/saving-to-gguf), [checkpointing](https://docs.unsloth.ai/basics/finetuning-from-last-checkpoint), [evaluation](https://docs.unsloth.ai/get-started/fine-tuning-llms-guide#evaluation) and more! -- Read our Guides for: [Fine-tuning](https://docs.unsloth.ai/get-started/fine-tuning-llms-guide), [Reinforcement Learning](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide), [Text-to-Speech (TTS)](https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning), [Vision](https://docs.unsloth.ai/basics/vision-fine-tuning) and [any model](https://docs.unsloth.ai/models/tutorials-how-to-fine-tune-and-run-llms). -- We support Huggingface's transformers, TRL, Trainer, Seq2SeqTrainer and Pytorch code. +* Go to our official [Documentation](https://unsloth.ai/docs) for [running models](https://unsloth.ai/docs/basics/inference-and-deployment), [saving to GGUF](https://unsloth.ai/docs/basics/inference-and-deployment/saving-to-gguf), [checkpointing](https://unsloth.ai/docs/basics/finetuning-from-last-checkpoint), [evaluation](https://unsloth.ai/docs/get-started/fine-tuning-llms-guide#evaluation) and more! +* Read our Guides for: [Fine-tuning](https://unsloth.ai/docs/get-started/fine-tuning-llms-guide), [Reinforcement Learning](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide), [Text-to-Speech (TTS)](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning), [Vision](https://unsloth.ai/docs/basics/vision-fine-tuning) and [any model](https://unsloth.ai/docs/models/tutorials-how-to-fine-tune-and-run-llms). +* We support Huggingface's transformers, TRL, Trainer, Seq2SeqTrainer and Pytorch code. Unsloth example code to fine-tune gpt-oss-20b: @@ -311,8 +302,9 @@ model, tokenizer = FastModel.from_pretrained( max_seq_length = 2048, # Choose any for long context! load_in_4bit = True, # 4-bit quantization. False = 16-bit LoRA. load_in_8bit = False, # 8-bit quantization - load_in_16bit = False, # [NEW!] 16-bit LoRA + load_in_16bit = False, # 16-bit LoRA full_finetuning = False, # Use for full fine-tuning. + trust_remote_code = False, # Enable to support new models # token = "hf_...", # use one if using gated models ) @@ -351,7 +343,7 @@ trainer = SFTTrainer( ) trainer.train() -# Go to https://docs.unsloth.ai for advanced tips like +# Go to https://unsloth.ai/docs for advanced tips like # (1) Saving to GGUF / merging to 16bit for vLLM or SGLang # (2) Continued training from a saved LoRA adapter # (3) Adding an evaluation loop / OOMs @@ -360,14 +352,15 @@ trainer.train() ## 💡 Reinforcement Learning -[RL](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide) including [GRPO](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide#training-with-grpo), [GSPO](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide/gspo-reinforcement-learning), **FP8** traning, DrGRPO, DAPO, PPO, Reward Modelling, Online DPO all work with Unsloth. -Read our [Reinforcement Learning Guide](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide) or our [advanced RL docs](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide/advanced-rl-documentation) for batching, generation & training parameters. +[RL](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide) including [GRPO](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide#training-with-grpo), [GSPO](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide/gspo-reinforcement-learning), [**FP8** training](https://unsloth.ai/docs/new/fp8-reinforcement-learning), DrGRPO, DAPO, PPO, Reward Modelling, Online DPO all work with Unsloth. + +Read our [Reinforcement Learning Guide](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide) or our [advanced RL docs](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide/advanced-rl-documentation) for batching, generation & training parameters. List of RL notebooks: - gpt-oss GSPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/gpt-oss-(20B)-GRPO.ipynb) -- Qwen2.5-VL GSPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen2_5_7B_VL_GRPO.ipynb) +- - ***FP8*** Qwen3-8B GRPO notebook (L4): [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_8B_FP8_GRPO.ipynb) +- Qwen2.3-VL GSPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_VL_(8B)-Vision-GRPO.ipynb) - Advanced Qwen3 GRPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_(4B)-GRPO.ipynb) -- ***FP8*** Qwen3-8B GRPO notebook (L4): [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_8B_FP8_GRPO.ipynb) - ORPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3_(8B)-ORPO.ipynb) - DPO Zephyr notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Zephyr_(7B)-DPO.ipynb) - KTO notebook: [Link](https://colab.research.google.com/drive/1MRgGtLWuZX4ypSfGguFgC-IblTvO2ivM?usp=sharing) @@ -427,6 +420,4 @@ You can cite the Unsloth repo as follows: - The [llama.cpp library](https://github.com/ggml-org/llama.cpp) that lets users save models with Unsloth - The Hugging Face team and their libraries: [transformers](https://github.com/huggingface/transformers) and [TRL](https://github.com/huggingface/trl) - The Pytorch and [Torch AO](https://github.com/unslothai/unsloth/pull/3391) team for their contributions -- [Erik](https://github.com/erikwijmans) for his help adding [Apple's ML Cross Entropy](https://github.com/apple/ml-cross-entropy) in Unsloth -- [Etherl](https://github.com/Etherll) for adding support for [TTS, diffusion and BERT models](https://github.com/unslothai/notebooks/pull/34) - And of course for every single person who has contributed or has used Unsloth! From 982ae7bbebc1bef9c24dae857c794ac82cd75981 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 31 Dec 2025 21:35:48 -0800 Subject: [PATCH 043/114] Fix correctness bugs in rl.py, rl_replacements.py, and vision.py (#3811) * Fix correctness bugs in rl.py, rl_replacements.py, and vision.py 1. rl_replacements.py (lines 864, 870): Fixed undefined `nanmin`/`nanmax` functions by using `.nan_to_num(nan=inf/-inf).min()/.max()` pattern. PyTorch doesn't have torch.nanmin/nanmax, so we replace NaN values before computing min/max. 2. vision.py (line 150): Fixed bug where code checked for "input" key but then accessed kwargs["input_ids"] instead of kwargs["input"]. 3. vision.py (line 159): Fixed bug where literal string "key" was used instead of the variable `key` when accessing kwargs. 4. rl.py (lines 903, 905): Fixed non-existent `MathError` exception by replacing with `ValueError`. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- unsloth/models/rl.py | 4 ++-- unsloth/models/rl_replacements.py | 10 ++++++++-- unsloth/models/vision.py | 4 ++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 03f2c44701..e1c43b8b85 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -900,9 +900,9 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): if "temperature" in call_args: check_temperature = ( "if temperature <= 0:\n" - " raise MathError('Unsloth: Please set a positive non-zero temperature since your results will be wrong.')\n" + " raise ValueError('Unsloth: Please set a positive non-zero temperature since your results will be wrong.')\n" "elif temperature >= 10:\n" - " raise MathError('Unsloth: Please set a positive non-zero temperature less than 10, since sampling will be quite erratic.')\n" + " raise ValueError('Unsloth: Please set a positive non-zero temperature less than 10, since sampling will be quite erratic.')\n" "\n" ) extra_args += check_temperature diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 3dfeea6871..5e079335ae 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -861,13 +861,19 @@ def grpo_trainer_compute_loss(function_name, function): else torch.tensor(0.0, device = self.model.device) ) self._metrics[mode]["sampling/importance_sampling_ratio/min"].append( - nanmin(self.accelerator.gather(min_importance_sampling_ratio)).item() + self.accelerator.gather(min_importance_sampling_ratio) + .nan_to_num(nan = float("inf")) + .min() + .item() ) self._metrics[mode]["sampling/importance_sampling_ratio/mean"].append( self.accelerator.gather(mean_importance_sampling_ratio).nanmean().item() ) self._metrics[mode]["sampling/importance_sampling_ratio/max"].append( - nanmax(self.accelerator.gather(max_importance_sampling_ratio)).item() + self.accelerator.gather(max_importance_sampling_ratio) + .nan_to_num(nan = float("-inf")) + .max() + .item() ) return loss diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 36cfbf0b17..c909f963b9 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -147,7 +147,7 @@ def unsloth_base_fast_generate( elif "input_ids" in kwargs: input_ids = kwargs["input_ids"] elif "input" in kwargs: - input_ids = kwargs["input_ids"] + input_ids = kwargs["input"] elif "input_features" in kwargs: input_ids = kwargs["input_features"] elif "input_embeds" in kwargs: @@ -156,7 +156,7 @@ def unsloth_base_fast_generate( input_ids = kwargs["inputs"] else: key = next(iter(kwargs.keys())) - if type(kwargs["key"]) is not torch.Tensor: + if type(kwargs[key]) is not torch.Tensor: raise TypeError("Unsloth: You need to pass in input_ids to .generate!") input_ids = kwargs[key] assert type(input_ids) is torch.Tensor From 963bc35a961d02fba7a0245938e2af306479d4d6 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 1 Jan 2026 02:36:33 -0800 Subject: [PATCH 044/114] Fix correctness bugs across multiple model files (#3813) 1. cohere.py:347-348 - Fixed wrong variable names in QK normalization. Used `Q`/`K` but variables were named `Qn`/`Kn`. This caused NameError when `use_qk_norm=True` (e.g., c4ai-command-r-plus models). 2. cohere.py:482 - Fixed wrong object reference in inference loop. Used `self.mlp` but should be `decoder_layer.mlp` since we're iterating through decoder layers. Caused AttributeError during inference. 3. falcon_h1.py:459,461 - Fixed wrong attribute names in inference path. Used `post_attention_layernorm` and `mlp` but Falcon H1 uses `pre_ff_layernorm` and `feed_forward`. Caused AttributeError during generation. 4. qwen3_moe.py:210 - Fixed wrong module path with incorrect capitalization. Used `transformers.models.Qwen3Moe` but should be `transformers.models.qwen3_moe`. Caused AttributeError when patching rotary embeddings. 5. qwen3_moe.py:239 - Fixed wrong model_patcher class. Used `FastQwen3Model` but should be `FastQwen3MoeModel` for MoE models. Caused incorrect patching for Qwen3 MoE models. 6. hf_hub.py:21-22 - Fixed floor division and missing return for billion values. Used `//` instead of `/` for millions, and had no return for values >= 1B. Caused incorrect formatting and None return for large numbers. 7. save.py:550 - Fixed self-assignment that did nothing. `sharded_ram_usage = sharded_ram_usage` should be `= max_shard_size`. Caused integer shard sizes to be ignored. 8. rl.py:562-567 - Fixed orphan string not included in length_check. The elif branch for max_seq_length validation was a standalone string expression, not concatenated to length_check. Caused silent skip of the max_seq_length > model_max_seq_length warning. 9. granite.py:49-52 - Fixed wrong model name and version in error message. Said "Gemma2" and "4.42.3" but should be "Granite" and "4.45.0". --- unsloth/models/cohere.py | 6 +++--- unsloth/models/falcon_h1.py | 4 ++-- unsloth/models/granite.py | 6 +++--- unsloth/models/qwen3_moe.py | 4 ++-- unsloth/models/rl.py | 6 +++++- unsloth/save.py | 2 +- unsloth/utils/hf_hub.py | 4 +++- 7 files changed, 19 insertions(+), 13 deletions(-) diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.py index a091a0173f..e9f56763d6 100644 --- a/unsloth/models/cohere.py +++ b/unsloth/models/cohere.py @@ -344,8 +344,8 @@ def CohereAttention_fast_forward_inference( Kn = Kn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2) Vn = Vn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2) if self.use_qk_norm: - Q = fast_layernorm_inference(self.q_norm, Q, self.q_norm_out_weight) - K = fast_layernorm_inference(self.k_norm, K, self.k_norm_out_weight) + Qn = fast_layernorm_inference(self.q_norm, Qn, self.q_norm_out_weight) + Kn = fast_layernorm_inference(self.k_norm, Kn, self.k_norm_out_weight) # cos, sin = self.rotary_emb(Vn, seq_len = kv_seq_len) # Qn, Kn = inplace_rope_embedding(Qn, Kn, cos, sin, position_ids) @@ -479,7 +479,7 @@ def CohereModel_fast_forward_inference( ) ) - hidden_states_mlp = fast_swiglu_inference(self.mlp, hidden_states) + hidden_states_mlp = fast_swiglu_inference(decoder_layer.mlp, hidden_states) residual += hidden_states_attention residual += hidden_states_mlp hidden_states = residual diff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.py index fc5ea458a6..428f49d727 100644 --- a/unsloth/models/falcon_h1.py +++ b/unsloth/models/falcon_h1.py @@ -456,9 +456,9 @@ def FalconH1DecoderLayer_fast_forward( # Fully Connected residual = hidden_states hidden_states = fast_rms_layernorm_inference( - self.post_attention_layernorm, hidden_states + self.pre_ff_layernorm, hidden_states ) - hidden_states = fast_swiglu_inference(self.mlp, hidden_states) + hidden_states = fast_swiglu_inference(self.feed_forward, hidden_states) hidden_states += residual else: residual = hidden_states diff --git a/unsloth/models/granite.py b/unsloth/models/granite.py index 2632ab6914..f85f1b641f 100644 --- a/unsloth/models/granite.py +++ b/unsloth/models/granite.py @@ -46,9 +46,9 @@ except: transformers_version = Version(transformers_version) if not transformers_version >= Version("4.45.0"): raise ImportError( - f"Unsloth: Your transformers version of {transformers_version} does not support Gemma2.\n" - f"The minimum required version is 4.42.3.\n" - f'Try `pip install --upgrade "transformers>=4.42.3"`\n' + f"Unsloth: Your transformers version of {transformers_version} does not support Granite.\n" + f"The minimum required version is 4.45.0.\n" + f'Try `pip install --upgrade "transformers>=4.45.0"`\n' f"to obtain the latest transformers build, then restart this session." ) diff --git a/unsloth/models/qwen3_moe.py b/unsloth/models/qwen3_moe.py index bec3fa7b0d..e1f8c71b6b 100644 --- a/unsloth/models/qwen3_moe.py +++ b/unsloth/models/qwen3_moe.py @@ -207,7 +207,7 @@ class FastQwen3MoeModel(FastQwen3Model): # https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py\ import transformers.models.qwen3_moe.modeling_qwen3_moe - transformers.models.Qwen3Moe.modeling_qwen3_moe.Qwen3MoeRotaryEmbedding = ( + transformers.models.qwen3_moe.modeling_qwen3_moe.Qwen3MoeRotaryEmbedding = ( LlamaRotaryEmbedding ) return @@ -236,7 +236,7 @@ class FastQwen3MoeModel(FastQwen3Model): device_map = device_map, rope_scaling = rope_scaling, fix_tokenizer = fix_tokenizer, - model_patcher = FastQwen3Model, + model_patcher = FastQwen3MoeModel, tokenizer_name = tokenizer_name, trust_remote_code = trust_remote_code, **kwargs, diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index e1c43b8b85..22189f459c 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -559,8 +559,12 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): " if args_max_seq_length is None and model_max_seq_length is not None:\n" " max_seq_length = model.max_seq_length\n" " if hasattr(args, 'max_seq_length'): args.max_seq_length = max_seq_length\n" + " elif args_max_seq_length is not None and model_max_seq_length is not None:\n" + " if args_max_seq_length > model_max_seq_length:\n" + " print('Unsloth: You set `max_seq_length` as ' + str(args_max_seq_length) + ' but '\n" + " 'the maximum the model supports is ' + str(model_max_seq_length) + '. We shall reduce it.')\n" + " args.max_seq_length = model_max_seq_length\n" ) - " elif args_max_seq_length is not None and model_max_seq_length is not None:\n" " if args_max_seq_length > model_max_seq_length:\n" " print('Unsloth: You set `max_seq_length` as ' + str(args_max_seq_length) + ' but \n" " the maximum the model supports is ' + str(model_max_seq_length) + '. We shall reduce it.')\n" " args.max_seq_length = model_max_seq_length\n" extra_args += length_check # At this point max_seq_length might be set, but trl is moving to max_length diff --git a/unsloth/save.py b/unsloth/save.py index 3a275cf0c3..ceb36854d2 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -547,7 +547,7 @@ def unsloth_save_model( elif mb_found: sharded_ram_usage = int(mb_found.group(1)) * 1024 * 1024 elif type(max_shard_size) is int: - sharded_ram_usage = sharded_ram_usage + sharded_ram_usage = max_shard_size # Switch to our fast saving modules if it's a slow PC! n_cpus = psutil.cpu_count(logical = False) diff --git a/unsloth/utils/hf_hub.py b/unsloth/utils/hf_hub.py index 75df00fbf0..e3960ba0ce 100644 --- a/unsloth/utils/hf_hub.py +++ b/unsloth/utils/hf_hub.py @@ -19,7 +19,9 @@ def formatted_int(value: int) -> str: elif value < MILLION: return f"{float(value) / 1000:,.1f}K" elif value < BILLION: - return f"{float(value) // 1000000:,.1f}M" + return f"{float(value) / 1000000:,.1f}M" + else: + return f"{float(value) / 1000000000:,.1f}B" def get_model_info( From f7e0f4b152b67479f3b3b889f198daa3b9b28691 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 1 Jan 2026 12:54:21 +0000 Subject: [PATCH 045/114] Add TODO comment for ensure_weight_tying in vision models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- unsloth/models/vision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 9f847f2837..b4ce718f46 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -930,7 +930,7 @@ class FastBaseModel: task_type = TaskType.CAUSAL_LM, temporary_location = "_unsloth_temporary_saved_buffers", qat_scheme = None, - ensure_weight_tying = False, + ensure_weight_tying = False, # [TODO] Add `ensure_weight_tying` for `modules_to_save` for vision models **kwargs, ): if os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1": From 1080d0c4dc15ec97e40eacf17e81ff04c8518c88 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Fri, 2 Jan 2026 07:19:08 +0000 Subject: [PATCH 046/114] Fix Gemma3 QAT training instability with int8-int4 scheme Gemma3 models have a large vocabulary (262144 tokens) which causes training loss to explode when using int8 embedding quantization. This fix auto-detects Gemma3 models and switches from int8-int4 (phone-deployment) to int4 weight-only QAT for stable training. --- unsloth/models/_utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index ccb547f58e..3851e18f92 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -2198,6 +2198,18 @@ def _prepare_model_for_qat( from torchao.quantization.granularity import PerGroup, PerAxis from torchao.quantization.qat import QATConfig + # Gemma3 models have issues with int8 embedding quantization due to their + # large vocabulary size (262144). Auto-switch to int4 weight-only instead. + if qat_scheme == "int8-int4": + model_types = get_transformers_model_type(model.config) + is_gemma3 = any("gemma3" in mt or "gemma_3" in mt for mt in model_types) + if is_gemma3: + print( + "Unsloth: Gemma3 has a large vocabulary causing int8 embedding issues. " + "Switching to int4 weight-only QAT for training stability." + ) + qat_scheme = "int4" + if not isinstance(qat_scheme, TorchAOConfig): torchao_config: Optional[TorchAOConfig] = None if qat_scheme == "fp8-int4": From ae219fe05225b768953d2e6cdaac97b8813d8746 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 2 Jan 2026 00:14:44 -0800 Subject: [PATCH 047/114] fix_huggingface_hub --- unsloth/__init__.py | 3 +++ unsloth/import_fixes.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/unsloth/__init__.py b/unsloth/__init__.py index d10a0f8030..c74b248a83 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -30,16 +30,19 @@ from .import_fixes import ( check_fbgemm_gpu_version, torchvision_compatibility_check, fix_diffusers_warnings, + fix_huggingface_hub, ) fix_message_factory_issue() check_fbgemm_gpu_version() torchvision_compatibility_check() fix_diffusers_warnings() +fix_huggingface_hub() del fix_message_factory_issue del check_fbgemm_gpu_version del torchvision_compatibility_check del fix_diffusers_warnings +del fix_huggingface_hub # This check is critical because Unsloth optimizes these libraries by modifying # their code at import time. If they're imported first, the original (slower, diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index efc7a7f4cd..f388f4ea8d 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -539,3 +539,10 @@ def fix_executorch(): def fix_diffusers_warnings(): # Silence Flax classes are deprecated and will be removed in Diffusers v1.0.0. os.environ["DIFFUSERS_VERBOSITY"] = "error" + + +def fix_huggingface_hub(): + # huggingface_hub.is_offline_mode got removed, so add it back + import huggingface_hub + if not hasattr(huggingface_hub, "is_offline_mode"): + huggingface_hub.is_offline_mode = lambda: huggingface_hub.constants.HF_HUB_OFFLINE From 13e1255b6c8a35c6f1a96c14e0153ddb14289e60 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 2 Jan 2026 02:48:28 -0800 Subject: [PATCH 048/114] Update loader.py --- unsloth/models/loader.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 91016a13ba..247c72f43f 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -204,6 +204,17 @@ class FastLanguageModel(FastLlamaModel): "Unsloth: Please install vLLM before enabling `fast_inference`!\n" "You can do this in a terminal via `pip install vllm`" ) + if DEVICE_TYPE_TORCH == "cuda": + for i in range(DEVICE_COUNT): + # [TODO] DGX Spark vLLM breaks + if "NVIDIA GB10" in str(torch.cuda.get_device_name(i)).upper(): + print( + "Unsloth: DGX Spark detected - `fast_inference=True` is currently broken as of January 2026.\n" + "Defaulting to native Unsloth inference." + ) + fast_inference = False + break + # [TODO] For now fast_inference only works with fast_inference ie vLLM if load_in_fp8 != False: if not fast_inference: @@ -744,6 +755,17 @@ class FastModel(FastBaseModel): "Unsloth: Please install vLLM before enabling `fast_inference`!\n" "You can do this in a terminal via `pip install vllm`" ) + if DEVICE_TYPE_TORCH == "cuda": + for i in range(DEVICE_COUNT): + # [TODO] DGX Spark vLLM breaks + if "NVIDIA GB10" in str(torch.cuda.get_device_name(i)).upper(): + print( + "Unsloth: DGX Spark detected - `fast_inference=True` is currently broken as of January 2026.\n" + "Defaulting to native Unsloth inference." + ) + fast_inference = False + break + # [TODO] For now fast_inference only works with fast_inference ie vLLM if load_in_fp8 != False: if not fast_inference: From a24695dcc2e8bd34eca2cdc00e93a20bc2704c65 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 2 Jan 2026 03:41:51 -0800 Subject: [PATCH 049/114] Update import_fixes.py --- unsloth/import_fixes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index f388f4ea8d..da0fbc613b 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -97,6 +97,8 @@ if os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") != "1": sys.stderr = HidePrintMessage(sys.stderr) # https://github.com/pytorch/FBGEMM/blob/d99cd96490ec4aabac2ee95b1e76ea4dcfcfa628/fbgemm_gpu/experimental/gemm/triton_gemm/utils.py#L43-L52 sys.stderr.add_filter("TMA benchmarks will be running") + # Skipping import of cpp extensions due to incompatible torch version 2.9.0+cu128 for torchao version 0.15.0 + logging.getLogger("torchao").setLevel(logging.ERROR) # Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype' From 01e8f78f139a728e7a3e2fb8817d393ccde21e45 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 2 Jan 2026 05:05:47 -0800 Subject: [PATCH 050/114] Update import_fixes.py --- unsloth/import_fixes.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index da0fbc613b..f0dde256c1 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -20,6 +20,7 @@ from packaging.version import Version as TrueVersion import re import logging import textwrap +import warnings # We cannot do from unsloth_zoo.log import logger since FBGEMM might cause seg faults. UNSLOTH_ENABLE_LOGGING = os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") in ( @@ -99,6 +100,8 @@ if os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") != "1": sys.stderr.add_filter("TMA benchmarks will be running") # Skipping import of cpp extensions due to incompatible torch version 2.9.0+cu128 for torchao version 0.15.0 logging.getLogger("torchao").setLevel(logging.ERROR) + # SyntaxWarning: invalid escape sequence '\.' + warnings.filterwarnings("ignore", message = "invalid escape sequence", category = SyntaxWarning) # Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype' From c7d5f1569c4509a485258773f274f1599d0953ff Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Fri, 2 Jan 2026 13:58:08 +0000 Subject: [PATCH 051/114] Add helpful error messages for fast_generate when fast_inference=False When users load a model with fast_inference=False but then try to use vLLM-style arguments with fast_generate, they previously got confusing errors. This adds a wrapper that detects common mistakes and provides helpful guidance: - Using sampling_params: explains to use HF generate args instead - Using lora_request: explains LoRA weights are already merged - Passing text strings: shows how to tokenize input first Changes: - Add make_fast_generate_wrapper to _utils.py - Apply wrapper in llama.py when fast_inference=False - Apply wrapper in vision.py when fast_inference=False --- unsloth/models/_utils.py | 56 ++++++++++++++++++++++++++++++++++++++++ unsloth/models/llama.py | 2 +- unsloth/models/vision.py | 2 +- 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 3851e18f92..c1f626ec66 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -73,6 +73,7 @@ __all__ = [ "verify_fp8_support_if_applicable", "_get_inference_mode_context_manager", "hf_login", + "make_fast_generate_wrapper", ] import torch @@ -2378,3 +2379,58 @@ def hf_login(token: Optional[str] = None) -> Optional[str]: except Exception as e: logger.info(f"Failed to login to huggingface using token with error: {e}") return token + + +def make_fast_generate_wrapper(original_generate): + """ + Creates a wrapper around model.generate that checks for incorrect + vLLM-style usage when fast_inference=False. + """ + @functools.wraps(original_generate) + def _fast_generate_wrapper(*args, **kwargs): + # Check for vLLM-specific arguments + if "sampling_params" in kwargs: + raise ValueError( + "Unsloth: `sampling_params` is only supported when `fast_inference=True` (vLLM). " + "Since `fast_inference=False`, use HuggingFace generate arguments instead:\n" + " model.fast_generate(**tokens.to('cuda'), max_new_tokens=64, temperature=1.0, top_p=0.95)" + ) + + if "lora_request" in kwargs: + raise ValueError( + "Unsloth: `lora_request` is only supported when `fast_inference=True` (vLLM). " + "Since `fast_inference=False`, LoRA weights are already merged into the model." + ) + + # Check if first positional argument is a string or list of strings + if len(args) > 0: + first_arg = args[0] + is_string_input = False + + if isinstance(first_arg, str): + is_string_input = True + elif isinstance(first_arg, (list, tuple)) and len(first_arg) > 0: + if isinstance(first_arg[0], str): + is_string_input = True + + if is_string_input: + raise ValueError( + "Unsloth: Passing text strings to `fast_generate` is only supported " + "when `fast_inference=True` (vLLM). Since `fast_inference=False`, you must " + "tokenize the input first:\n\n" + " messages = tokenizer.apply_chat_template(\n" + " [{\"role\": \"user\", \"content\": \"Your prompt here\"}],\n" + " tokenize=True, add_generation_prompt=True,\n" + " return_tensors=\"pt\", return_dict=True\n" + " )\n" + " output = model.fast_generate(\n" + " **messages.to('cuda'),\n" + " max_new_tokens=64,\n" + " temperature=1.0,\n" + " )" + ) + + # Call original generate + return original_generate(*args, **kwargs) + + return _fast_generate_wrapper diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 29d41f4bb1..92d51b73ad 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -2326,7 +2326,7 @@ class FastLlamaModel: attn_implementation = "eager", **kwargs, ) - model.fast_generate = model.generate + model.fast_generate = make_fast_generate_wrapper(model.generate) model.fast_generate_batches = None else: from unsloth_zoo.vllm_utils import ( diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 1924373f67..6c5356e0b9 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -673,7 +673,7 @@ class FastBaseModel: **kwargs, ) if hasattr(model, "generate"): - model.fast_generate = model.generate + model.fast_generate = make_fast_generate_wrapper(model.generate) model.fast_generate_batches = error_out_no_vllm if offload_embedding: if bool( From f23735af0a673994ed005ab8d3f96a9aa8a6aefd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 2 Jan 2026 13:58:49 +0000 Subject: [PATCH 052/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/import_fixes.py | 9 +++++++-- unsloth/models/_utils.py | 5 +++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index f0dde256c1..bb6996a3e3 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -101,7 +101,9 @@ if os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") != "1": # Skipping import of cpp extensions due to incompatible torch version 2.9.0+cu128 for torchao version 0.15.0 logging.getLogger("torchao").setLevel(logging.ERROR) # SyntaxWarning: invalid escape sequence '\.' - warnings.filterwarnings("ignore", message = "invalid escape sequence", category = SyntaxWarning) + warnings.filterwarnings( + "ignore", message = "invalid escape sequence", category = SyntaxWarning + ) # Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype' @@ -549,5 +551,8 @@ def fix_diffusers_warnings(): def fix_huggingface_hub(): # huggingface_hub.is_offline_mode got removed, so add it back import huggingface_hub + if not hasattr(huggingface_hub, "is_offline_mode"): - huggingface_hub.is_offline_mode = lambda: huggingface_hub.constants.HF_HUB_OFFLINE + huggingface_hub.is_offline_mode = ( + lambda: huggingface_hub.constants.HF_HUB_OFFLINE + ) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index c1f626ec66..1cead3afaf 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -2386,6 +2386,7 @@ def make_fast_generate_wrapper(original_generate): Creates a wrapper around model.generate that checks for incorrect vLLM-style usage when fast_inference=False. """ + @functools.wraps(original_generate) def _fast_generate_wrapper(*args, **kwargs): # Check for vLLM-specific arguments @@ -2419,9 +2420,9 @@ def make_fast_generate_wrapper(original_generate): "when `fast_inference=True` (vLLM). Since `fast_inference=False`, you must " "tokenize the input first:\n\n" " messages = tokenizer.apply_chat_template(\n" - " [{\"role\": \"user\", \"content\": \"Your prompt here\"}],\n" + ' [{"role": "user", "content": "Your prompt here"}],\n' " tokenize=True, add_generation_prompt=True,\n" - " return_tensors=\"pt\", return_dict=True\n" + ' return_tensors="pt", return_dict=True\n' " )\n" " output = model.fast_generate(\n" " **messages.to('cuda'),\n" From d688d3f564195b397bcb0bb7bea69061945cedfd Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 2 Jan 2026 06:07:16 -0800 Subject: [PATCH 053/114] Bug fixes --- pyproject.toml | 4 ++-- unsloth/models/_utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index decc0e9f5f..20e3fd847f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ huggingfacenotorch = [ ] huggingface = [ "unsloth[huggingfacenotorch]", - "unsloth_zoo>=2025.12.7", + "unsloth_zoo>=2025.12.8", "torchvision", "unsloth[triton]", ] @@ -523,7 +523,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3 ; ('linux' in sys_platform)", ] colab-new = [ - "unsloth_zoo>=2025.12.7", + "unsloth_zoo>=2025.12.8", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,<=4.57.3", diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 1cead3afaf..545ba4794a 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.12.9" +__version__ = "2025.12.10" __all__ = [ "SUPPORTS_BFLOAT16", From 9a3908c55266f8f41a7d26dd02d701c75d8c8cc5 Mon Sep 17 00:00:00 2001 From: Fizza-Mukhtar Date: Fri, 2 Jan 2026 08:42:59 -0800 Subject: [PATCH 054/114] Make llama.cpp CURL support optional during CMake builds --- unsloth/save.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/unsloth/save.py b/unsloth/save.py index ceb36854d2..29d9cdcaff 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -130,6 +130,10 @@ ALLOWED_QUANTS = { "q3_k_xs": "3-bit extra small quantization", } +def has_curl(): + return shutil.which("curl") is not None + +CURL_FLAG = "-DLLAMA_CURL=ON" if has_curl() else "-DLLAMA_CURL=OFF" def print_quantization_methods(): for key, value in ALLOWED_QUANTS.items(): @@ -879,8 +883,9 @@ def install_llama_cpp_make_non_blocking(): # Uses new CMAKE n_jobs = max(int(psutil.cpu_count()), 1) # Use less CPUs since 1.5x faster check = os.system( - "cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF -DLLAMA_CURL=ON" + f"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}" ) + if check != 0: raise RuntimeError( f"*** Unsloth: Failed compiling llama.cpp using os.system(...) with error {check}. Please report this ASAP!" @@ -991,11 +996,12 @@ def install_llama_cpp_old(version = -10): if try_execute(commands) == "CMAKE": # Instead use CMAKE commands = [ - "cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF -DLLAMA_CURL=ON", + "cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}", f"cmake --build llama.cpp/build --config Release -j{psutil.cpu_count()*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}", "cp llama.cpp/build/bin/llama-* llama.cpp", "rm -rf llama.cpp/build", ] + try_execute(commands) # Check if successful @@ -1037,7 +1043,7 @@ def install_llama_cpp_blocking(use_cuda = False): if try_execute(commands) == "CMAKE": # Instead use CMAKE commands = [ - "cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF -DLLAMA_CURL=ON", + "cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}", f"cmake --build llama.cpp/build --config Release -j{psutil.cpu_count()*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}", "cp llama.cpp/build/bin/llama-* llama.cpp", "rm -rf llama.cpp/build", From 8fa3228590a38379851a770a822adce859c7ba26 Mon Sep 17 00:00:00 2001 From: Fizza-Mukhtar Date: Fri, 2 Jan 2026 08:55:58 -0800 Subject: [PATCH 055/114] Make llama.cpp CURL support optional during CMake builds --- unsloth/save.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unsloth/save.py b/unsloth/save.py index 29d9cdcaff..359010dbe2 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -996,7 +996,7 @@ def install_llama_cpp_old(version = -10): if try_execute(commands) == "CMAKE": # Instead use CMAKE commands = [ - "cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}", + f"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}", f"cmake --build llama.cpp/build --config Release -j{psutil.cpu_count()*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}", "cp llama.cpp/build/bin/llama-* llama.cpp", "rm -rf llama.cpp/build", @@ -1043,7 +1043,7 @@ def install_llama_cpp_blocking(use_cuda = False): if try_execute(commands) == "CMAKE": # Instead use CMAKE commands = [ - "cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}", + f"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}", f"cmake --build llama.cpp/build --config Release -j{psutil.cpu_count()*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}", "cp llama.cpp/build/bin/llama-* llama.cpp", "rm -rf llama.cpp/build", From c50b7499ff2f6f0485c6afb51d6c6aea209ae28e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 2 Jan 2026 16:58:04 +0000 Subject: [PATCH 056/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/save.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/unsloth/save.py b/unsloth/save.py index 359010dbe2..714df7682a 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -130,11 +130,14 @@ ALLOWED_QUANTS = { "q3_k_xs": "3-bit extra small quantization", } + def has_curl(): return shutil.which("curl") is not None + CURL_FLAG = "-DLLAMA_CURL=ON" if has_curl() else "-DLLAMA_CURL=OFF" + def print_quantization_methods(): for key, value in ALLOWED_QUANTS.items(): print(f'"{key}" ==> {value}') From 1ea6585b0ce2faed080c79ab8699b72b683de50c Mon Sep 17 00:00:00 2001 From: Kaitao Yang Date: Sat, 3 Jan 2026 22:38:37 -0800 Subject: [PATCH 057/114] remove redundant code of has_block --- unsloth/utils/attention_dispatch.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py index ccd49dada8..0e5f3c1951 100644 --- a/unsloth/utils/attention_dispatch.py +++ b/unsloth/utils/attention_dispatch.py @@ -219,16 +219,10 @@ def run_attention( ) if config.n_groups != 1 and not requires_grad: - if has_block: - out = out.view(bsz, q_len, config.n_kv_heads, config.n_groups, head_dim) - else: - out = out.view(bsz, q_len, config.n_kv_heads, config.n_groups, head_dim) + out = out.view(bsz, q_len, config.n_kv_heads, config.n_groups, head_dim) out = out.reshape(bsz, q_len, n_heads, head_dim) else: - if has_block: - out = out.view(bsz, q_len, n_heads, head_dim) - else: - out = out.view(bsz, q_len, n_heads, head_dim) + out = out.view(bsz, q_len, n_heads, head_dim) return out else: local_mask = context.attention_mask From 3d15865bbc569ce1e3847caa9191390e53d55ac9 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Sun, 4 Jan 2026 12:21:39 +0000 Subject: [PATCH 058/114] rl.py fixes: buffer reset, safer attribute access, typo fix 1. Auto-reset gradient checkpointing buffers after trainer.train() - Import and call reset_unsloth_gradient_checkpointing_buffers() in prepare_for_training_mode wrapper to free memory after training while keeping buffers ready for subsequent runs 2. Replace eval/exec with safer getattr/setattr - eval(f"trl.trainer.{trainer}") -> getattr(trl.trainer, trainer) - exec(f"...{unwrap} = ...") -> setattr(current_trainer, unwrap, ...) - exec(f"Trainer.prediction_step=...") -> direct assignment 3. Fix psutil.cpu_count() potentially returning None - Change psutil.cpu_count()+4 to (psutil.cpu_count() or 1)+4 - Prevents TypeError on systems where cpu_count() returns None 4. Fix typo: oriignal_is_vlm_text -> original_is_vlm_text --- unsloth/models/rl.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 22189f459c..9ea57e32d3 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -199,15 +199,15 @@ def PatchRL(FastLanguageModel): unwrap = "unwrap_model_for_generation" for trainer in trainers: try: - current_trainer = eval(f"trl.trainer.{trainer}") + current_trainer = getattr(trl.trainer, trainer) except: continue if hasattr(current_trainer, unwrap): try: - exec(f"trl.trainer.{trainer}.{unwrap} = unsloth_{unwrap}") + setattr(current_trainer, unwrap, unsloth_unwrap_model_for_generation) except: continue - exec(f"Trainer.prediction_step=unsloth_prediction_step") + Trainer.prediction_step = unsloth_prediction_step selective_log_softmax = RL_REPLACEMENTS["selective_log_softmax"] @@ -234,6 +234,7 @@ from transformers.training_args import ParallelMode # Also patches W&B since multiple runs must use wandb.finish() import functools from types import MethodType +from unsloth_zoo.gradient_checkpointing import reset_unsloth_gradient_checkpointing_buffers def prepare_for_training_mode(f): @functools.wraps(f) def wrapper(self, *args, **kwargs): @@ -244,6 +245,11 @@ def prepare_for_training_mode(f): # Return inference mode if hasattr(self, 'model') and hasattr(self.model, "for_inference"): self.model.for_inference() + # Reset gradient checkpointing buffers to free memory while staying ready for next run + try: + reset_unsloth_gradient_checkpointing_buffers() + except: + pass # Patch W&B to enable logging on future runs, otherwise it'll overwrite the first run try: import wandb @@ -817,7 +823,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): num_proc_check = ( "if dataset_num_proc is None:\n" " import psutil\n" - " dataset_num_proc = min(max(psutil.cpu_count()+4, 2), 64)\n" + " dataset_num_proc = min(max((psutil.cpu_count() or 1)+4, 2), 64)\n" " memory_gb_left = psutil.virtual_memory().available / (1024**3)\n" " if memory_gb_left <= 4: dataset_num_proc = 1 # Too risky, so set to 1\n" " elif memory_gb_left <= 6: dataset_num_proc = min(2, dataset_num_proc)\n" @@ -994,10 +1000,10 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): # Temporary patch _is_vlm to False # as of 0.22 it only exists in sfttrainer - oriignal_is_vlm_text = "self._is_vlm = True" + original_is_vlm_text = "self._is_vlm = True" new_is_vlm_text = "self._is_vlm = False" RLTrainer_source = RLTrainer_source.replace( - oriignal_is_vlm_text, new_is_vlm_text + original_is_vlm_text, new_is_vlm_text ) # Remove multiple doc strings From 08d619fca1a61774db4f2225b96723c9ffa1573c Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Sun, 4 Jan 2026 12:57:10 +0000 Subject: [PATCH 059/114] Handle older unsloth-zoo without reset_unsloth_gradient_checkpointing_buffers --- unsloth/models/rl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 9ea57e32d3..88aeeda8a1 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -234,7 +234,10 @@ from transformers.training_args import ParallelMode # Also patches W&B since multiple runs must use wandb.finish() import functools from types import MethodType -from unsloth_zoo.gradient_checkpointing import reset_unsloth_gradient_checkpointing_buffers +try: + from unsloth_zoo.gradient_checkpointing import reset_unsloth_gradient_checkpointing_buffers +except: + def reset_unsloth_gradient_checkpointing_buffers(): pass def prepare_for_training_mode(f): @functools.wraps(f) def wrapper(self, *args, **kwargs): From d31ec48a94482fa1265ebc670866acf700d8226a Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Sun, 4 Jan 2026 12:58:45 +0000 Subject: [PATCH 060/114] Fix psutil.cpu_count() potentially returning None in save.py --- unsloth/save.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/unsloth/save.py b/unsloth/save.py index 714df7682a..071e032c53 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -879,12 +879,12 @@ def install_llama_cpp_make_non_blocking(): IS_CMAKE = False if check == 0: # Uses old MAKE - n_jobs = max(int(psutil.cpu_count() * 1.5), 1) + n_jobs = max(int((psutil.cpu_count() or 1) * 1.5), 1) full_command = ["make", "all", "-j" + str(n_jobs), "-C", "llama.cpp"] IS_CMAKE = False else: # Uses new CMAKE - n_jobs = max(int(psutil.cpu_count()), 1) # Use less CPUs since 1.5x faster + n_jobs = max(int(psutil.cpu_count() or 1), 1) # Use less CPUs since 1.5x faster check = os.system( f"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}" ) @@ -994,13 +994,13 @@ def install_llama_cpp_old(version = -10): # Try using MAKE commands = [ "make clean -C llama.cpp", - f"make all -j{psutil.cpu_count()*2} -C llama.cpp", + f"make all -j{(psutil.cpu_count() or 1)*2} -C llama.cpp", ] if try_execute(commands) == "CMAKE": # Instead use CMAKE commands = [ f"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}", - f"cmake --build llama.cpp/build --config Release -j{psutil.cpu_count()*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}", + f"cmake --build llama.cpp/build --config Release -j{(psutil.cpu_count() or 1)*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}", "cp llama.cpp/build/bin/llama-* llama.cpp", "rm -rf llama.cpp/build", ] @@ -1040,14 +1040,14 @@ def install_llama_cpp_blocking(use_cuda = False): "make clean -C llama.cpp", # https://github.com/ggerganov/llama.cpp/issues/7062 # Weirdly GPU conversion for GGUF breaks?? - # f"{use_cuda} make all -j{psutil.cpu_count()*2} -C llama.cpp", - f"make all -j{psutil.cpu_count()*2} -C llama.cpp", + # f"{use_cuda} make all -j{(psutil.cpu_count() or 1)*2} -C llama.cpp", + f"make all -j{(psutil.cpu_count() or 1)*2} -C llama.cpp", ] if try_execute(commands) == "CMAKE": # Instead use CMAKE commands = [ f"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}", - f"cmake --build llama.cpp/build --config Release -j{psutil.cpu_count()*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}", + f"cmake --build llama.cpp/build --config Release -j{(psutil.cpu_count() or 1)*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}", "cp llama.cpp/build/bin/llama-* llama.cpp", "rm -rf llama.cpp/build", ] From 402e7d6285a3610350953c5f2d954fc5c8ddd3d2 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Sun, 4 Jan 2026 13:03:06 +0000 Subject: [PATCH 061/114] Respect user quantization_config --- unsloth/models/loader.py | 152 ++++++++++++++++++++++++++++++--------- unsloth/models/vision.py | 7 +- 2 files changed, 122 insertions(+), 37 deletions(-) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 247c72f43f..645c23d50b 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -151,8 +151,41 @@ class FastLanguageModel(FastLlamaModel): *args, **kwargs, ): + # Respect user-provided quantization_config (e.g. BitsAndBytesConfig) + quantization_config = kwargs.get("quantization_config", None) + if quantization_config is not None: + if getattr(quantization_config, "load_in_4bit", False): + load_in_4bit = True + load_in_8bit = False + if getattr(quantization_config, "load_in_8bit", False): + load_in_8bit = True + load_in_4bit = False + + load_in_4bit_kwargs = load_in_4bit + load_in_8bit_kwargs = load_in_8bit + if quantization_config is not None: + load_in_4bit_kwargs = False + load_in_8bit_kwargs = False + # Login to allow private models token = hf_login(token) + # Align dtype with bnb_4bit_compute_dtype if provided and dtype is unset. + if dtype is None and quantization_config is not None: + bnb_compute_dtype = None + if isinstance(quantization_config, dict): + if quantization_config.get("load_in_4bit", False): + bnb_compute_dtype = quantization_config.get( + "bnb_4bit_compute_dtype", None + ) + else: + if getattr(quantization_config, "load_in_4bit", False): + bnb_compute_dtype = getattr( + quantization_config, "bnb_4bit_compute_dtype", None + ) + if isinstance(bnb_compute_dtype, str): + bnb_compute_dtype = getattr(torch, bnb_compute_dtype, None) + if isinstance(bnb_compute_dtype, torch.dtype): + dtype = bnb_compute_dtype if load_in_8bit or full_finetuning or qat_scheme is not None: return FastModel.from_pretrained( model_name = model_name, @@ -546,7 +579,7 @@ class FastLanguageModel(FastLlamaModel): model_name = model_name, max_seq_length = max_seq_length, dtype = _get_dtype(dtype), - load_in_4bit = load_in_4bit, + load_in_4bit = load_in_4bit_kwargs, token = token, device_map = device_map, rope_scaling = rope_scaling, @@ -583,22 +616,30 @@ class FastLanguageModel(FastLlamaModel): ) if load_in_4bit: - # Fix up bitsandbytes config - compute_dtype = dtype_from_config(model.config) - quantization_config = { - # Sometimes compute_dtype is not a string!! - "bnb_4bit_compute_dtype": compute_dtype, - "bnb_4bit_quant_type": "nf4", - "bnb_4bit_use_double_quant": True, - "llm_int8_enable_fp32_cpu_offload": False, - "llm_int8_has_fp16_weight": False, - "llm_int8_skip_modules": None, - "llm_int8_threshold": 6.0, - "load_in_4bit": True, - "load_in_8bit": False, - "quant_method": "bitsandbytes", - } - model.config.update({"quantization_config": quantization_config}) + # Fix up bitsandbytes config, but respect user-provided quantization_config + if quantization_config is None: + compute_dtype = dtype_from_config(model.config) + quantization_config = { + # Sometimes compute_dtype is not a string!! + "bnb_4bit_compute_dtype": compute_dtype, + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": True, + "llm_int8_enable_fp32_cpu_offload": False, + "llm_int8_has_fp16_weight": False, + "llm_int8_skip_modules": None, + "llm_int8_threshold": 6.0, + "load_in_4bit": True, + "load_in_8bit": False, + "quant_method": "bitsandbytes", + } + model.config.update({"quantization_config": quantization_config}) + else: + if hasattr(quantization_config, "to_dict"): + model.config.update( + {"quantization_config": quantization_config.to_dict()} + ) + elif isinstance(quantization_config, dict): + model.config.update({"quantization_config": quantization_config}) if load_in_fp8 != False: _tag_model_with_fp8_torchao_config(model, fp8_mode) @@ -690,12 +731,45 @@ class FastModel(FastBaseModel): *args, **kwargs, ): + # Respect user-provided quantization_config (e.g. BitsAndBytesConfig) + quantization_config = kwargs.get("quantization_config", None) + if quantization_config is not None: + if getattr(quantization_config, "load_in_4bit", False): + load_in_4bit = True + load_in_8bit = False + if getattr(quantization_config, "load_in_8bit", False): + load_in_8bit = True + load_in_4bit = False + + load_in_4bit_kwargs = load_in_4bit + load_in_8bit_kwargs = load_in_8bit + if quantization_config is not None: + load_in_4bit_kwargs = False + load_in_8bit_kwargs = False + # Login to allow private models token = hf_login(token) if whisper_language is not None: assert type(whisper_language) is str if whisper_task is not None: assert type(whisper_task) is str + # Align dtype with bnb_4bit_compute_dtype if provided and dtype is unset. + if dtype is None and quantization_config is not None: + bnb_compute_dtype = None + if isinstance(quantization_config, dict): + if quantization_config.get("load_in_4bit", False): + bnb_compute_dtype = quantization_config.get( + "bnb_4bit_compute_dtype", None + ) + else: + if getattr(quantization_config, "load_in_4bit", False): + bnb_compute_dtype = getattr( + quantization_config, "bnb_4bit_compute_dtype", None + ) + if isinstance(bnb_compute_dtype, str): + bnb_compute_dtype = getattr(torch, bnb_compute_dtype, None) + if isinstance(bnb_compute_dtype, torch.dtype): + dtype = bnb_compute_dtype SUPPORTS_BFLOAT16 = is_bfloat16_supported() if dtype is None: dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16 @@ -1173,8 +1247,8 @@ class FastModel(FastBaseModel): model_name = model_name, max_seq_length = max_seq_length, dtype = _get_dtype(dtype), - load_in_4bit = load_in_4bit, - load_in_8bit = load_in_8bit, + load_in_4bit = load_in_4bit_kwargs, + load_in_8bit = load_in_8bit_kwargs, load_in_16bit = load_in_16bit, full_finetuning = full_finetuning, token = token, @@ -1220,22 +1294,30 @@ class FastModel(FastBaseModel): ) if load_in_4bit: - # Fix up bitsandbytes config - compute_dtype = dtype_from_config(model.config) - quantization_config = { - # Sometimes compute_dtype is not a string!! - "bnb_4bit_compute_dtype": compute_dtype, - "bnb_4bit_quant_type": "nf4", - "bnb_4bit_use_double_quant": True, - "llm_int8_enable_fp32_cpu_offload": False, - "llm_int8_has_fp16_weight": False, - "llm_int8_skip_modules": None, - "llm_int8_threshold": 6.0, - "load_in_4bit": True, - "load_in_8bit": False, - "quant_method": "bitsandbytes", - } - model.config.update({"quantization_config": quantization_config}) + # Fix up bitsandbytes config, but respect user-provided quantization_config + if quantization_config is None: + compute_dtype = dtype_from_config(model.config) + quantization_config = { + # Sometimes compute_dtype is not a string!! + "bnb_4bit_compute_dtype": compute_dtype, + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": True, + "llm_int8_enable_fp32_cpu_offload": False, + "llm_int8_has_fp16_weight": False, + "llm_int8_skip_modules": None, + "llm_int8_threshold": 6.0, + "load_in_4bit": True, + "load_in_8bit": False, + "quant_method": "bitsandbytes", + } + model.config.update({"quantization_config": quantization_config}) + else: + if hasattr(quantization_config, "to_dict"): + model.config.update( + {"quantization_config": quantization_config.to_dict()} + ) + elif isinstance(quantization_config, dict): + model.config.update({"quantization_config": quantization_config}) if load_in_fp8 != False: _tag_model_with_fp8_torchao_config(model, fp8_mode) diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 6c5356e0b9..6de942d7d2 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -529,6 +529,7 @@ class FastBaseModel: del kwargs["attn_implementation"] bnb_config = None + user_quantization_config = kwargs.get("quantization_config", None) if full_finetuning and (load_in_4bit or load_in_8bit): print( "Unsloth: You selected full finetuning support, but 4bit / 8bit is enabled - disabling LoRA / QLoRA." @@ -596,7 +597,8 @@ class FastBaseModel: ): pass else: - kwargs["quantization_config"] = bnb_config + if user_quantization_config is None: + kwargs["quantization_config"] = bnb_config else: if auto_config is None: auto_config = AutoConfig.from_pretrained( @@ -641,7 +643,8 @@ class FastBaseModel: ) except: pass - kwargs["quantization_config"] = quantization_config + if user_quantization_config is None: + kwargs["quantization_config"] = quantization_config # Check if using forced float32 - we load it in bfloat16, then cast to float16! torch_dtype = dtype From bfa225b00c5faba087bae3f75d8aa55803cacb52 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Sun, 4 Jan 2026 13:14:03 +0000 Subject: [PATCH 062/114] Handle dict quantization_config flags --- unsloth/models/loader.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 645c23d50b..4488ad9a07 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -154,10 +154,16 @@ class FastLanguageModel(FastLlamaModel): # Respect user-provided quantization_config (e.g. BitsAndBytesConfig) quantization_config = kwargs.get("quantization_config", None) if quantization_config is not None: - if getattr(quantization_config, "load_in_4bit", False): + if isinstance(quantization_config, dict): + q_load_in_4bit = quantization_config.get("load_in_4bit", False) + q_load_in_8bit = quantization_config.get("load_in_8bit", False) + else: + q_load_in_4bit = getattr(quantization_config, "load_in_4bit", False) + q_load_in_8bit = getattr(quantization_config, "load_in_8bit", False) + if q_load_in_4bit: load_in_4bit = True load_in_8bit = False - if getattr(quantization_config, "load_in_8bit", False): + if q_load_in_8bit: load_in_8bit = True load_in_4bit = False @@ -734,10 +740,16 @@ class FastModel(FastBaseModel): # Respect user-provided quantization_config (e.g. BitsAndBytesConfig) quantization_config = kwargs.get("quantization_config", None) if quantization_config is not None: - if getattr(quantization_config, "load_in_4bit", False): + if isinstance(quantization_config, dict): + q_load_in_4bit = quantization_config.get("load_in_4bit", False) + q_load_in_8bit = quantization_config.get("load_in_8bit", False) + else: + q_load_in_4bit = getattr(quantization_config, "load_in_4bit", False) + q_load_in_8bit = getattr(quantization_config, "load_in_8bit", False) + if q_load_in_4bit: load_in_4bit = True load_in_8bit = False - if getattr(quantization_config, "load_in_8bit", False): + if q_load_in_8bit: load_in_8bit = True load_in_4bit = False From e22ca346cf687cc79393f0a122ce4c15ba965e63 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Sun, 4 Jan 2026 13:18:15 +0000 Subject: [PATCH 063/114] Keep 4bit flag for fast_inference --- unsloth/models/loader.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 4488ad9a07..eb3b21e206 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -167,12 +167,6 @@ class FastLanguageModel(FastLlamaModel): load_in_8bit = True load_in_4bit = False - load_in_4bit_kwargs = load_in_4bit - load_in_8bit_kwargs = load_in_8bit - if quantization_config is not None: - load_in_4bit_kwargs = False - load_in_8bit_kwargs = False - # Login to allow private models token = hf_login(token) # Align dtype with bnb_4bit_compute_dtype if provided and dtype is unset. @@ -581,6 +575,12 @@ class FastLanguageModel(FastLlamaModel): if fast_inference: fast_inference, model_name = fast_inference_setup(model_name, model_config) + load_in_4bit_kwargs = load_in_4bit + load_in_8bit_kwargs = load_in_8bit + if quantization_config is not None and not fast_inference: + load_in_4bit_kwargs = False + load_in_8bit_kwargs = False + model, tokenizer = dispatch_model.from_pretrained( model_name = model_name, max_seq_length = max_seq_length, @@ -753,12 +753,6 @@ class FastModel(FastBaseModel): load_in_8bit = True load_in_4bit = False - load_in_4bit_kwargs = load_in_4bit - load_in_8bit_kwargs = load_in_8bit - if quantization_config is not None: - load_in_4bit_kwargs = False - load_in_8bit_kwargs = False - # Login to allow private models token = hf_login(token) if whisper_language is not None: @@ -1255,6 +1249,12 @@ class FastModel(FastBaseModel): if auto_model is None: auto_model = AutoModelForVision2Seq if is_vlm else AutoModelForCausalLM + load_in_4bit_kwargs = load_in_4bit + load_in_8bit_kwargs = load_in_8bit + if quantization_config is not None and not fast_inference: + load_in_4bit_kwargs = False + load_in_8bit_kwargs = False + model, tokenizer = FastBaseModel.from_pretrained( model_name = model_name, max_seq_length = max_seq_length, From e63c2744ec0762e8688de50a57a5391216f06a53 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 4 Jan 2026 06:12:44 -0800 Subject: [PATCH 064/114] Versioning --- pyproject.toml | 4 ++-- unsloth/__init__.py | 2 +- unsloth/models/_utils.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 20e3fd847f..e7b84f3c8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ huggingfacenotorch = [ ] huggingface = [ "unsloth[huggingfacenotorch]", - "unsloth_zoo>=2025.12.8", + "unsloth_zoo>=2026.1.1", "torchvision", "unsloth[triton]", ] @@ -523,7 +523,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3 ; ('linux' in sys_platform)", ] colab-new = [ - "unsloth_zoo>=2025.12.8", + "unsloth_zoo>=2026.1.1", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,<=4.57.3", diff --git a/unsloth/__init__.py b/unsloth/__init__.py index c74b248a83..d9633e8ec1 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -79,7 +79,7 @@ from importlib.metadata import PackageNotFoundError # Check for unsloth_zoo try: unsloth_zoo_version = importlib_version("unsloth_zoo") - if Version(unsloth_zoo_version) < Version("2025.12.4"): + if Version(unsloth_zoo_version) < Version("2026.1.1"): print( "Unsloth: Please update Unsloth and Unsloth-Zoo to the latest version!\n" "Do this via `pip install --upgrade --force-reinstall --no-cache-dir --no-deps unsloth unsloth_zoo`" diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 545ba4794a..5952d4af0c 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.12.10" +__version__ = "2026.1.1" __all__ = [ "SUPPORTS_BFLOAT16", From b5addbc936933ad3ca682a0cc2f3eeececfba321 Mon Sep 17 00:00:00 2001 From: Kaitao Yang Date: Sun, 4 Jan 2026 09:21:44 -0800 Subject: [PATCH 065/114] remove unused variable BlockDiagonalCausalMask --- unsloth/utils/attention_dispatch.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py index 0e5f3c1951..a7620549be 100644 --- a/unsloth/utils/attention_dispatch.py +++ b/unsloth/utils/attention_dispatch.py @@ -32,9 +32,6 @@ from ..utils.packing import ( if HAS_FLASH_ATTENTION: from flash_attn import flash_attn_func, flash_attn_varlen_func HAS_XFORMERS = xformers is not None -BlockDiagonalCausalMask = None -if HAS_XFORMERS: - BlockDiagonalCausalMask = xformers.attn_bias.BlockDiagonalCausalMask SDPA_HAS_GQA = "enable_gqa" in (scaled_dot_product_attention.__doc__ or "") FLASH_VARLEN = "flash_varlen" From 6c6d0dfef1bce443b2030dd46f04e9fcbb981dff Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 05:02:53 +0000 Subject: [PATCH 066/114] Fix vLLM PDL bug on Blackwell GPUs (B200/B100) vLLM's LoRA Triton kernels use tl.extra.cuda.gdc_wait() for PDL optimization on SM90+ GPUs. This fails on SM100 (Blackwell) during CUDA graph capture because Triton's pipeliner cannot handle gdc_wait in complex kernels. This fix: - Detects SM100 GPUs and applies the workaround automatically - Sets TRITON_DISABLE_PDL=1 environment variable - Monkey-patches supports_pdl to return False in lora_expand_op and lora_shrink_op - Checks GitHub issue #30872 status (with 3s timeout) to auto-disable the workaround once the upstream fix is merged - Includes quick internet connectivity check (0.5s) to avoid delays when offline Fixes the error: 'tt.elementwise_inline_asm' op pipeliner doesn't know how to predicate this op LLVM ERROR: Fatal pipeliner error See: https://github.com/vllm-project/vllm/issues/30872 --- unsloth/__init__.py | 3 + unsloth/import_fixes.py | 121 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) diff --git a/unsloth/__init__.py b/unsloth/__init__.py index d9633e8ec1..86fb00fe0e 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -126,6 +126,7 @@ from .import_fixes import ( fix_xformers_performance_issue, fix_vllm_aimv2_issue, fix_vllm_guided_decoding_params, + fix_vllm_pdl_blackwell, ignore_logger_messages, patch_ipykernel_hf_xet, patch_trackio, @@ -138,6 +139,7 @@ from .import_fixes import ( fix_xformers_performance_issue() fix_vllm_aimv2_issue() fix_vllm_guided_decoding_params() +fix_vllm_pdl_blackwell() ignore_logger_messages() patch_ipykernel_hf_xet() patch_trackio() @@ -149,6 +151,7 @@ fix_executorch() del fix_xformers_performance_issue del fix_vllm_aimv2_issue del fix_vllm_guided_decoding_params +del fix_vllm_pdl_blackwell del ignore_logger_messages del patch_ipykernel_hf_xet del patch_trackio diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index bb6996a3e3..91ba35e21e 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -556,3 +556,124 @@ def fix_huggingface_hub(): huggingface_hub.is_offline_mode = ( lambda: huggingface_hub.constants.HF_HUB_OFFLINE ) + + +def fix_vllm_pdl_blackwell(): + """ + Fix vLLM PDL (Programmatic Dependent Launch) bug on Blackwell GPUs (SM100). + + The issue: vLLM's LoRA Triton kernels use tl.extra.cuda.gdc_wait() for PDL + optimization on SM90+ GPUs. This fails on SM100 (B200/B100) during CUDA graph + capture because Triton's pipeliner can't handle gdc_wait in complex kernels. + + See: https://github.com/vllm-project/vllm/issues/30872 + """ + if importlib.util.find_spec("vllm") is None: + return + + # Check if we have a CUDA GPU + try: + import torch + if not torch.cuda.is_available(): + return + major, minor = torch.cuda.get_device_capability() + except Exception: + return + + # Only SM100 (Blackwell) is affected - SM90 (Hopper) works fine + if major != 10: + return + + gpu_name = torch.cuda.get_device_name() + + # Check if vLLM has the PDL-related modules before doing internet check + try: + has_expand_op = importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_expand_op") is not None + except (ModuleNotFoundError, ValueError): + has_expand_op = False + try: + has_shrink_op = importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_shrink_op") is not None + except (ModuleNotFoundError, ValueError): + has_shrink_op = False + if not has_expand_op and not has_shrink_op: + # Old vLLM version without PDL support - just set env var to be safe + os.environ["TRITON_DISABLE_PDL"] = "1" + logger.info( + f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM{major}{minor} ({gpu_name}) - " + f"vLLM PDL modules not found" + ) + return + + # Check if GitHub issue is closed (fix merged upstream) + issue_closed = False + try: + import socket + import urllib.request + import json as json_module + + # Quick internet connectivity check (0.5s timeout) + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(0.5) + try: + sock.connect(("api.github.com", 443)) + has_internet = True + except (socket.timeout, OSError): + has_internet = False + finally: + sock.close() + + if has_internet: + api_url = "https://api.github.com/repos/vllm-project/vllm/issues/30872" + req = urllib.request.Request( + api_url, + headers={ + "User-Agent": "Unsloth-PDL-Fix", + "Accept": "application/vnd.github.v3+json", + } + ) + with urllib.request.urlopen(req, timeout=3) as response: + data = json_module.loads(response.read().decode()) + issue_closed = data.get("state") == "closed" + except Exception: + # If we can't check, assume issue is still open (apply fix to be safe) + pass + + if issue_closed: + logger.info( + f"Unsloth: SM{major}{minor} ({gpu_name}) detected but PDL issue #30872 " + f"is closed - skipping PDL fix" + ) + return + + # Apply the PDL fix + os.environ["TRITON_DISABLE_PDL"] = "1" + + def fake_supports_pdl(device=None): + return False + + patched = [] + + try: + import vllm.lora.ops.triton_ops.lora_expand_op as expand_op + expand_op.supports_pdl = fake_supports_pdl + patched.append("lora_expand_op") + except (ImportError, ModuleNotFoundError, AttributeError): + pass + + try: + import vllm.lora.ops.triton_ops.lora_shrink_op as shrink_op + shrink_op.supports_pdl = fake_supports_pdl + patched.append("lora_shrink_op") + except (ImportError, ModuleNotFoundError, AttributeError): + pass + + if patched: + logger.info( + f"Unsloth: Applied PDL fix for SM{major}{minor} ({gpu_name}) - " + f"patched: {', '.join(patched)}" + ) + else: + # Just set the env var - vLLM might be an older version without supports_pdl + logger.info( + f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM{major}{minor} ({gpu_name})" + ) From efe949c941a752e3f1872b84d2fc9b1b54e9358c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 05:03:28 +0000 Subject: [PATCH 067/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/import_fixes.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 91ba35e21e..e8c5e16df4 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -574,6 +574,7 @@ def fix_vllm_pdl_blackwell(): # Check if we have a CUDA GPU try: import torch + if not torch.cuda.is_available(): return major, minor = torch.cuda.get_device_capability() @@ -588,11 +589,17 @@ def fix_vllm_pdl_blackwell(): # Check if vLLM has the PDL-related modules before doing internet check try: - has_expand_op = importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_expand_op") is not None + has_expand_op = ( + importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_expand_op") + is not None + ) except (ModuleNotFoundError, ValueError): has_expand_op = False try: - has_shrink_op = importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_shrink_op") is not None + has_shrink_op = ( + importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_shrink_op") + is not None + ) except (ModuleNotFoundError, ValueError): has_shrink_op = False if not has_expand_op and not has_shrink_op: @@ -626,12 +633,12 @@ def fix_vllm_pdl_blackwell(): api_url = "https://api.github.com/repos/vllm-project/vllm/issues/30872" req = urllib.request.Request( api_url, - headers={ + headers = { "User-Agent": "Unsloth-PDL-Fix", "Accept": "application/vnd.github.v3+json", - } + }, ) - with urllib.request.urlopen(req, timeout=3) as response: + with urllib.request.urlopen(req, timeout = 3) as response: data = json_module.loads(response.read().decode()) issue_closed = data.get("state") == "closed" except Exception: @@ -648,13 +655,14 @@ def fix_vllm_pdl_blackwell(): # Apply the PDL fix os.environ["TRITON_DISABLE_PDL"] = "1" - def fake_supports_pdl(device=None): + def fake_supports_pdl(device = None): return False patched = [] try: import vllm.lora.ops.triton_ops.lora_expand_op as expand_op + expand_op.supports_pdl = fake_supports_pdl patched.append("lora_expand_op") except (ImportError, ModuleNotFoundError, AttributeError): @@ -662,6 +670,7 @@ def fix_vllm_pdl_blackwell(): try: import vllm.lora.ops.triton_ops.lora_shrink_op as shrink_op + shrink_op.supports_pdl = fake_supports_pdl patched.append("lora_shrink_op") except (ImportError, ModuleNotFoundError, AttributeError): From 36c9a841eb959f279b0170041f86e43c7421b514 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 05:03:56 +0000 Subject: [PATCH 068/114] Sync chat_template from tokenizer to vLLM When using base models with custom chat templates applied after loading, vLLM's internal tokenizer may not have the chat_template set. This causes issues during RL training with vLLM inference. This fix syncs the chat_template from the processing_class (the tokenizer you loaded and configured) to vLLM's internal tokenizer during trainer initialization, but only if vLLM's tokenizer does not already have one set. --- unsloth/models/rl.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 88aeeda8a1..20dafaaaa4 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -694,6 +694,20 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): ) RLTrainer_post += training_check + # Sync chat_template from processing_class to vLLM's tokenizer + # This fixes base models that have custom chat templates applied after loading + if "model" in call_args: + vllm_chat_template_sync = ( + "if hasattr(self, 'llm') and self.llm is not None and hasattr(self.llm, 'get_tokenizer'):\n" + " _vllm_tok = self.llm.get_tokenizer()\n" + " _pc = getattr(self, 'processing_class', None)\n" + " if _pc is not None and getattr(_pc, 'chat_template', None) is not None:\n" + " if _vllm_tok.chat_template is None:\n" + " _vllm_tok.chat_template = _pc.chat_template\n" + "pass\n" + ) + RLTrainer_post += vllm_chat_template_sync + # Edit optional metrics other_metrics_processor = "" if trainer_file in RL_METRICS_CHANGES: From fbdb3b524e93ea99d8845696e9c40ce64bef349d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 05:10:24 +0000 Subject: [PATCH 069/114] Add tokenizer fallback for chat_template sync --- unsloth/models/rl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 20dafaaaa4..b75ae383db 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -700,7 +700,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): vllm_chat_template_sync = ( "if hasattr(self, 'llm') and self.llm is not None and hasattr(self.llm, 'get_tokenizer'):\n" " _vllm_tok = self.llm.get_tokenizer()\n" - " _pc = getattr(self, 'processing_class', None)\n" + " _pc = getattr(self, 'processing_class', None) or getattr(self, 'tokenizer', None)\n" " if _pc is not None and getattr(_pc, 'chat_template', None) is not None:\n" " if _vllm_tok.chat_template is None:\n" " _vllm_tok.chat_template = _pc.chat_template\n" From 227c31f0caf3cb30a06a77c0a1b010cc7e09007d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 05:24:52 +0000 Subject: [PATCH 070/114] Address review feedback: refactor and scan all GPUs - Add _spec_exists helper function to reduce duplication - Scan all GPUs for SM100 instead of just device 0 - Use loop for module patching to improve maintainability --- unsloth/import_fixes.py | 85 ++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 44 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index e8c5e16df4..7d368d027a 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -571,42 +571,44 @@ def fix_vllm_pdl_blackwell(): if importlib.util.find_spec("vllm") is None: return - # Check if we have a CUDA GPU + # Check if any CUDA GPU is SM100 (Blackwell) try: import torch if not torch.cuda.is_available(): return - major, minor = torch.cuda.get_device_capability() + + # Scan all GPUs for SM100 - fix applies globally via env var and monkey-patch + has_sm100 = False + sm100_gpu_name = None + for i in range(torch.cuda.device_count()): + major, minor = torch.cuda.get_device_capability(i) + if major == 10: + has_sm100 = True + sm100_gpu_name = torch.cuda.get_device_name(i) + break + + if not has_sm100: + return except Exception: return - # Only SM100 (Blackwell) is affected - SM90 (Hopper) works fine - if major != 10: - return - - gpu_name = torch.cuda.get_device_name() + # Helper to check if module spec exists + def _spec_exists(name): + try: + return importlib.util.find_spec(name) is not None + except (ModuleNotFoundError, ValueError): + return False # Check if vLLM has the PDL-related modules before doing internet check - try: - has_expand_op = ( - importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_expand_op") - is not None - ) - except (ModuleNotFoundError, ValueError): - has_expand_op = False - try: - has_shrink_op = ( - importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_shrink_op") - is not None - ) - except (ModuleNotFoundError, ValueError): - has_shrink_op = False + has_expand_op = _spec_exists("vllm.lora.ops.triton_ops.lora_expand_op") + has_shrink_op = _spec_exists("vllm.lora.ops.triton_ops.lora_shrink_op") + if not has_expand_op and not has_shrink_op: # Old vLLM version without PDL support - just set env var to be safe os.environ["TRITON_DISABLE_PDL"] = "1" logger.info( - f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM{major}{minor} ({gpu_name}) - " + f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM100 ({sm100_gpu_name}) - " f"vLLM PDL modules not found" ) return @@ -633,12 +635,12 @@ def fix_vllm_pdl_blackwell(): api_url = "https://api.github.com/repos/vllm-project/vllm/issues/30872" req = urllib.request.Request( api_url, - headers = { + headers={ "User-Agent": "Unsloth-PDL-Fix", "Accept": "application/vnd.github.v3+json", }, ) - with urllib.request.urlopen(req, timeout = 3) as response: + with urllib.request.urlopen(req, timeout=3) as response: data = json_module.loads(response.read().decode()) issue_closed = data.get("state") == "closed" except Exception: @@ -647,7 +649,7 @@ def fix_vllm_pdl_blackwell(): if issue_closed: logger.info( - f"Unsloth: SM{major}{minor} ({gpu_name}) detected but PDL issue #30872 " + f"Unsloth: SM100 ({sm100_gpu_name}) detected but PDL issue #30872 " f"is closed - skipping PDL fix" ) return @@ -655,34 +657,29 @@ def fix_vllm_pdl_blackwell(): # Apply the PDL fix os.environ["TRITON_DISABLE_PDL"] = "1" - def fake_supports_pdl(device = None): + def fake_supports_pdl(device=None): return False patched = [] - - try: - import vllm.lora.ops.triton_ops.lora_expand_op as expand_op - - expand_op.supports_pdl = fake_supports_pdl - patched.append("lora_expand_op") - except (ImportError, ModuleNotFoundError, AttributeError): - pass - - try: - import vllm.lora.ops.triton_ops.lora_shrink_op as shrink_op - - shrink_op.supports_pdl = fake_supports_pdl - patched.append("lora_shrink_op") - except (ImportError, ModuleNotFoundError, AttributeError): - pass + modules_to_patch = { + "lora_expand_op": "vllm.lora.ops.triton_ops.lora_expand_op", + "lora_shrink_op": "vllm.lora.ops.triton_ops.lora_shrink_op", + } + for name, path in modules_to_patch.items(): + try: + module = importlib.import_module(path) + module.supports_pdl = fake_supports_pdl + patched.append(name) + except (ImportError, ModuleNotFoundError, AttributeError): + pass if patched: logger.info( - f"Unsloth: Applied PDL fix for SM{major}{minor} ({gpu_name}) - " + f"Unsloth: Applied PDL fix for SM100 ({sm100_gpu_name}) - " f"patched: {', '.join(patched)}" ) else: # Just set the env var - vLLM might be an older version without supports_pdl logger.info( - f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM{major}{minor} ({gpu_name})" + f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM100 ({sm100_gpu_name})" ) From eac1f6b0101ce12ae3d630160f9e3d8593a70779 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 05:24:59 +0000 Subject: [PATCH 071/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/import_fixes.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 7d368d027a..77693d4cf3 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -635,12 +635,12 @@ def fix_vllm_pdl_blackwell(): api_url = "https://api.github.com/repos/vllm-project/vllm/issues/30872" req = urllib.request.Request( api_url, - headers={ + headers = { "User-Agent": "Unsloth-PDL-Fix", "Accept": "application/vnd.github.v3+json", }, ) - with urllib.request.urlopen(req, timeout=3) as response: + with urllib.request.urlopen(req, timeout = 3) as response: data = json_module.loads(response.read().decode()) issue_closed = data.get("state") == "closed" except Exception: @@ -657,7 +657,7 @@ def fix_vllm_pdl_blackwell(): # Apply the PDL fix os.environ["TRITON_DISABLE_PDL"] = "1" - def fake_supports_pdl(device=None): + def fake_supports_pdl(device = None): return False patched = [] @@ -680,6 +680,4 @@ def fix_vllm_pdl_blackwell(): ) else: # Just set the env var - vLLM might be an older version without supports_pdl - logger.info( - f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM100 ({sm100_gpu_name})" - ) + logger.info(f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM100 ({sm100_gpu_name})") From ba548ff8c22b055c5acaa02eb0d67c2e238413ce Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 05:25:53 +0000 Subject: [PATCH 072/114] Combine nested if statements for clarity --- unsloth/models/rl.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index b75ae383db..e1ecd6df2f 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -701,9 +701,8 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): "if hasattr(self, 'llm') and self.llm is not None and hasattr(self.llm, 'get_tokenizer'):\n" " _vllm_tok = self.llm.get_tokenizer()\n" " _pc = getattr(self, 'processing_class', None) or getattr(self, 'tokenizer', None)\n" - " if _pc is not None and getattr(_pc, 'chat_template', None) is not None:\n" - " if _vllm_tok.chat_template is None:\n" - " _vllm_tok.chat_template = _pc.chat_template\n" + " if _pc is not None and getattr(_pc, 'chat_template', None) is not None and _vllm_tok.chat_template is None:\n" + " _vllm_tok.chat_template = _pc.chat_template\n" "pass\n" ) RLTrainer_post += vllm_chat_template_sync From 35219633ab161f062a826f84b037ac18f6390e7e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 06:53:42 +0000 Subject: [PATCH 073/114] Fix PDL patch: target utils.py source module and clear lru_cache - Patch vllm.lora.ops.triton_ops.utils directly where supports_pdl is defined - Clear lru_cache before patching to prevent stale cached results - Add fused_moe_lora_op to consumer modules list - Use *args, **kwargs in fake function for compatibility --- unsloth/import_fixes.py | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 77693d4cf3..469674b29e 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -601,10 +601,11 @@ def fix_vllm_pdl_blackwell(): return False # Check if vLLM has the PDL-related modules before doing internet check + has_utils = _spec_exists("vllm.lora.ops.triton_ops.utils") has_expand_op = _spec_exists("vllm.lora.ops.triton_ops.lora_expand_op") has_shrink_op = _spec_exists("vllm.lora.ops.triton_ops.lora_shrink_op") - if not has_expand_op and not has_shrink_op: + if not has_utils and not has_expand_op and not has_shrink_op: # Old vLLM version without PDL support - just set env var to be safe os.environ["TRITON_DISABLE_PDL"] = "1" logger.info( @@ -657,19 +658,39 @@ def fix_vllm_pdl_blackwell(): # Apply the PDL fix os.environ["TRITON_DISABLE_PDL"] = "1" - def fake_supports_pdl(device = None): + def fake_supports_pdl(*args, **kwargs): return False patched = [] - modules_to_patch = { + + # First, patch the source module (utils.py) where supports_pdl is defined. + # This is critical because supports_pdl uses @lru_cache - we must clear the + # cache to prevent stale cached results from the original function. + try: + utils_module = importlib.import_module("vllm.lora.ops.triton_ops.utils") + if hasattr(utils_module, "supports_pdl"): + original_fn = utils_module.supports_pdl + if hasattr(original_fn, "cache_clear"): + original_fn.cache_clear() + utils_module.supports_pdl = fake_supports_pdl + patched.append("utils") + except (ImportError, ModuleNotFoundError, AttributeError): + pass + + # Also patch the consumer modules that import supports_pdl from utils. + # This ensures the patched function is used even if the module was already + # imported before this fix runs. + consumer_modules = { "lora_expand_op": "vllm.lora.ops.triton_ops.lora_expand_op", "lora_shrink_op": "vllm.lora.ops.triton_ops.lora_shrink_op", + "fused_moe_lora_op": "vllm.lora.ops.triton_ops.fused_moe_lora_op", } - for name, path in modules_to_patch.items(): + for name, path in consumer_modules.items(): try: module = importlib.import_module(path) - module.supports_pdl = fake_supports_pdl - patched.append(name) + if hasattr(module, "supports_pdl"): + module.supports_pdl = fake_supports_pdl + patched.append(name) except (ImportError, ModuleNotFoundError, AttributeError): pass From b9bbf4771002ce67b119c8f1ebe4eb0b9087866e Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 5 Jan 2026 07:02:36 +0000 Subject: [PATCH 074/114] Improve TRL compatibility and GRPO state restore --- unsloth/kernels/cross_entropy_loss.py | 2 +- unsloth/models/cohere.py | 4 +- unsloth/models/gemma.py | 4 +- unsloth/models/gemma2.py | 4 +- unsloth/models/granite.py | 4 +- unsloth/models/loader_utils.py | 1 - unsloth/models/rl.py | 65 ++++++++++++++++++++++++--- unsloth/trainer.py | 4 +- 8 files changed, 67 insertions(+), 21 deletions(-) diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.py index 912e6f7e3f..fbb14013ff 100644 --- a/unsloth/kernels/cross_entropy_loss.py +++ b/unsloth/kernels/cross_entropy_loss.py @@ -24,7 +24,7 @@ from .utils import ( is_cdna, ) from transformers.models.llama.modeling_llama import logger -from packaging.version import Version +from unsloth_zoo.utils import Version from unsloth_zoo.loss_utils import ( patch_loss_functions as _patch_loss_functions, diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.py index e9f56763d6..c33317ee02 100644 --- a/unsloth/models/cohere.py +++ b/unsloth/models/cohere.py @@ -15,7 +15,7 @@ from .llama import * from ._utils import __version__ from unsloth_zoo.hf_utils import dtype_from_config -from unsloth_zoo.utils import _get_dtype +from unsloth_zoo.utils import _get_dtype, Version from ..utils.packing import get_packed_info_from_kwargs from ..utils.attention_dispatch import ( AttentionConfig, @@ -35,8 +35,6 @@ try: repeat_kv, ) except: - from packaging.version import Version - transformers_version = Version(transformers_version) if not transformers_version >= Version("4.42"): raise ImportError( diff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.py index 291d442673..1789a9cd92 100644 --- a/unsloth/models/gemma.py +++ b/unsloth/models/gemma.py @@ -14,7 +14,7 @@ from .llama import * from ._utils import __version__ -from unsloth_zoo.utils import _get_dtype +from unsloth_zoo.utils import _get_dtype, Version from unsloth_zoo.hf_utils import dtype_from_config from ..utils.packing import ( build_sdpa_packed_attention_mask, @@ -34,8 +34,6 @@ try: repeat_kv, ) except: - from packaging.version import Version - transformers_version = Version(transformers_version) if not transformers_version >= Version("4.38"): raise ImportError( diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.py index 4b2503b8a1..16d04955d3 100644 --- a/unsloth/models/gemma2.py +++ b/unsloth/models/gemma2.py @@ -14,7 +14,7 @@ from .llama import * from ._utils import __version__ -from unsloth_zoo.utils import _get_dtype +from unsloth_zoo.utils import _get_dtype, Version from unsloth_zoo.hf_utils import dtype_from_config from ..utils.packing import get_packed_info_from_kwargs from ..utils.attention_dispatch import ( @@ -41,8 +41,6 @@ try: repeat_kv, ) except: - from packaging.version import Version - transformers_version = Version(transformers_version) if not transformers_version >= Version("4.42"): raise ImportError( diff --git a/unsloth/models/granite.py b/unsloth/models/granite.py index f85f1b641f..aae746aed1 100644 --- a/unsloth/models/granite.py +++ b/unsloth/models/granite.py @@ -15,7 +15,7 @@ from .llama import * import os from ._utils import __version__ -from unsloth_zoo.utils import _get_dtype +from unsloth_zoo.utils import _get_dtype, Version from unsloth_zoo.hf_utils import dtype_from_config from ..utils.packing import get_packed_info_from_kwargs from ..utils.attention_dispatch import ( @@ -41,8 +41,6 @@ try: GraniteForCausalLM, ) except: - from packaging.version import Version - transformers_version = Version(transformers_version) if not transformers_version >= Version("4.45.0"): raise ImportError( diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 85332e1116..fe2a89d893 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -28,7 +28,6 @@ from .mapper import ( ) # https://github.com/huggingface/transformers/pull/26037 allows 4 bit loading! -from packaging.version import Version from transformers import __version__ as transformers_version from unsloth.models._utils import TorchAOConfig from unsloth_zoo.utils import Version diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 88aeeda8a1..35a15d03a6 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -43,10 +43,28 @@ torch_compile_options = { "triton.cudagraphs": False, } -from trl import __version__ as trl_version +# vLLM compatibility shim (TRL expects GuidedDecodingParams even if vLLM doesn't provide it) +try: + import vllm.sampling_params as _unsloth_vllm_sp + if not hasattr(_unsloth_vllm_sp, "GuidedDecodingParams"): + class GuidedDecodingParams: + def __init__(self, **kwargs): + self.kwargs = kwargs + _unsloth_vllm_sp.GuidedDecodingParams = GuidedDecodingParams +except Exception: + pass + +from trl import __version__ as trl_version_raw +from importlib.metadata import version as importlib_version from unsloth_zoo.utils import Version -trl_version = Version(trl_version) +try: + trl_version = Version(trl_version_raw) +except Exception: + try: + trl_version = Version(importlib_version("trl")) + except Exception: + trl_version = Version("0.0.0") def vLLMSamplingParams(**kwargs): @@ -220,7 +238,7 @@ RLTrainer_replacement = ''' import os from typing import * from dataclasses import dataclass, field -from packaging.version import Version +from unsloth_zoo.utils import Version import torch import numpy as np from contextlib import nullcontext @@ -242,12 +260,18 @@ def prepare_for_training_mode(f): @functools.wraps(f) def wrapper(self, *args, **kwargs): # Enable training mode + _was_training = None + if hasattr(self, 'model') and hasattr(self.model, "training"): + _was_training = self.model.training if hasattr(self, 'model') and hasattr(self.model, "for_training"): self.model.for_training() output = f(self, *args, **kwargs) - # Return inference mode + # Restore previous mode when possible if hasattr(self, 'model') and hasattr(self.model, "for_inference"): - self.model.for_inference() + if _was_training is False: + self.model.for_inference() + elif _was_training is True and hasattr(self.model, "for_training"): + self.model.for_training() # Reset gradient checkpointing buffers to free memory while staying ready for next run try: reset_unsloth_gradient_checkpointing_buffers() @@ -331,6 +355,27 @@ class Unsloth{RLTrainer_name}(_Unsloth{RLTrainer_name}): pass ''' +def _wrap_grpo_generate_and_score(trainer_cls): + if not hasattr(trainer_cls, "_generate_and_score_completions"): + return + original = trainer_cls._generate_and_score_completions + if getattr(original, "_unsloth_restore_training_wrapped", False): + return + + def wrapped(self, *args, **kwargs): + was_training = getattr(getattr(self, "model", None), "training", None) + try: + return original(self, *args, **kwargs) + finally: + if was_training is False and hasattr(self, "model") and hasattr(self.model, "for_inference"): + try: + self.model.for_inference() + except Exception: + pass + + wrapped._unsloth_restore_training_wrapped = True + trainer_cls._generate_and_score_completions = wrapped + def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): # Patch for vLLM and Unsloth PEFT @@ -1059,6 +1104,16 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): globals(), ) + if trainer_file == "grpo_trainer": + try: + _wrap_grpo_generate_and_score( + getattr(created_module, f"Unsloth{RLTrainer_name}") + ) + except Exception as e: + logger.info( + f"Unsloth: Could not wrap _generate_and_score_completions for {RLTrainer_name}: {e}" + ) + def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, imports): init = inspect.getsource(RLTrainer.__init__) diff --git a/unsloth/trainer.py b/unsloth/trainer.py index 0d98cff305..858dcf2cd3 100644 --- a/unsloth/trainer.py +++ b/unsloth/trainer.py @@ -211,7 +211,7 @@ def _backwards_compatible_trainer(trainer_class, config_class): if "processing_class" in trainer_params and "tokenizer" in kwargs: kwargs["processing_class"] = kwargs.pop("tokenizer") - if ("args" in kwargs) and (Version(trl.__version__) >= Version("0.13.0.dev0")): + if ("args" in kwargs) and (Version(trl) >= Version("0.13.0.dev0")): training_args = kwargs.pop("args", None) # Get parameters that Trainer.__init__ actually expects @@ -412,7 +412,7 @@ def _patch_trl_trainer(): if hasattr(trl, "__UNSLOTH_BACKWARDS_COMPATIBLE__"): return - if Version(trl.__version__) <= Version("0.11.0"): + if Version(trl) <= Version("0.11.0"): return import trl.trainer From 1a9543fadd80e5df817690360ac80da000083064 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 07:03:34 +0000 Subject: [PATCH 075/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/rl.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 35a15d03a6..23cbbf0256 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -46,10 +46,13 @@ torch_compile_options = { # vLLM compatibility shim (TRL expects GuidedDecodingParams even if vLLM doesn't provide it) try: import vllm.sampling_params as _unsloth_vllm_sp + if not hasattr(_unsloth_vllm_sp, "GuidedDecodingParams"): + class GuidedDecodingParams: def __init__(self, **kwargs): self.kwargs = kwargs + _unsloth_vllm_sp.GuidedDecodingParams = GuidedDecodingParams except Exception: pass @@ -355,6 +358,7 @@ class Unsloth{RLTrainer_name}(_Unsloth{RLTrainer_name}): pass ''' + def _wrap_grpo_generate_and_score(trainer_cls): if not hasattr(trainer_cls, "_generate_and_score_completions"): return @@ -367,7 +371,11 @@ def _wrap_grpo_generate_and_score(trainer_cls): try: return original(self, *args, **kwargs) finally: - if was_training is False and hasattr(self, "model") and hasattr(self.model, "for_inference"): + if ( + was_training is False + and hasattr(self, "model") + and hasattr(self.model, "for_inference") + ): try: self.model.for_inference() except Exception: From aff2dc9061faba13bae73035ac47b780a21c60fb Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 10:02:11 +0000 Subject: [PATCH 076/114] Add None check for vLLM tokenizer - Check _vllm_tok is not None before accessing attributes - Use getattr for safer chat_template access --- unsloth/models/rl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index e1ecd6df2f..fd0c69bb0e 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -701,7 +701,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): "if hasattr(self, 'llm') and self.llm is not None and hasattr(self.llm, 'get_tokenizer'):\n" " _vllm_tok = self.llm.get_tokenizer()\n" " _pc = getattr(self, 'processing_class', None) or getattr(self, 'tokenizer', None)\n" - " if _pc is not None and getattr(_pc, 'chat_template', None) is not None and _vllm_tok.chat_template is None:\n" + " if _vllm_tok is not None and _pc is not None and getattr(_pc, 'chat_template', None) is not None and getattr(_vllm_tok, 'chat_template', None) is None:\n" " _vllm_tok.chat_template = _pc.chat_template\n" "pass\n" ) From 6bf555a34c17820f3931f2e9ebfe8c9fb4fee229 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 12:32:16 +0000 Subject: [PATCH 077/114] Remove unnecessary PDL module existence check Old vLLM versions without PDL modules don't need the fix. The patching code already handles missing modules gracefully. --- unsloth/import_fixes.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 469674b29e..ef647ec65a 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -593,27 +593,6 @@ def fix_vllm_pdl_blackwell(): except Exception: return - # Helper to check if module spec exists - def _spec_exists(name): - try: - return importlib.util.find_spec(name) is not None - except (ModuleNotFoundError, ValueError): - return False - - # Check if vLLM has the PDL-related modules before doing internet check - has_utils = _spec_exists("vllm.lora.ops.triton_ops.utils") - has_expand_op = _spec_exists("vllm.lora.ops.triton_ops.lora_expand_op") - has_shrink_op = _spec_exists("vllm.lora.ops.triton_ops.lora_shrink_op") - - if not has_utils and not has_expand_op and not has_shrink_op: - # Old vLLM version without PDL support - just set env var to be safe - os.environ["TRITON_DISABLE_PDL"] = "1" - logger.info( - f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM100 ({sm100_gpu_name}) - " - f"vLLM PDL modules not found" - ) - return - # Check if GitHub issue is closed (fix merged upstream) issue_closed = False try: From 9b6d536e0ee0ccc8eb2ddb9bb733044b182fd13e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 12:34:32 +0000 Subject: [PATCH 078/114] Keep PDL module check but remove unnecessary env var setting The check skips the GitHub API call for old vLLM versions. No need to set TRITON_DISABLE_PDL for versions without PDL support. --- unsloth/import_fixes.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index ef647ec65a..e8c4a2f665 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -593,6 +593,22 @@ def fix_vllm_pdl_blackwell(): except Exception: return + # Helper to check if module spec exists + def _spec_exists(name): + try: + return importlib.util.find_spec(name) is not None + except (ModuleNotFoundError, ValueError): + return False + + # Check if vLLM has the PDL-related modules before doing internet check + has_utils = _spec_exists("vllm.lora.ops.triton_ops.utils") + has_expand_op = _spec_exists("vllm.lora.ops.triton_ops.lora_expand_op") + has_shrink_op = _spec_exists("vllm.lora.ops.triton_ops.lora_shrink_op") + + if not has_utils and not has_expand_op and not has_shrink_op: + # Old vLLM version without PDL support - nothing to patch + return + # Check if GitHub issue is closed (fix merged upstream) issue_closed = False try: From 9ced3523aa73b9161ec87b2f9c2a62c3d0378b7a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 13:15:17 +0000 Subject: [PATCH 079/114] Replace GitHub API check with vLLM version check for PDL fix The GitHub issue check had issues: 1. Network latency on import 2. Issue being closed does not mean the fix is in the installed vLLM version Now skip the PDL workaround if vLLM version > 0.13.2, which is when the upstream fix is expected to be included. --- unsloth/import_fixes.py | 43 +++++++---------------------------------- 1 file changed, 7 insertions(+), 36 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index e8c4a2f665..86a504b7b2 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -609,47 +609,18 @@ def fix_vllm_pdl_blackwell(): # Old vLLM version without PDL support - nothing to patch return - # Check if GitHub issue is closed (fix merged upstream) - issue_closed = False + # Check if vLLM version includes the fix (expected in versions > 0.13.2) try: - import socket - import urllib.request - import json as json_module - - # Quick internet connectivity check (0.5s timeout) - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(0.5) - try: - sock.connect(("api.github.com", 443)) - has_internet = True - except (socket.timeout, OSError): - has_internet = False - finally: - sock.close() - - if has_internet: - api_url = "https://api.github.com/repos/vllm-project/vllm/issues/30872" - req = urllib.request.Request( - api_url, - headers = { - "User-Agent": "Unsloth-PDL-Fix", - "Accept": "application/vnd.github.v3+json", - }, + vllm_version = Version(importlib_version("vllm")) + if vllm_version > Version("0.13.2"): + logger.info( + f"Unsloth: SM100 ({sm100_gpu_name}) detected but vLLM {vllm_version} " + f"should include PDL fix - skipping workaround" ) - with urllib.request.urlopen(req, timeout = 3) as response: - data = json_module.loads(response.read().decode()) - issue_closed = data.get("state") == "closed" + return except Exception: - # If we can't check, assume issue is still open (apply fix to be safe) pass - if issue_closed: - logger.info( - f"Unsloth: SM100 ({sm100_gpu_name}) detected but PDL issue #30872 " - f"is closed - skipping PDL fix" - ) - return - # Apply the PDL fix os.environ["TRITON_DISABLE_PDL"] = "1" From cb42ce8dae17efa1f52c09104703da180e3eadd3 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 13:19:37 +0000 Subject: [PATCH 080/114] Address review feedback: add constant and debug logging --- unsloth/import_fixes.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 86a504b7b2..958173213d 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -609,17 +609,18 @@ def fix_vllm_pdl_blackwell(): # Old vLLM version without PDL support - nothing to patch return - # Check if vLLM version includes the fix (expected in versions > 0.13.2) + # Check if vLLM version includes the fix + VLLM_PDL_FIX_VERSION = "0.13.2" try: vllm_version = Version(importlib_version("vllm")) - if vllm_version > Version("0.13.2"): + if vllm_version > Version(VLLM_PDL_FIX_VERSION): logger.info( f"Unsloth: SM100 ({sm100_gpu_name}) detected but vLLM {vllm_version} " f"should include PDL fix - skipping workaround" ) return - except Exception: - pass + except Exception as e: + logger.debug(f"Unsloth: vLLM version check failed ({e}), applying PDL workaround.") # Apply the PDL fix os.environ["TRITON_DISABLE_PDL"] = "1" From c612bfe3a3472df436e951aee2930185d334b3c4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 13:19:44 +0000 Subject: [PATCH 081/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/import_fixes.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 958173213d..1e05e462e9 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -620,7 +620,9 @@ def fix_vllm_pdl_blackwell(): ) return except Exception as e: - logger.debug(f"Unsloth: vLLM version check failed ({e}), applying PDL workaround.") + logger.debug( + f"Unsloth: vLLM version check failed ({e}), applying PDL workaround." + ) # Apply the PDL fix os.environ["TRITON_DISABLE_PDL"] = "1" From dc986cd7e2aa6ca24c100e3b8791956f966e5ca0 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 5 Jan 2026 13:29:58 +0000 Subject: [PATCH 082/114] Drop rl.py GRPO changes from this branch --- unsloth/models/rl.py | 86 ++++++++++---------------------------------- 1 file changed, 18 insertions(+), 68 deletions(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 23cbbf0256..fd0c69bb0e 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -43,31 +43,10 @@ torch_compile_options = { "triton.cudagraphs": False, } -# vLLM compatibility shim (TRL expects GuidedDecodingParams even if vLLM doesn't provide it) -try: - import vllm.sampling_params as _unsloth_vllm_sp - - if not hasattr(_unsloth_vllm_sp, "GuidedDecodingParams"): - - class GuidedDecodingParams: - def __init__(self, **kwargs): - self.kwargs = kwargs - - _unsloth_vllm_sp.GuidedDecodingParams = GuidedDecodingParams -except Exception: - pass - -from trl import __version__ as trl_version_raw -from importlib.metadata import version as importlib_version +from trl import __version__ as trl_version from unsloth_zoo.utils import Version -try: - trl_version = Version(trl_version_raw) -except Exception: - try: - trl_version = Version(importlib_version("trl")) - except Exception: - trl_version = Version("0.0.0") +trl_version = Version(trl_version) def vLLMSamplingParams(**kwargs): @@ -241,7 +220,7 @@ RLTrainer_replacement = ''' import os from typing import * from dataclasses import dataclass, field -from unsloth_zoo.utils import Version +from packaging.version import Version import torch import numpy as np from contextlib import nullcontext @@ -263,18 +242,12 @@ def prepare_for_training_mode(f): @functools.wraps(f) def wrapper(self, *args, **kwargs): # Enable training mode - _was_training = None - if hasattr(self, 'model') and hasattr(self.model, "training"): - _was_training = self.model.training if hasattr(self, 'model') and hasattr(self.model, "for_training"): self.model.for_training() output = f(self, *args, **kwargs) - # Restore previous mode when possible + # Return inference mode if hasattr(self, 'model') and hasattr(self.model, "for_inference"): - if _was_training is False: - self.model.for_inference() - elif _was_training is True and hasattr(self.model, "for_training"): - self.model.for_training() + self.model.for_inference() # Reset gradient checkpointing buffers to free memory while staying ready for next run try: reset_unsloth_gradient_checkpointing_buffers() @@ -359,32 +332,6 @@ pass ''' -def _wrap_grpo_generate_and_score(trainer_cls): - if not hasattr(trainer_cls, "_generate_and_score_completions"): - return - original = trainer_cls._generate_and_score_completions - if getattr(original, "_unsloth_restore_training_wrapped", False): - return - - def wrapped(self, *args, **kwargs): - was_training = getattr(getattr(self, "model", None), "training", None) - try: - return original(self, *args, **kwargs) - finally: - if ( - was_training is False - and hasattr(self, "model") - and hasattr(self.model, "for_inference") - ): - try: - self.model.for_inference() - except Exception: - pass - - wrapped._unsloth_restore_training_wrapped = True - trainer_cls._generate_and_score_completions = wrapped - - def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): # Patch for vLLM and Unsloth PEFT import trl @@ -747,6 +694,19 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): ) RLTrainer_post += training_check + # Sync chat_template from processing_class to vLLM's tokenizer + # This fixes base models that have custom chat templates applied after loading + if "model" in call_args: + vllm_chat_template_sync = ( + "if hasattr(self, 'llm') and self.llm is not None and hasattr(self.llm, 'get_tokenizer'):\n" + " _vllm_tok = self.llm.get_tokenizer()\n" + " _pc = getattr(self, 'processing_class', None) or getattr(self, 'tokenizer', None)\n" + " if _vllm_tok is not None and _pc is not None and getattr(_pc, 'chat_template', None) is not None and getattr(_vllm_tok, 'chat_template', None) is None:\n" + " _vllm_tok.chat_template = _pc.chat_template\n" + "pass\n" + ) + RLTrainer_post += vllm_chat_template_sync + # Edit optional metrics other_metrics_processor = "" if trainer_file in RL_METRICS_CHANGES: @@ -1112,16 +1072,6 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): globals(), ) - if trainer_file == "grpo_trainer": - try: - _wrap_grpo_generate_and_score( - getattr(created_module, f"Unsloth{RLTrainer_name}") - ) - except Exception as e: - logger.info( - f"Unsloth: Could not wrap _generate_and_score_completions for {RLTrainer_name}: {e}" - ) - def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, imports): init = inspect.getsource(RLTrainer.__init__) From 7fd3a6c177fc242e94b16aee861bb541ae0c25c6 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 5 Jan 2026 13:39:03 +0000 Subject: [PATCH 083/114] Restore TRL version fallback in rl.py --- unsloth/models/rl.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index fd0c69bb0e..11a5215c99 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -43,10 +43,28 @@ torch_compile_options = { "triton.cudagraphs": False, } -from trl import __version__ as trl_version +# vLLM compatibility shim (TRL expects GuidedDecodingParams even if vLLM doesn't provide it) +try: + import vllm.sampling_params as _unsloth_vllm_sp + if not hasattr(_unsloth_vllm_sp, "GuidedDecodingParams"): + class GuidedDecodingParams: + def __init__(self, **kwargs): + self.kwargs = kwargs + _unsloth_vllm_sp.GuidedDecodingParams = GuidedDecodingParams +except Exception: + pass + +from trl import __version__ as trl_version_raw +from importlib.metadata import version as importlib_version from unsloth_zoo.utils import Version -trl_version = Version(trl_version) +try: + trl_version = Version(trl_version_raw) +except Exception: + try: + trl_version = Version(importlib_version("trl")) + except Exception: + trl_version = Version("0.0.0") def vLLMSamplingParams(**kwargs): From 27e9a672a2ab2ff0d878c27e319646c3f03586a0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 13:39:15 +0000 Subject: [PATCH 084/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/rl.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 11a5215c99..2f6aef2709 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -46,10 +46,13 @@ torch_compile_options = { # vLLM compatibility shim (TRL expects GuidedDecodingParams even if vLLM doesn't provide it) try: import vllm.sampling_params as _unsloth_vllm_sp + if not hasattr(_unsloth_vllm_sp, "GuidedDecodingParams"): + class GuidedDecodingParams: def __init__(self, **kwargs): self.kwargs = kwargs + _unsloth_vllm_sp.GuidedDecodingParams = GuidedDecodingParams except Exception: pass From 506bcc48e54c43ca642f5fc0475ca9d5caf545d8 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 5 Jan 2026 13:50:48 +0000 Subject: [PATCH 085/114] Fix GRPO training state restoration --- unsloth/models/rl.py | 46 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 4ea36519d9..1327208c46 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -238,12 +238,18 @@ def prepare_for_training_mode(f): @functools.wraps(f) def wrapper(self, *args, **kwargs): # Enable training mode + _was_training = None + if hasattr(self, 'model') and hasattr(self.model, "training"): + _was_training = self.model.training if hasattr(self, 'model') and hasattr(self.model, "for_training"): self.model.for_training() output = f(self, *args, **kwargs) - # Return inference mode + # Restore previous mode when possible if hasattr(self, 'model') and hasattr(self.model, "for_inference"): - self.model.for_inference() + if _was_training is False: + self.model.for_inference() + elif _was_training is True and hasattr(self.model, "for_training"): + self.model.for_training() # Patch W&B to enable logging on future runs, otherwise it'll overwrite the first run try: import wandb @@ -323,6 +329,32 @@ pass ''' +def _wrap_grpo_generate_and_score(trainer_cls): + if not hasattr(trainer_cls, "_generate_and_score_completions"): + return + original = trainer_cls._generate_and_score_completions + if getattr(original, "_unsloth_restore_training_wrapped", False): + return + + def wrapped(self, *args, **kwargs): + was_training = getattr(getattr(self, "model", None), "training", None) + try: + return original(self, *args, **kwargs) + finally: + if ( + was_training is False + and hasattr(self, "model") + and hasattr(self.model, "for_inference") + ): + try: + self.model.for_inference() + except Exception: + pass + + wrapped._unsloth_restore_training_wrapped = True + trainer_cls._generate_and_score_completions = wrapped + + def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): # Patch for vLLM and Unsloth PEFT import trl @@ -1046,6 +1078,16 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): globals(), ) + if trainer_file == "grpo_trainer": + try: + _wrap_grpo_generate_and_score( + getattr(created_module, f"Unsloth{RLTrainer_name}") + ) + except Exception as e: + logger.info( + f"Unsloth: Could not wrap _generate_and_score_completions for {RLTrainer_name}: {e}" + ) + def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, imports): init = inspect.getsource(RLTrainer.__init__) From 77e7f736419bf021c6e4502cea27939132a06bbd Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 5 Jan 2026 13:55:08 +0000 Subject: [PATCH 086/114] Revert rl_replacements GRPO edits --- unsloth/models/rl_replacements.py | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index f0f0386bd1..5e079335ae 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -259,7 +259,6 @@ def grpo_trainer__generate_and_score_completions(function_name, function): # The new multi-line string that will replace the line above replacement_lines = """ batch_size = self.args.per_device_train_batch_size if mode == "train" else self.args.per_device_eval_batch_size - _was_training = self.model.training try: # TRL 0.23.1 and below path if not has_images: @@ -389,20 +388,6 @@ def grpo_trainer__generate_and_score_completions(function_name, function): function = patched - match = re.search(r"^(\s*)return output", function, re.MULTILINE) - - if match: - indent = match.group(1) - new_code = ( - indent - + "if not _was_training:\n" - + indent - + " self.model.for_inference()\n" - + indent - + "return output" - ) - function = function.replace(f"{indent}return output", new_code) - return function @@ -876,13 +861,19 @@ def grpo_trainer_compute_loss(function_name, function): else torch.tensor(0.0, device = self.model.device) ) self._metrics[mode]["sampling/importance_sampling_ratio/min"].append( - nanmin(self.accelerator.gather(min_importance_sampling_ratio)).item() + self.accelerator.gather(min_importance_sampling_ratio) + .nan_to_num(nan = float("inf")) + .min() + .item() ) self._metrics[mode]["sampling/importance_sampling_ratio/mean"].append( self.accelerator.gather(mean_importance_sampling_ratio).nanmean().item() ) self._metrics[mode]["sampling/importance_sampling_ratio/max"].append( - nanmax(self.accelerator.gather(max_importance_sampling_ratio)).item() + self.accelerator.gather(max_importance_sampling_ratio) + .nan_to_num(nan = float("-inf")) + .max() + .item() ) return loss @@ -964,11 +955,15 @@ def openenv_vllm_reload_weights(): return if Version(importlib_version("trl")) < Version("0.26.0"): return + try: import trl.experimental.openenv.utils as openenv_utils import trl.experimental.openenv as openenv except ImportError as e: logger.info(f"Unsloth: Failed to import trl openenv: {e}") + logger.info( + "Unsloth: trl.experimental.openenv not available — skipping RL openenv patches." + ) return src = inspect.getsource(openenv_utils.generate_rollout_completions) From e7fe25ee43002eaf1cee7be2e2aa3fa6d3811f8a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 07:37:08 -0800 Subject: [PATCH 087/114] Versioning --- pyproject.toml | 4 ++-- unsloth/__init__.py | 2 +- unsloth/models/_utils.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e7b84f3c8e..7fa249e64c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ huggingfacenotorch = [ ] huggingface = [ "unsloth[huggingfacenotorch]", - "unsloth_zoo>=2026.1.1", + "unsloth_zoo>=2026.1.2", "torchvision", "unsloth[triton]", ] @@ -523,7 +523,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3 ; ('linux' in sys_platform)", ] colab-new = [ - "unsloth_zoo>=2026.1.1", + "unsloth_zoo>=2026.1.2", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,<=4.57.3", diff --git a/unsloth/__init__.py b/unsloth/__init__.py index 86fb00fe0e..5b571cd456 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -79,7 +79,7 @@ from importlib.metadata import PackageNotFoundError # Check for unsloth_zoo try: unsloth_zoo_version = importlib_version("unsloth_zoo") - if Version(unsloth_zoo_version) < Version("2026.1.1"): + if Version(unsloth_zoo_version) < Version("2026.1.2"): print( "Unsloth: Please update Unsloth and Unsloth-Zoo to the latest version!\n" "Do this via `pip install --upgrade --force-reinstall --no-cache-dir --no-deps unsloth unsloth_zoo`" diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 5952d4af0c..b38c5860b3 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2026.1.1" +__version__ = "2026.1.2" __all__ = [ "SUPPORTS_BFLOAT16", From 9a5b824903e9071ba17c3fa6757185ddcd1287bf Mon Sep 17 00:00:00 2001 From: Datta Nimmaturi Date: Tue, 6 Jan 2026 09:53:20 +0000 Subject: [PATCH 088/114] Disable stats when modelscope is being used --- unsloth/models/_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index b38c5860b3..a2e1d78012 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -1197,7 +1197,8 @@ def get_statistics(local_files_only = False): # You can disable this by setting UNSLOTH_DISABLE_STATISTICS import os - if "UNSLOTH_DISABLE_STATISTICS" in os.environ: + global USE_MODELSCOPE + if "UNSLOTH_DISABLE_STATISTICS" in os.environ or USE_MODELSCOPE: return if local_files_only: return From 1d84ba52870c86f77820664bd598060d1db6bb39 Mon Sep 17 00:00:00 2001 From: Datta Nimmaturi Date: Tue, 6 Jan 2026 15:30:06 +0530 Subject: [PATCH 089/114] Check env var explicitly Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- unsloth/models/_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index a2e1d78012..3cbb85ff4d 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -1197,8 +1197,7 @@ def get_statistics(local_files_only = False): # You can disable this by setting UNSLOTH_DISABLE_STATISTICS import os - global USE_MODELSCOPE - if "UNSLOTH_DISABLE_STATISTICS" in os.environ or USE_MODELSCOPE: + if "UNSLOTH_DISABLE_STATISTICS" in os.environ or os.environ.get("UNSLOTH_USE_MODELSCOPE", "0") == "1": return if local_files_only: return From 46d212c480b6d492b16b67493dab8ffe15a5b138 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 6 Jan 2026 10:00:16 +0000 Subject: [PATCH 090/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 3cbb85ff4d..e6c4a12874 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -1197,7 +1197,10 @@ def get_statistics(local_files_only = False): # You can disable this by setting UNSLOTH_DISABLE_STATISTICS import os - if "UNSLOTH_DISABLE_STATISTICS" in os.environ or os.environ.get("UNSLOTH_USE_MODELSCOPE", "0") == "1": + if ( + "UNSLOTH_DISABLE_STATISTICS" in os.environ + or os.environ.get("UNSLOTH_USE_MODELSCOPE", "0") == "1" + ): return if local_files_only: return From 1f85f39e0a85324de87cda091f28300dea853e3d Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Thu, 8 Jan 2026 04:14:53 +0000 Subject: [PATCH 091/114] Fix FBGEMM/CUTLASS errors on SM100 (Blackwell) GPUs This PR fixes the "Arch conditional MMA instruction used without targeting appropriate compute capability. Aborting." errors that occur when using FBGEMM on Blackwell GPUs (B200/B100, SM100). Changes: - Add stderr filters in import_fixes.py for CUTLASS/FBGEMM MMA errors - Add warning filters for various deprecation messages - Update check_fbgemm_gpu_version() to disable FBGEMM instead of raising an error when old versions are detected - Update test_has_fbgemm() in fp8.py to catch broader CUTLASS/CUDA errors and gracefully fall back to Triton kernels - Update loader_utils.py to disable FBGEMM instead of raising ValueError for old fbgemm_gpu versions The key behavior change is that FBGEMM errors no longer crash the script. Instead, FBGEMM is disabled and Triton kernels are used automatically. This allows Unsloth to work on SM100 GPUs where CUTLASS SM90 kernels fail, and also gracefully handles old FBGEMM versions. --- unsloth/import_fixes.py | 34 +++++++++++++++++++++++++++++++--- unsloth/kernels/fp8.py | 22 +++++++++++++++++++--- unsloth/models/loader_utils.py | 10 +++++++--- 3 files changed, 57 insertions(+), 9 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 1e05e462e9..5ad341e2ac 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -94,16 +94,40 @@ class HidePrintMessage: if os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") != "1": import sys - # Apply to stderr for FBGEMM + # Apply to stderr for FBGEMM and CUTLASS errors sys.stderr = HidePrintMessage(sys.stderr) # https://github.com/pytorch/FBGEMM/blob/d99cd96490ec4aabac2ee95b1e76ea4dcfcfa628/fbgemm_gpu/experimental/gemm/triton_gemm/utils.py#L43-L52 sys.stderr.add_filter("TMA benchmarks will be running") + # CUTLASS/FBGEMM MMA instruction error on SM90 vs SM100 (Blackwell) GPUs + # https://github.com/NVIDIA/cutlass/blob/main/include/cutlass/gemm/kernel/sm90_gemm_tma_warpspecialized.hpp + sys.stderr.add_filter("Arch conditional MMA instruction used without targeting") + # CUTLASS arch conditional errors for various architectures + sys.stderr.add_filter("CUTE_INVALID_CONTROL_PATH") + # CUTLASS TMA-related errors when not targeting correct architecture + sys.stderr.add_filter("Trying to use tma without CUTE_ARCH_TMA") # Skipping import of cpp extensions due to incompatible torch version 2.9.0+cu128 for torchao version 0.15.0 logging.getLogger("torchao").setLevel(logging.ERROR) + # Also filter torchao print to stderr about cpp extensions + sys.stderr.add_filter("Skipping import of cpp extensions") # SyntaxWarning: invalid escape sequence '\.' warnings.filterwarnings( "ignore", message = "invalid escape sequence", category = SyntaxWarning ) + # PYTORCH_CUDA_ALLOC_CONF is deprecated warning from torch + warnings.filterwarnings( + "ignore", message = "PYTORCH_CUDA_ALLOC_CONF is deprecated" + ) + # TF32 precision deprecation warning from torch + warnings.filterwarnings( + "ignore", message = "Please use the new API settings to control TF32" + ) + # Deprecation warnings from torchao + warnings.filterwarnings( + "ignore", message = "`int4_weight_only` is deprecated" + ) + warnings.filterwarnings( + "ignore", message = "`int8_weight_only` is deprecated" + ) # Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype' @@ -323,10 +347,14 @@ def check_fbgemm_gpu_version(): except: return # We noticed some SegFault or bad alloc errors on lower versions of fbgemm_gpu. + # Instead of raising an error, disable FBGEMM and fall back to Triton kernels. if Version(fbgemm_gpu_version) < Version("1.4.0"): - raise ImportError( - f"Unsloth: fbgemm_gpu_genai=={fbgemm_gpu_version} detected. It might cause unexpected issues like segmentation faults. Please uninstall the current one by doing `pip uninstall fbgemm-gpu` && `pip install fbgemm-gpu` to install fbgemm-gpu 1.4.0 or newer!" + os.environ["UNSLOTH_HAS_FBGEMM"] = "0" + logger.info( + f"Unsloth: fbgemm_gpu_genai=={fbgemm_gpu_version} is old and may cause issues. " + f"Disabling FBGEMM - using Triton kernels instead." ) + return logger.info(f"Unsloth: fbgemm_gpu_genai=={fbgemm_gpu_version} detected.") diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py index 3093bf61b1..e9f9161709 100644 --- a/unsloth/kernels/fp8.py +++ b/unsloth/kernels/fp8.py @@ -523,6 +523,7 @@ def fp8_fbgemm_block_linear(X, weight, weight_scale, bias = None): def test_has_fbgemm(): # We must manually check if the faster FBGEMM works on the specific GPU # For example RTX 5090 and RTX 4090 does not work + # Also SM100 (Blackwell B200/B100) GPUs fail with CUTLASS SM90 kernels # [TODO] Investigate with TorchAO why FBGEMM fails on consumer GPUs M, N, K = 128, 128, 128 xq = torch.ones(M, K, dtype = torch.float8_e4m3fn, device = "cuda") @@ -537,10 +538,25 @@ def test_has_fbgemm(): has_fbgemm = True del out except Exception as e: - e = str(e) - if "cutlass cannot initialize" in e.lower(): + error_str = str(e).lower() + # Catch any CUTLASS/CUDA errors and disable FBGEMM + # This includes MMA instruction errors, architecture mismatches, kernel launch failures, etc. + cutlass_cuda_errors = ( + "cutlass", + "cuda error", + "cuda runtime error", + "no kernel image", + "arch conditional", + "mma instruction", + "compute capability", + "cute_invalid_control_path", + "tma", + ) + is_cutlass_cuda_error = any(err in error_str for err in cutlass_cuda_errors) + + if is_cutlass_cuda_error: print( - f"Unsloth: FBGEMM on the current GPU cannot load - will switch to Triton kernels" + "Unsloth: FBGEMM on the current GPU cannot load - will switch to Triton kernels" ) else: print( diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index fe2a89d893..9656cc9d26 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -408,7 +408,7 @@ def _get_fp8_mode_and_check_settings( if Version(torchao.__version__) < Version("0.15.0"): raise ValueError(error_message) - # If fbgemm_gpu_genai is installed, check if it's >= 1.4.1 + # If fbgemm_gpu_genai is installed and old, disable FBGEMM and use Triton instead if ( importlib.util.find_spec("fbgemm_gpu") is not None and importlib.util.find_spec("fbgemm_gpu.experimental") is not None @@ -416,7 +416,11 @@ def _get_fp8_mode_and_check_settings( import fbgemm_gpu.experimental.gen_ai if Version(fbgemm_gpu.__version__) < Version("1.4.1"): - raise ValueError( - "Unsloth: On the fly `load_in_fp8` is only compatible with fbgemm_gpu_genai 1.4.1+. Try `unsloth/Qwen3-8B` instead." + # Old FBGEMM version - disable and use Triton kernels instead + os.environ["UNSLOTH_HAS_FBGEMM"] = "0" + from unsloth_zoo.log import logger + logger.info( + f"Unsloth: fbgemm_gpu_genai=={fbgemm_gpu.__version__} is old for FP8 loading. " + f"Using Triton kernels instead." ) return fp8_mode From 41b7fe0c672f518b41519f25f049f87084694f1c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 8 Jan 2026 04:15:17 +0000 Subject: [PATCH 092/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/import_fixes.py | 12 +++--------- unsloth/models/loader_utils.py | 1 + 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 5ad341e2ac..27e5342e20 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -114,20 +114,14 @@ if os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") != "1": "ignore", message = "invalid escape sequence", category = SyntaxWarning ) # PYTORCH_CUDA_ALLOC_CONF is deprecated warning from torch - warnings.filterwarnings( - "ignore", message = "PYTORCH_CUDA_ALLOC_CONF is deprecated" - ) + warnings.filterwarnings("ignore", message = "PYTORCH_CUDA_ALLOC_CONF is deprecated") # TF32 precision deprecation warning from torch warnings.filterwarnings( "ignore", message = "Please use the new API settings to control TF32" ) # Deprecation warnings from torchao - warnings.filterwarnings( - "ignore", message = "`int4_weight_only` is deprecated" - ) - warnings.filterwarnings( - "ignore", message = "`int8_weight_only` is deprecated" - ) + warnings.filterwarnings("ignore", message = "`int4_weight_only` is deprecated") + warnings.filterwarnings("ignore", message = "`int8_weight_only` is deprecated") # Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype' diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 9656cc9d26..1e5533c25c 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -419,6 +419,7 @@ def _get_fp8_mode_and_check_settings( # Old FBGEMM version - disable and use Triton kernels instead os.environ["UNSLOTH_HAS_FBGEMM"] = "0" from unsloth_zoo.log import logger + logger.info( f"Unsloth: fbgemm_gpu_genai=={fbgemm_gpu.__version__} is old for FP8 loading. " f"Using Triton kernels instead." From 24bbe8a97a0bcb97b204eca82f680e04c5634b07 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 8 Jan 2026 11:35:00 +0000 Subject: [PATCH 093/114] Fix bugs and add improvements to RawTextDataLoader - Fix test file: use return_tokenized instead of return_tensors - Fix test file: use text_dataset instead of undefined dataset variable - Move parameter validation to constructor (fail fast on invalid params) - Add labels field in tokenized output for causal LM training - Add empty file handling with clear error message - Add tests for constructor validation and labels field --- tests/test_raw_text.py | 23 ++++++++++++++++++++--- unsloth/dataprep/raw_text.py | 19 +++++++++++-------- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/tests/test_raw_text.py b/tests/test_raw_text.py index 5306c68fa6..7c7272a551 100644 --- a/tests/test_raw_text.py +++ b/tests/test_raw_text.py @@ -100,12 +100,12 @@ def test_raw_text_loader(): loader = RawTextDataLoader(tokenizer, chunk_size = 5, stride = 2) # Test loading with text output (legacy mode) - text_dataset = loader.load_from_file(test_file, return_tensors = False) + text_dataset = loader.load_from_file(test_file, return_tokenized = False) assert len(text_dataset) > 0, "Should create at least one chunk" assert "text" in text_dataset.column_names, "Dataset should have 'text' column" # Test loading with tokenized output (new efficient mode) - tokenized_dataset = loader.load_from_file(test_file, return_tensors = True) + tokenized_dataset = loader.load_from_file(test_file, return_tokenized = True) assert len(tokenized_dataset) > 0, "Should create at least one tokenized chunk" assert ( "input_ids" in tokenized_dataset.column_names @@ -124,13 +124,30 @@ def test_raw_text_loader(): first_sample["attention_mask"] ), "input_ids and attention_mask should have same length" + # Verify labels field exists (for causal LM training) + assert "labels" in tokenized_dataset.column_names, "Dataset should have 'labels' column" + assert first_sample["labels"] == first_sample["input_ids"], "labels should match input_ids" + + # Test constructor validation + try: + bad_loader = RawTextDataLoader(tokenizer, chunk_size = 0, stride = 2) + assert False, "Should raise ValueError for chunk_size=0" + except ValueError as e: + assert "chunk_size must be positive" in str(e) + + try: + bad_loader = RawTextDataLoader(tokenizer, chunk_size = 5, stride = 10) + assert False, "Should raise ValueError for stride >= chunk_size" + except ValueError as e: + assert "stride" in str(e) and "chunk_size" in str(e) + # Test preprocessor preprocessor = TextPreprocessor() clean_text = preprocessor.clean_text(" messy text \n\n\n ") assert "messy text" in clean_text, "Should clean text properly" # Test validation - stats = preprocessor.validate_dataset(dataset) + stats = preprocessor.validate_dataset(text_dataset) assert stats["total_samples"] > 0, "Should count samples" assert "warnings" in stats, "Should include warnings" diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py index f7c1bf7856..da64565bbc 100644 --- a/unsloth/dataprep/raw_text.py +++ b/unsloth/dataprep/raw_text.py @@ -36,6 +36,12 @@ SUPPORTED_FORMATS = { class RawTextDataLoader: def __init__(self, tokenizer, chunk_size = 2048, stride = 512, return_tokenized = True): + if chunk_size <= 0: + raise ValueError(f"chunk_size must be positive, got {chunk_size}") + if stride >= chunk_size: + raise ValueError( + f"stride ({stride}) must be smaller than chunk_size ({chunk_size})" + ) self.tokenizer = tokenizer self.chunk_size = chunk_size self.stride = stride @@ -52,6 +58,8 @@ class RawTextDataLoader: return_tokenized = self.return_tokenized file_format = self.detect_format(file_path) text_content = self._read_file_by_format(file_path, file_format) + if not text_content or not text_content.strip(): + raise ValueError(f"File '{file_path}' is empty or contains only whitespace") chunks = self.smart_chunk_text( text_content, self.chunk_size, self.stride, return_tokenized ) @@ -86,8 +94,10 @@ class RawTextDataLoader: # Reorganize the data structure for Dataset.from_dict input_ids = [chunk["input_ids"] for chunk in chunks] attention_mask = [chunk["attention_mask"] for chunk in chunks] + # Labels are same as input_ids for causal LM training + labels = [list(ids) for ids in input_ids] return Dataset.from_dict( - {"input_ids": input_ids, "attention_mask": attention_mask} + {"input_ids": input_ids, "attention_mask": attention_mask, "labels": labels} ) else: # If chunks are text strings (backward compatibility) @@ -101,13 +111,6 @@ class RawTextDataLoader: 3. Maintains context with stride overlap 4. Returns tokenized chunks directly (more efficient) or text chunks """ - if chunk_size <= 0: - raise ValueError(f"chunk_size must be positive, got {chunk_size}") - if stride >= chunk_size: - raise ValueError( - f"stride ({stride}) must be smaller than chunk_size ({chunk_size}) to progress the chunking loop" - ) - # First pass: tokenize the entire text to get accurate token counts tokenized = self.tokenizer(text, return_tensors = "pt", add_special_tokens = False) tokens = tokenized["input_ids"] From 3ac4f3c21318b69b22183a309f8abfe58a8f75a9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 8 Jan 2026 11:35:21 +0000 Subject: [PATCH 094/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_raw_text.py | 8 ++++++-- unsloth/dataprep/raw_text.py | 6 +++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/test_raw_text.py b/tests/test_raw_text.py index 7c7272a551..9f2e8cda4e 100644 --- a/tests/test_raw_text.py +++ b/tests/test_raw_text.py @@ -125,8 +125,12 @@ def test_raw_text_loader(): ), "input_ids and attention_mask should have same length" # Verify labels field exists (for causal LM training) - assert "labels" in tokenized_dataset.column_names, "Dataset should have 'labels' column" - assert first_sample["labels"] == first_sample["input_ids"], "labels should match input_ids" + assert ( + "labels" in tokenized_dataset.column_names + ), "Dataset should have 'labels' column" + assert ( + first_sample["labels"] == first_sample["input_ids"] + ), "labels should match input_ids" # Test constructor validation try: diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py index da64565bbc..ba010edabb 100644 --- a/unsloth/dataprep/raw_text.py +++ b/unsloth/dataprep/raw_text.py @@ -97,7 +97,11 @@ class RawTextDataLoader: # Labels are same as input_ids for causal LM training labels = [list(ids) for ids in input_ids] return Dataset.from_dict( - {"input_ids": input_ids, "attention_mask": attention_mask, "labels": labels} + { + "input_ids": input_ids, + "attention_mask": attention_mask, + "labels": labels, + } ) else: # If chunks are text strings (backward compatibility) From 1cdf751f8eaad722be9f248ce097645504d1c3e8 Mon Sep 17 00:00:00 2001 From: Rachel Li Date: Thu, 8 Jan 2026 18:44:22 -0500 Subject: [PATCH 095/114] Fix Kaggle telemetry misclassification when COLAB_ keys exist Problem: Kaggle notebook environments can expose both KAGGLE_* and COLAB_* environment keys. _get_statistics currently checks COLAB_ before KAGGLE_, causing Kaggle sessions to be labeled colab/colabpro. Prefer filesystem markers (e.g. /kaggle/working, /content + /opt/colab) before env-key heuristics, then fall back to the existing env-key checks. This avoids misclassification when providers leak overlapping env vars. Kaggle test notebook: https://www.kaggle.com/code/hnxnq07/kaggle-stats-gathering-test --- unsloth/models/_utils.py | 165 +++++++++++++++++++++------------------ 1 file changed, 90 insertions(+), 75 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index e6c4a12874..77564c03d5 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -1108,85 +1108,100 @@ def _get_statistics(statistics = None, force_download = True): if statistics is not None: pass - elif "\nCOLAB_" in keynames and n_cpus == 1: - statistics = "colab" - elif "\nCOLAB_" in keynames: - statistics = "colabpro" - elif "\nKAGGLE_" in keynames: - statistics = "kaggle" - elif "\nRUNPOD_" in keynames: - statistics = "runpod" - elif "\nAWS_" in keynames: - statistics = "aws" - elif "\nAZURE_" in keynames: - statistics = "azure" - # elif "\nK_" in keynames or "\nFUNCTION_" in keynames: statistics = "gcp" - elif "\nINVOCATION_ID" in keynames: - statistics = "lambda" - # else: statistics = "other" else: - - def try_vllm_check(): - vendor_files = ( - "/sys/class/dmi/id/product_version", - "/sys/class/dmi/id/bios_vendor", - "/sys/class/dmi/id/product_name", - "/sys/class/dmi/id/chassis_asset_tag", - "/sys/class/dmi/id/sys_vendor", - ) + # Prefer filesystem markers (harder to misidentify) before env-key matching + try: from pathlib import Path - for vendor_file in vendor_files: - path = Path(vendor_file) - if path.is_file(): - file_content = path.read_text().lower() - if "amazon" in file_content: - return "aws" - elif "microsoft corporation" in file_content: - return "azure" - elif "google" in file_content: - return "gcp" - return "other" - - pass - try: - statistics = try_vllm_check() - except: - statistics = "other" - if statistics is not None: - import tempfile - from huggingface_hub import snapshot_download - from unsloth_zoo.rl_environments import execute_with_time_limit - - if has_internet(): - - def stats_check(): - with tempfile.TemporaryDirectory(ignore_cleanup_errors = True) as f: - snapshot_download( - f"unslothai/{statistics}", - force_download = True, - cache_dir = f, - local_dir = f, + if Path("/kaggle/working").exists(): + statistics = "kaggle" + elif Path("/content").exists() and Path("/opt/colab").exists(): + statistics = "colab" if n_cpus == 1 else "colabpro" + elif Path("/runpod-volume").exists(): + statistics = "runpod" + except Exception: + pass + # Fallback to env-key detection + if statistics is None: + if "\nKAGGLE_" in keynames: + statistics = "kaggle" + elif "\nCOLAB_" in keynames and n_cpus == 1: + statistics = "colab" + elif "\nCOLAB_" in keynames: + statistics = "colabpro" + elif "\nRUNPOD_" in keynames: + statistics = "runpod" + elif "\nAWS_" in keynames: + statistics = "aws" + elif "\nAZURE_" in keynames: + statistics = "azure" + # elif "\nK_" in keynames or "\nFUNCTION_" in keynames: statistics = "gcp" + elif "\nINVOCATION_ID" in keynames: + statistics = "lambda" + # else: statistics = "other" + else: + + def try_vllm_check(): + vendor_files = ( + "/sys/class/dmi/id/product_version", + "/sys/class/dmi/id/bios_vendor", + "/sys/class/dmi/id/product_name", + "/sys/class/dmi/id/chassis_asset_tag", + "/sys/class/dmi/id/sys_vendor", ) - - time_limited_stats_check = execute_with_time_limit(120)(stats_check) - try: - time_limited_stats_check() - except TimeoutError: - raise TimeoutError( - "Unsloth: HuggingFace seems to be down after trying for 120 seconds :(\n" - "Check https://status.huggingface.co/ for more details.\n" - "As a temporary measure, use modelscope with the same model name ie:\n" - "```\n" - "pip install modelscope\n" - "import os; os.environ['UNSLOTH_USE_MODELSCOPE'] = '1'\n" - "from unsloth import FastLanguageModel\n" - "model = FastLanguageModel.from_pretrained('unsloth/gpt-oss-20b')\n" - "```" - ) - except: - # Try no time limit check - stats_check() + from pathlib import Path + + for vendor_file in vendor_files: + path = Path(vendor_file) + if path.is_file(): + file_content = path.read_text().lower() + if "amazon" in file_content: + return "aws" + elif "microsoft corporation" in file_content: + return "azure" + elif "google" in file_content: + return "gcp" + return "other" + + pass + try: + statistics = try_vllm_check() + except: + statistics = "other" + if statistics is not None: + import tempfile + from huggingface_hub import snapshot_download + from unsloth_zoo.rl_environments import execute_with_time_limit + + if has_internet(): + + def stats_check(): + with tempfile.TemporaryDirectory(ignore_cleanup_errors = True) as f: + snapshot_download( + f"unslothai/{statistics}", + force_download = True, + cache_dir = f, + local_dir = f, + ) + + time_limited_stats_check = execute_with_time_limit(120)(stats_check) + try: + time_limited_stats_check() + except TimeoutError: + raise TimeoutError( + "Unsloth: HuggingFace seems to be down after trying for 120 seconds :(\n" + "Check https://status.huggingface.co/ for more details.\n" + "As a temporary measure, use modelscope with the same model name ie:\n" + "```\n" + "pip install modelscope\n" + "import os; os.environ['UNSLOTH_USE_MODELSCOPE'] = '1'\n" + "from unsloth import FastLanguageModel\n" + "model = FastLanguageModel.from_pretrained('unsloth/gpt-oss-20b')\n" + "```" + ) + except: + # Try no time limit check + stats_check() def get_statistics(local_files_only = False): From 67bef80b1f2fcfe1f4d6ef0bd96cca2f7955ef83 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 8 Jan 2026 23:49:54 +0000 Subject: [PATCH 096/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/_utils.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 77564c03d5..756d884de1 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -1140,7 +1140,7 @@ def _get_statistics(statistics = None, force_download = True): statistics = "lambda" # else: statistics = "other" else: - + def try_vllm_check(): vendor_files = ( "/sys/class/dmi/id/product_version", @@ -1150,7 +1150,7 @@ def _get_statistics(statistics = None, force_download = True): "/sys/class/dmi/id/sys_vendor", ) from pathlib import Path - + for vendor_file in vendor_files: path = Path(vendor_file) if path.is_file(): @@ -1162,7 +1162,7 @@ def _get_statistics(statistics = None, force_download = True): elif "google" in file_content: return "gcp" return "other" - + pass try: statistics = try_vllm_check() @@ -1172,18 +1172,20 @@ def _get_statistics(statistics = None, force_download = True): import tempfile from huggingface_hub import snapshot_download from unsloth_zoo.rl_environments import execute_with_time_limit - + if has_internet(): - + def stats_check(): - with tempfile.TemporaryDirectory(ignore_cleanup_errors = True) as f: + with tempfile.TemporaryDirectory( + ignore_cleanup_errors = True + ) as f: snapshot_download( f"unslothai/{statistics}", force_download = True, cache_dir = f, local_dir = f, ) - + time_limited_stats_check = execute_with_time_limit(120)(stats_check) try: time_limited_stats_check() From e13160ddc8febd25b569bc31970efce7dd9c57a7 Mon Sep 17 00:00:00 2001 From: Rachel Li Date: Thu, 8 Jan 2026 19:04:30 -0500 Subject: [PATCH 097/114] Update _utils.py fixed indentation --- unsloth/models/_utils.py | 68 +++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 756d884de1..9bf02beb30 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -1121,6 +1121,7 @@ def _get_statistics(statistics = None, force_download = True): statistics = "runpod" except Exception: pass + # Fallback to env-key detection if statistics is None: if "\nKAGGLE_" in keynames: @@ -1168,42 +1169,43 @@ def _get_statistics(statistics = None, force_download = True): statistics = try_vllm_check() except: statistics = "other" - if statistics is not None: - import tempfile - from huggingface_hub import snapshot_download - from unsloth_zoo.rl_environments import execute_with_time_limit + + if statistics is not None: + import tempfile + from huggingface_hub import snapshot_download + from unsloth_zoo.rl_environments import execute_with_time_limit - if has_internet(): + if has_internet(): - def stats_check(): - with tempfile.TemporaryDirectory( - ignore_cleanup_errors = True - ) as f: - snapshot_download( - f"unslothai/{statistics}", - force_download = True, - cache_dir = f, - local_dir = f, - ) - - time_limited_stats_check = execute_with_time_limit(120)(stats_check) - try: - time_limited_stats_check() - except TimeoutError: - raise TimeoutError( - "Unsloth: HuggingFace seems to be down after trying for 120 seconds :(\n" - "Check https://status.huggingface.co/ for more details.\n" - "As a temporary measure, use modelscope with the same model name ie:\n" - "```\n" - "pip install modelscope\n" - "import os; os.environ['UNSLOTH_USE_MODELSCOPE'] = '1'\n" - "from unsloth import FastLanguageModel\n" - "model = FastLanguageModel.from_pretrained('unsloth/gpt-oss-20b')\n" - "```" + def stats_check(): + with tempfile.TemporaryDirectory( + ignore_cleanup_errors = True + ) as f: + snapshot_download( + f"unslothai/{statistics}", + force_download = True, + cache_dir = f, + local_dir = f, ) - except: - # Try no time limit check - stats_check() + + time_limited_stats_check = execute_with_time_limit(120)(stats_check) + try: + time_limited_stats_check() + except TimeoutError: + raise TimeoutError( + "Unsloth: HuggingFace seems to be down after trying for 120 seconds :(\n" + "Check https://status.huggingface.co/ for more details.\n" + "As a temporary measure, use modelscope with the same model name ie:\n" + "```\n" + "pip install modelscope\n" + "import os; os.environ['UNSLOTH_USE_MODELSCOPE'] = '1'\n" + "from unsloth import FastLanguageModel\n" + "model = FastLanguageModel.from_pretrained('unsloth/gpt-oss-20b')\n" + "```" + ) + except: + # Try no time limit check + stats_check() def get_statistics(local_files_only = False): From e7d68f3e5788d21d5afaac182099716c9b5e248e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 9 Jan 2026 00:04:59 +0000 Subject: [PATCH 098/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/_utils.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 9bf02beb30..701e6f633c 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -1121,7 +1121,7 @@ def _get_statistics(statistics = None, force_download = True): statistics = "runpod" except Exception: pass - + # Fallback to env-key detection if statistics is None: if "\nKAGGLE_" in keynames: @@ -1169,7 +1169,7 @@ def _get_statistics(statistics = None, force_download = True): statistics = try_vllm_check() except: statistics = "other" - + if statistics is not None: import tempfile from huggingface_hub import snapshot_download @@ -1178,9 +1178,7 @@ def _get_statistics(statistics = None, force_download = True): if has_internet(): def stats_check(): - with tempfile.TemporaryDirectory( - ignore_cleanup_errors = True - ) as f: + with tempfile.TemporaryDirectory(ignore_cleanup_errors = True) as f: snapshot_download( f"unslothai/{statistics}", force_download = True, From 84701c55ffa7437d11d0e32c5d7a6e77681862c4 Mon Sep 17 00:00:00 2001 From: Rachel Li Date: Thu, 8 Jan 2026 19:20:24 -0500 Subject: [PATCH 099/114] Fix telemetry ping regression for explicit statistics Fixed Codex regression: keep snapshot_download pings for explicit statistics values; detection only runs when statistics is None. Also replaced bare except. --- unsloth/models/_utils.py | 62 ++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 701e6f633c..0cb6c8975e 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -1170,40 +1170,40 @@ def _get_statistics(statistics = None, force_download = True): except: statistics = "other" - if statistics is not None: - import tempfile - from huggingface_hub import snapshot_download - from unsloth_zoo.rl_environments import execute_with_time_limit + if statistics is not None: + import tempfile + from huggingface_hub import snapshot_download + from unsloth_zoo.rl_environments import execute_with_time_limit - if has_internet(): + if has_internet(): - def stats_check(): - with tempfile.TemporaryDirectory(ignore_cleanup_errors = True) as f: - snapshot_download( - f"unslothai/{statistics}", - force_download = True, - cache_dir = f, - local_dir = f, - ) - - time_limited_stats_check = execute_with_time_limit(120)(stats_check) - try: - time_limited_stats_check() - except TimeoutError: - raise TimeoutError( - "Unsloth: HuggingFace seems to be down after trying for 120 seconds :(\n" - "Check https://status.huggingface.co/ for more details.\n" - "As a temporary measure, use modelscope with the same model name ie:\n" - "```\n" - "pip install modelscope\n" - "import os; os.environ['UNSLOTH_USE_MODELSCOPE'] = '1'\n" - "from unsloth import FastLanguageModel\n" - "model = FastLanguageModel.from_pretrained('unsloth/gpt-oss-20b')\n" - "```" + def stats_check(): + with tempfile.TemporaryDirectory(ignore_cleanup_errors = True) as f: + snapshot_download( + f"unslothai/{statistics}", + force_download = True, + cache_dir = f, + local_dir = f, ) - except: - # Try no time limit check - stats_check() + + time_limited_stats_check = execute_with_time_limit(120)(stats_check) + try: + time_limited_stats_check() + except TimeoutError: + raise TimeoutError( + "Unsloth: HuggingFace seems to be down after trying for 120 seconds :(\n" + "Check https://status.huggingface.co/ for more details.\n" + "As a temporary measure, use modelscope with the same model name ie:\n" + "```\n" + "pip install modelscope\n" + "import os; os.environ['UNSLOTH_USE_MODELSCOPE'] = '1'\n" + "from unsloth import FastLanguageModel\n" + "model = FastLanguageModel.from_pretrained('unsloth/gpt-oss-20b')\n" + "```" + ) + except Exception: + # Try no time limit check + stats_check() def get_statistics(local_files_only = False): From 1193c9f5268f33123054a58c8baf699bf2597a80 Mon Sep 17 00:00:00 2001 From: Rachel Li Date: Thu, 8 Jan 2026 19:32:33 -0500 Subject: [PATCH 100/114] Fix Kaggle telemetry detection & address review feedback - Fix Kaggle misclassification by prioritizing filesystem markers over env vars - Preserve telemetry pings when statistics is explicitly provided - Replace bare except with except Exception - Minor cleanup based on automated review feedback --- unsloth/models/_utils.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 0cb6c8975e..77c8da2576 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -1106,9 +1106,7 @@ def _get_statistics(statistics = None, force_download = True): global USE_MODELSCOPE USE_MODELSCOPE = os.environ.get("UNSLOTH_USE_MODELSCOPE", "0") == "1" - if statistics is not None: - pass - else: + if statistics is None: # Prefer filesystem markers (harder to misidentify) before env-key matching try: from pathlib import Path @@ -1150,7 +1148,6 @@ def _get_statistics(statistics = None, force_download = True): "/sys/class/dmi/id/chassis_asset_tag", "/sys/class/dmi/id/sys_vendor", ) - from pathlib import Path for vendor_file in vendor_files: path = Path(vendor_file) @@ -1163,11 +1160,10 @@ def _get_statistics(statistics = None, force_download = True): elif "google" in file_content: return "gcp" return "other" - - pass + try: statistics = try_vllm_check() - except: + except Exception: statistics = "other" if statistics is not None: From 3ce1060dd162cc6a1451e57ae129e384343878e0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 9 Jan 2026 00:33:00 +0000 Subject: [PATCH 101/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 77c8da2576..b0ad0a3082 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -1160,7 +1160,7 @@ def _get_statistics(statistics = None, force_download = True): elif "google" in file_content: return "gcp" return "other" - + try: statistics = try_vllm_check() except Exception: From 0bff0ffbe504d35ba407c20baf3250e1d25d81c6 Mon Sep 17 00:00:00 2001 From: Kaitao Yang Date: Wed, 7 Jan 2026 22:54:35 -0800 Subject: [PATCH 102/114] reduce code duplication by _offload_frozen_module_for_training --- unsloth/models/llama.py | 91 ++++++++++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 34 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 92d51b73ad..cfd900c460 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -146,6 +146,59 @@ torch_nn_functional_softmax = torch.nn.functional.softmax # SDPA has GQA internally SDPA_HAS_GQA = "enable_gqa" in scaled_dot_product_attention.__doc__ +from peft.tuners.tuners_utils import ModulesToSaveWrapper + + +def _offload_frozen_module_for_training( + module: ModulesToSaveWrapper, + device_type: str, + offload_device: str = "cpu", +) -> None: + """ + Offload frozen module to CPU and configure trainable copy for mixed precision training. + + This function optimizes memory usage by: + 1. Moving the trainable copy to the target device with appropriate precision + 2. Offloading the original frozen module to CPU/disk to free VRAM + 3. Converting float16 to float32 for compatibility with certain GPUs (e.g., Tesla T4) + + Args: + module: The module to configure. Must be a ModulesToSaveWrapper with a + `modules_to_save` attribute containing trainable and original modules. + device_type: Target device string for training (e.g., "cuda:0", "xpu:0") + offload_device: Device to offload frozen parameters (default: "cpu") + Note: Currently only "cpu" is supported; disk offloading is planned. + + Returns: + None (modifies module in-place) + + Note: + - Float16 weights are automatically promoted to float32 for GPU compatibility + - Original frozen parameters are moved to CPU to reduce active VRAM usage + - Future versions will support disk-based offloading for even larger models + + See Also: + - https://github.com/unslothai/unsloth/pull/1200 (Tesla T4 float32 requirement) + """ + # Early return with explicit None if module doesn't support mixed precision training + if not hasattr(module, "modules_to_save"): + return None + + new_dtype = module.modules_to_save.default.weight.dtype + if new_dtype == torch.float16: + # See https://github.com/unslothai/unsloth/pull/1200 + # Tesla T4 must use float32 and not float16 + new_dtype = torch.float32 + + module.modules_to_save.default.to( + device = device_type, dtype = new_dtype, non_blocking = True + ) + module.modules_to_save.default.requires_grad_(True) + + # [TODO] Move old module to CPU - should be disk! + module.original_module.to(device = offload_device, non_blocking = True) + module.original_module.requires_grad_(False) + # Fix new HF's inference code def _fast_prepare_inputs_for_generation( @@ -2711,46 +2764,16 @@ class FastLlamaModel: "Unsloth: Training embed_tokens in mixed precision to save VRAM" ) - new_dtype = model.get_input_embeddings().modules_to_save.default.weight.dtype - if new_dtype == torch.float16: - # See https://github.com/unslothai/unsloth/pull/1200 - # Tesla T4 must use float32 and not float16 - new_dtype = torch.float32 - - model.get_input_embeddings().modules_to_save.default.to( - device = DEVICE_TYPE_TORCH, dtype = new_dtype, non_blocking = True + _offload_frozen_module_for_training( + model.get_input_embeddings(), DEVICE_TYPE_TORCH ) - model.get_input_embeddings().modules_to_save.default.requires_grad_( - True - ) - - # [TODO] Move old embed_tokens to CPU - should be disk! - model.get_input_embeddings().original_module.to( - device = "cpu", non_blocking = True - ) - model.get_input_embeddings().original_module.requires_grad_(False) if "lm_head" in new_target_modules: print("Unsloth: Training lm_head in mixed precision to save VRAM") - new_dtype = model.get_output_embeddings().modules_to_save.default.weight.dtype - if new_dtype == torch.float16: - # See https://github.com/unslothai/unsloth/pull/1200 - # Tesla T4 must use float32 and not float16 - new_dtype = torch.float32 - - model.get_output_embeddings().modules_to_save.default.to( - device = DEVICE_TYPE_TORCH, dtype = new_dtype, non_blocking = True + _offload_frozen_module_for_training( + model.get_output_embeddings(), DEVICE_TYPE_TORCH ) - model.get_output_embeddings().modules_to_save.default.requires_grad_( - True - ) - - # [TODO] Move old lm_head to CPU - should be disk! - model.get_output_embeddings().original_module.to( - device = "cpu", non_blocking = True - ) - model.get_output_embeddings().original_module.requires_grad_(False) return model else: From 6465496ab2b7885f1b3d7c98046b14bb1f7f2ac6 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Fri, 9 Jan 2026 23:24:39 +0000 Subject: [PATCH 103/114] fix: use peft.utils.other for ModulesToSaveWrapper import ModulesToSaveWrapper was removed from peft.tuners.tuners_utils in PEFT 0.16.0. The class has been available in peft.utils.other since at least PEFT 0.7.1, which is the minimum version Unsloth requires. This fixes the ImportError when using PEFT >= 0.16.0. --- unsloth/models/llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index cfd900c460..39f2ba1460 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -146,7 +146,7 @@ torch_nn_functional_softmax = torch.nn.functional.softmax # SDPA has GQA internally SDPA_HAS_GQA = "enable_gqa" in scaled_dot_product_attention.__doc__ -from peft.tuners.tuners_utils import ModulesToSaveWrapper +from peft.utils.other import ModulesToSaveWrapper def _offload_frozen_module_for_training( From 5b422f7a0634c059cd1885b189d83fea7ec148e3 Mon Sep 17 00:00:00 2001 From: Duc-Viet Hoang Date: Mon, 12 Jan 2026 10:03:54 +0700 Subject: [PATCH 104/114] Complete disable `gradient_checkpointing` for vision when `use_gradient_checkpointing=False` --- unsloth/models/rl.py | 6 ++++-- unsloth/models/vision.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index e945a80354..9ec4d76ed3 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -264,17 +264,19 @@ def prepare_for_training_mode(f): def wrapper(self, *args, **kwargs): # Enable training mode _was_training = None + # Get gradient checkpointing setting from training arguments + use_gc = getattr(self.args, 'gradient_checkpointing', True) if hasattr(self, 'model') and hasattr(self.model, "training"): _was_training = self.model.training if hasattr(self, 'model') and hasattr(self.model, "for_training"): - self.model.for_training() + self.model.for_training(use_gradient_checkpointing=use_gc) output = f(self, *args, **kwargs) # Restore previous mode when possible if hasattr(self, 'model') and hasattr(self.model, "for_inference"): if _was_training is False: self.model.for_inference() elif _was_training is True and hasattr(self.model, "for_training"): - self.model.for_training() + self.model.for_training(use_gradient_checkpointing=use_gc) # Reset gradient checkpointing buffers to free memory while staying ready for next run try: reset_unsloth_gradient_checkpointing_buffers() diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 6de942d7d2..4e03e0a168 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -1273,7 +1273,7 @@ class FastBaseModel: # Since transformers 4.53, must turn on explicitly for module in model.modules(): if hasattr(module, "gradient_checkpointing"): - module.gradient_checkpointing = True + module.gradient_checkpointing = use_gradient_checkpointing # Also re-enable training for embeddings for NEFTune if hasattr(model, "get_input_embeddings"): From eaf3f932e007f809fe67cef56765886f8f9b1cae Mon Sep 17 00:00:00 2001 From: Francesco Bertolotti Date: Mon, 12 Jan 2026 16:19:43 +0100 Subject: [PATCH 105/114] wrong number of dimensions --- unsloth/kernels/swiglu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/kernels/swiglu.py b/unsloth/kernels/swiglu.py index b321f5179e..9e2680e862 100644 --- a/unsloth/kernels/swiglu.py +++ b/unsloth/kernels/swiglu.py @@ -128,7 +128,7 @@ def _DWf_DW_dfg_kernel( def swiglu_DWf_DW_dfg_kernel(DW, e, g): - batch_seq_len, hd = e.shape + batch, seq_len, hd = e.shape n_elements = e.numel() grid = lambda meta: (triton.cdiv(n_elements, meta["BLOCK_SIZE"]),) with torch_gpu_device(e.device): From 2f8c4d962be5e09b54a7b3ea6c9372d684204139 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 12 Jan 2026 19:08:13 +0000 Subject: [PATCH 106/114] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.14.10 → v0.14.11](https://github.com/astral-sh/ruff-pre-commit/compare/v0.14.10...v0.14.11) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 545c7899aa..cc188674b7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.10 + rev: v0.14.11 hooks: - id: ruff args: From 0f0b87078157435861e74ada0892d5269907d544 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 12 Jan 2026 21:32:20 -0800 Subject: [PATCH 107/114] Apply suggestion from @danielhanchen --- unsloth/kernels/swiglu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/kernels/swiglu.py b/unsloth/kernels/swiglu.py index 9e2680e862..b3ae9d40e6 100644 --- a/unsloth/kernels/swiglu.py +++ b/unsloth/kernels/swiglu.py @@ -128,7 +128,7 @@ def _DWf_DW_dfg_kernel( def swiglu_DWf_DW_dfg_kernel(DW, e, g): - batch, seq_len, hd = e.shape + batch_seq_len, hd = e.shape # Flattened to 2D, so 1st dim is bsz * seq_len n_elements = e.numel() grid = lambda meta: (triton.cdiv(n_elements, meta["BLOCK_SIZE"]),) with torch_gpu_device(e.device): From 45eeae95c5f7257a4fa39fa9c0f70933541d83f5 Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Wed, 14 Jan 2026 03:45:35 -0800 Subject: [PATCH 108/114] Update template.md --- .github/ISSUE_TEMPLATE/bug---issue.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug---issue.md b/.github/ISSUE_TEMPLATE/bug---issue.md index 397d725f95..83e0fd73a9 100644 --- a/.github/ISSUE_TEMPLATE/bug---issue.md +++ b/.github/ISSUE_TEMPLATE/bug---issue.md @@ -18,4 +18,4 @@ assignees: '' Put Minimal code to reproduce error here ###Remove Hugging Face token### ``` -🦥 You can also ask via our Reddit page: https://www.reddit.com/r/unsloth/ +🦥 You can also ask via our Reddit page: https://reddit.com/r/unsloth/ From 7f6dc63dc89aa5706b5650d95b9ff82bb5cb3c42 Mon Sep 17 00:00:00 2001 From: Datta Nimmaturi Date: Thu, 15 Jan 2026 11:11:50 +0000 Subject: [PATCH 109/114] use non lora model as base for RL --- unsloth/models/rl.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index e945a80354..62141b8700 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -1166,6 +1166,38 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import "model = self._prepare_peft_model(model, peft_config, args)\n", "pass\n" ) + # Skip add_adapter("ref") for reference model computation + # Unsloth: We comment out the "ref" adapter creation because: + # 1. We want to use the original BASE MODEL as the reference model, not the SFT/LoRA model + # 2. PEFT doesn't allow multiple adapters when target_parameters is used (MoE models) + # When "ref" is not in peft_config, GRPO/RLOO fallback uses disable_adapter() + # which gives the base model logits - exactly what we want + add_adapter_block_pattern = ( + r'([ \t]*)' # Capture leading indentation + r'if\s+is_peft_available\(\)\s+and\s+is_peft_model\(model\)\s+and\s+args\.beta\s*!=\s*0\.0\s*:' + r'(.*?)' # Match the entire block until ref_param.data.copy_ + r'ref_param\.data\.copy_\(param\.data\)' + ) + def comment_out_block(match): + """Comment out each line in the matched block, preserving indentation.""" + full_match = match.group(0) + indent = match.group(1) + lines = full_match.split('\n') + commented_lines = [] + # Add explanation comment first + commented_lines.append(f"{indent}# Unsloth: Commented out - use base model as reference, not SFT/LoRA model") + # Comment out each line - insert # after leading whitespace to preserve indentation + for line in lines: + if line.strip(): + stripped = line.lstrip() + leading_ws = line[:len(line) - len(stripped)] + commented_lines.append(f"{leading_ws}# {stripped}") + else: + commented_lines.append(line) + return '\n'.join(commented_lines) + init = re.sub(add_adapter_block_pattern, comment_out_block, init, flags=re.DOTALL) + + # Set use_vllm if not set if "args.use_vllm" in init and "model" in init and "args" in init: # .*? matches first match. .+? matches final match. From f5dde984a16c6e2903c7e36bd39ed7e6aad0f404 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 15 Jan 2026 11:25:10 +0000 Subject: [PATCH 110/114] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/rl.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 62141b8700..d5524ff5f5 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -1173,30 +1173,33 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import # When "ref" is not in peft_config, GRPO/RLOO fallback uses disable_adapter() # which gives the base model logits - exactly what we want add_adapter_block_pattern = ( - r'([ \t]*)' # Capture leading indentation - r'if\s+is_peft_available\(\)\s+and\s+is_peft_model\(model\)\s+and\s+args\.beta\s*!=\s*0\.0\s*:' - r'(.*?)' # Match the entire block until ref_param.data.copy_ - r'ref_param\.data\.copy_\(param\.data\)' + r"([ \t]*)" # Capture leading indentation + r"if\s+is_peft_available\(\)\s+and\s+is_peft_model\(model\)\s+and\s+args\.beta\s*!=\s*0\.0\s*:" + r"(.*?)" # Match the entire block until ref_param.data.copy_ + r"ref_param\.data\.copy_\(param\.data\)" ) + def comment_out_block(match): """Comment out each line in the matched block, preserving indentation.""" full_match = match.group(0) indent = match.group(1) - lines = full_match.split('\n') + lines = full_match.split("\n") commented_lines = [] # Add explanation comment first - commented_lines.append(f"{indent}# Unsloth: Commented out - use base model as reference, not SFT/LoRA model") + commented_lines.append( + f"{indent}# Unsloth: Commented out - use base model as reference, not SFT/LoRA model" + ) # Comment out each line - insert # after leading whitespace to preserve indentation for line in lines: if line.strip(): stripped = line.lstrip() - leading_ws = line[:len(line) - len(stripped)] + leading_ws = line[: len(line) - len(stripped)] commented_lines.append(f"{leading_ws}# {stripped}") else: commented_lines.append(line) - return '\n'.join(commented_lines) - init = re.sub(add_adapter_block_pattern, comment_out_block, init, flags=re.DOTALL) + return "\n".join(commented_lines) + init = re.sub(add_adapter_block_pattern, comment_out_block, init, flags = re.DOTALL) # Set use_vllm if not set if "args.use_vllm" in init and "model" in init and "args" in init: From 2164423ea6e7ef0e475dfd0d01068b68e38f6094 Mon Sep 17 00:00:00 2001 From: pluesclues <136766175+pluesclues@users.noreply.github.com> Date: Thu, 15 Jan 2026 08:01:19 -0500 Subject: [PATCH 111/114] Merge pull request #3628 from pluesclues/alternative_compute_chunked_loss Chunk Across Batch and Context length for logprob calculations for grpo --- unsloth/models/rl.py | 28 ++- unsloth/models/rl_replacements.py | 303 +++++++++++++++++++++++------- 2 files changed, 267 insertions(+), 64 deletions(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 14a07e193a..9788207c99 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -231,11 +231,13 @@ def PatchRL(FastLanguageModel): Trainer.prediction_step = unsloth_prediction_step +grpo_selective_log_softmax = RL_REPLACEMENTS["grpo_selective_log_softmax"] selective_log_softmax = RL_REPLACEMENTS["selective_log_softmax"] calculate_pad_tokens_in_prompt = RL_REPLACEMENTS["calculate_pad_tokens_in_prompt"] create_completion_attention_mask = RL_REPLACEMENTS["create_completion_attention_mask"] left_pack_padding = RL_REPLACEMENTS["left_pack_padding"] align_logprobs_with_mask = RL_REPLACEMENTS["align_logprobs_with_mask"] +autotune_batch_and_chunks = RL_REPLACEMENTS["grpo_autotune_batch_and_chunks"] RLTrainer_replacement = ''' import os @@ -247,7 +249,6 @@ import numpy as np from contextlib import nullcontext from torch.nn import functional as F import inspect -import psutil from transformers import DataCollatorForSeq2Seq, DataCollatorForLanguageModeling as TransformersDataCollatorForLanguageModeling from transformers.training_args import ParallelMode @@ -300,11 +301,13 @@ torch_compile_options = {{ "triton.cudagraphs" : False, }} +{grpo_selective_log_softmax_code} {selective_log_softmax_code} {calculate_pad_tokens_in_prompt_code} {create_completion_attention_mask_code} {left_pack_padding_code} {align_logprobs_with_mask_code} +{autotune_batch_and_chunks_code} {RL_pre} @@ -321,10 +324,20 @@ class Unsloth{RLConfig_name}({RLConfig_name}): default = -1, metadata = {{'help': 'Chunk size to reduce memory usage. -1 is most efficient.'}}, ) + unsloth_logit_chunk_multiplier : Optional[int] = field( + default = None, + metadata = {{'help': 'Multiplier for chunked logit computations.'}}, + ) + unsloth_grpo_mini_batch : Optional[int] = field( + default = None, + metadata = {{'help': 'Mini batch size for GRPO hidden state accumulation. Default is None unless user defines it.'}}, + ) {max_seq_length_pre} def __init__({RLConfig_arguments}, vllm_sampling_params = None, unsloth_num_chunks = -1, + unsloth_logit_chunk_multiplier = None, + unsloth_grpo_mini_batch = None, {max_seq_length_call} **kwargs, ): @@ -332,6 +345,15 @@ class Unsloth{RLConfig_name}({RLConfig_name}): super().__init__({RLConfig_call_args}{RLConfig_kwargs}) self.vllm_sampling_params = vllm_sampling_params self.unsloth_num_chunks = unsloth_num_chunks + if unsloth_grpo_mini_batch is not None: + if self.generation_batch_size >= unsloth_grpo_mini_batch: + self.unsloth_grpo_mini_batch = unsloth_grpo_mini_batch + else: + raise ValueError( + f"Unsloth GRPO mini batch size needs to be less than or equal to the effective generation batch size, " + f"which is self.per_device_train_batch_size * gradient_accumulation_steps." + ) + self.unsloth_logit_chunk_multiplier = unsloth_logit_chunk_multiplier {max_seq_length_post} pass @@ -1029,6 +1051,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): # Selective log softmax and other functions selective_log_softmax_code = inspect.getsource(selective_log_softmax) + grpo_selective_log_softmax_code = inspect.getsource(grpo_selective_log_softmax) calculate_pad_tokens_in_prompt_code = inspect.getsource( calculate_pad_tokens_in_prompt ) @@ -1037,6 +1060,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): ) left_pack_padding_code = inspect.getsource(left_pack_padding) align_logprobs_with_mask_code = inspect.getsource(align_logprobs_with_mask) + autotune_batch_and_chunks_code = inspect.getsource(autotune_batch_and_chunks) # Get final source code RLTrainer_source = RLTrainer_replacement.format( RLTrainer_name = RLTrainer_name, @@ -1058,8 +1082,10 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): max_seq_length_call = max_seq_length_call, max_seq_length_post = max_seq_length_post, selective_log_softmax_code = selective_log_softmax_code, + grpo_selective_log_softmax_code = grpo_selective_log_softmax_code, calculate_pad_tokens_in_prompt_code = calculate_pad_tokens_in_prompt_code, create_completion_attention_mask_code = create_completion_attention_mask_code, + autotune_batch_and_chunks_code = autotune_batch_and_chunks_code, left_pack_padding_code = left_pack_padding_code, align_logprobs_with_mask_code = align_logprobs_with_mask_code, ) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 5e079335ae..ff36da125d 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -50,7 +50,7 @@ RL_ADDITIONAL_FUNCTIONS = defaultdict(list) torch_compile_options = { "epilogue_fusion": True, - "max_autotune": True, + "max_autotune": False, # I saw speedups, but not sure if this has issues in collab "shape_padding": True, "trace.enabled": False, "triton.cudagraphs": False, @@ -258,18 +258,20 @@ def grpo_trainer__generate_and_score_completions(function_name, function): # The new multi-line string that will replace the line above replacement_lines = """ + max_left_pad = None batch_size = self.args.per_device_train_batch_size if mode == "train" else self.args.per_device_eval_batch_size try: # TRL 0.23.1 and below path if not has_images: # Left pad prompt before calculation old and ref hidden states - prompt_completion_ids = left_pack_padding(prompt_completion_ids, self.processing_class.pad_token_id) - self.model.for_training() + left_pad_tokens_per_prompt = calculate_pad_tokens_in_prompt(prompt_completion_ids, logits_to_keep, self.processing_class.pad_token_id) + max_left_pad = torch.max(left_pad_tokens_per_prompt).item() except: # TRL 0.24.0 and below path if images is None: # Left pad prompt before calculation old and ref hidden states - prompt_completion_ids = left_pack_padding(prompt_completion_ids, self.processing_class.pad_token_id) + left_pad_tokens_per_prompt = calculate_pad_tokens_in_prompt(prompt_completion_ids, logits_to_keep, self.processing_class.pad_token_id) + max_left_pad = torch.max(left_pad_tokens_per_prompt).item() self.model.for_training()""" function = function.replace(line_to_replace, replacement_lines) @@ -346,17 +348,45 @@ def grpo_trainer__generate_and_score_completions(function_name, function): if self.use_vllm:""" function = function.replace(replace_part, new_replacement) + # Important note: we disable TRL's importance sampling logic + # It is disabled because the LLM path moves left padding to the right. + # We must adjust the vLLM sampling_logprob tensor in Unsloth to account for this. + string_to_find = "if self.use_vllm and self.vllm_importance_sampling_correction:" + + replacement_string = ( + "if False and self.use_vllm and self.vllm_importance_sampling_correction:" + ) + + function = function.replace(string_to_find, replacement_string) + string_to_find = """ if "image_sizes" in prompt_inputs: output["image_sizes"] = prompt_inputs["image_sizes"]""" replacement_string = """ if "image_sizes" in prompt_inputs: output["image_sizes"] = prompt_inputs["image_sizes"] - - if self.use_vllm: - try: + if max_left_pad is not None: + output["max_left_pad"] = torch.tensor(prompt_ids.shape[0] * [max_left_pad]).unsqueeze(-1) + try: + if self.use_vllm and getattr(self, "vllm_importance_sampling_correction", False): output["sampling_per_token_logps"] = sampling_per_token_logps - except NameError: - output["sampling_per_token_logps"] = None""" + except NameError: + output["sampling_per_token_logps"] = None""" + + function = function.replace(string_to_find, replacement_string) + + # This path is for TRL 0.24.0 images is a variable exclusive to this version + string_to_find = """ if images is not None: + output["num_images"] = num_images""" + + replacement_string = """ if images is not None: + output["num_images"] = num_images + if max_left_pad is not None: + output["max_left_pad"] = torch.tensor(prompt_ids.shape[0] * [max_left_pad]).unsqueeze(-1) + try: + if self.use_vllm and getattr(self, "vllm_importance_sampling_correction", False): + output["sampling_per_token_logps"] = sampling_per_token_logps + except NameError: + output["sampling_per_token_logps"] = None""" function = function.replace(string_to_find, replacement_string) @@ -532,12 +562,12 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function): *args, **kwargs, ): + # All Unsloth code here in this function is licensed under AGPL3 # if True: # os.environ.get('UNSLOTH_USE_NEW_MODEL', '0') == '0': # return None, None # logps, entropies Unsloth efficient GRPO if compute_efficient: return None, None else: - # Otherwise, calculate normally: if not hasattr(self, "_autocast_dtype"): self._autocast_dtype = ( torch.float16 @@ -556,47 +586,199 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function): kwargs.get("image_sizes", None), ) - os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "1" - unwrapped_model = self.accelerator.unwrap_model( model, keep_fp32_wrapper = False ) - with torch.amp.autocast(device_type = "cuda", dtype = self._autocast_dtype): - with _get_inference_mode_context_manager(model): - if pixel_values is None: - attention_mask = input_ids != self.processing_class.pad_token_id - attention_mask = attention_mask.to(attention_mask.dtype) - # We add 1 to `logits_to_keep` because the last logits of the sequence is later excluded - logits = unwrapped_model( - input_ids = input_ids, - attention_mask = attention_mask, - pixel_values = pixel_values, - image_grid_thw = image_grid_thw, - pixel_attention_mask = pixel_attention_mask, - image_sizes = image_sizes, - # logits_to_keep = logits_to_keep + 1, - ).logits + lm_head = self.model.get_output_embeddings().weight + + dtype_bytes = ( + 16 if self._autocast_dtype in [torch.float16, torch.bfloat16] else 32 + ) + total_rows = input_ids.shape[0] + seq_len = input_ids.shape[1] + hidden_dim = lm_head.shape[1] + vocab_dim = lm_head.shape[0] + + if self.args.unsloth_grpo_mini_batch is None: + B, multiplier = autotune_batch_and_chunks( + total_rows, + seq_len, + hidden_dim, + vocab_dim, + dtype_bytes, + self.args.unsloth_logit_chunk_multiplier, + ) + B = total_rows // B + else: + B = self.args.unsloth_grpo_mini_batch + + if self.args.unsloth_logit_chunk_multiplier is None: + multiplier = max(4, seq_len // 4096) + else: + multiplier = self.args.unsloth_logit_chunk_multiplier + + all_logprobs_list = [] + if pixel_values is None: + left_pad_tokens_per_prompt = calculate_pad_tokens_in_prompt( + input_ids, logits_to_keep, self.processing_class.pad_token_id + ) + max_left_pad = torch.max(left_pad_tokens_per_prompt).item() + input_ids = left_pack_padding( + input_ids, self.processing_class.pad_token_id + ) + attention_mask = input_ids != self.processing_class.pad_token_id + attention_mask = attention_mask.to(attention_mask.dtype) + else: + max_left_pad = 0 + + # input_ids_chunks = torch.chunk(input_ids, chunks = B, dim = 0) + attention_mask_chunks = torch.chunk(attention_mask, chunks = B, dim = 0) + + def chunk_optional(tensor, chunks): + if tensor is None: + return [None] * chunks + return torch.chunk(tensor, chunks = chunks, dim = 0) + + import math + + total_samples = input_ids.shape[0] + batch_size = math.ceil(total_samples / B) + + input_ids_chunks = [] + attention_mask_chunks = [] + pixel_values_chunks = [] + image_grid_thw_chunks = [] + pixel_attention_mask_chunks = [] + + current_pixel_idx = 0 + # TRL 0.23.0 batching logic + for start in range(0, total_samples, batch_size): + end = start + batch_size + + input_ids_chunks.append(input_ids[start:end]) + attention_mask_chunks.append(attention_mask[start:end]) + + if image_grid_thw is not None and pixel_values is not None: + grid_slice = image_grid_thw[start:end] + image_grid_thw_chunks.append(grid_slice) + + batch_pixel_count = grid_slice.prod(dim = -1).sum().item() + + start_pixel_idx = current_pixel_idx + end_pixel_idx = current_pixel_idx + batch_pixel_count + + pixel_values_chunks.append( + pixel_values[start_pixel_idx:end_pixel_idx] + ) + + if pixel_attention_mask is not None: + pixel_attention_mask_chunks.append( + pixel_attention_mask[start_pixel_idx:end_pixel_idx] + ) else: - logits = unwrapped_model( - input_ids = input_ids, - attention_mask = attention_mask, - pixel_values = pixel_values, - image_grid_thw = image_grid_thw, - pixel_attention_mask = pixel_attention_mask, - image_sizes = image_sizes, - logits_to_keep = logits_to_keep + 1, - ).logits + pixel_attention_mask_chunks.append(None) + current_pixel_idx = end_pixel_idx + + else: + pixel_values_chunks.append(None) + image_grid_thw_chunks.append(None) + pixel_attention_mask_chunks.append(None) + + if image_sizes is not None and not isinstance(image_sizes, torch.Tensor): + image_sizes_chunks = [[size] for size in image_sizes] + else: + image_sizes_chunks = chunk_optional(image_sizes, B) + + temperature = self.temperature + logit_softcapping = getattr(model.config, "final_logit_softcapping", 0) + if logit_softcapping is None: + logit_softcapping = 0 + logit_scale_multiply = getattr(model.config, "logit_scale", 0) + if logit_scale_multiply is None: + logit_scale_multiply = 0 + logit_scale_divide = getattr(model.config, "logits_scaling", 0) + if logit_scale_divide is None: + logit_scale_divide = 0 + + zipped_inputs = zip( + input_ids_chunks, + attention_mask_chunks, + pixel_values_chunks, + image_grid_thw_chunks, + pixel_attention_mask_chunks, + image_sizes_chunks, + ) + os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "1" + + with _get_inference_mode_context_manager(model): + for ( + input_ids_chunk, + attention_mask_chunk, + pixel_values_chunk, + image_grid_thw_chunk, + pixel_attention_mask_chunk, + image_sizes_chunk, + ) in zipped_inputs: + with torch.amp.autocast( + device_type = "cuda", dtype = self._autocast_dtype + ): + if pixel_values is None: + logits_chunk = unwrapped_model( + input_ids = input_ids_chunk, + attention_mask = attention_mask_chunk, + pixel_values = pixel_values_chunk, + image_grid_thw = image_grid_thw_chunk, + pixel_attention_mask = pixel_attention_mask_chunk, + image_sizes = image_sizes_chunk, + ).logits + + completion_input_ids_chunk = input_ids_chunk[ + :, -(logits_to_keep + max_left_pad) : + ] + logits_chunk = logits_chunk[ + :, -(logits_to_keep + max_left_pad + 1) :, : + ] + logits_chunk = logits_chunk[:, :-1, :] + else: + # Essentially, for VLMs we do not go via the optimized path in models/, + # so we don't encounter the Flash Attn left-padding issue. + logits_chunk = unwrapped_model( + input_ids = input_ids_chunk, + attention_mask = attention_mask_chunk, + pixel_values = pixel_values_chunk, + image_grid_thw = image_grid_thw_chunk, + pixel_attention_mask = pixel_attention_mask_chunk, + image_sizes = image_sizes_chunk, + logits_to_keep = logits_to_keep + 1, + ).logits + + logits_chunk = logits_chunk[:, :-1, :] + completion_input_ids_chunk = input_ids_chunk[ + :, -logits_to_keep: + ] + + logprobs_chunk = chunked_hidden_states_selective_log_softmax( + logits_chunk, + lm_head, + completion_input_ids_chunk, + chunks = input_ids_chunk.shape[0] * multiplier, + logit_scale_multiply = logit_scale_multiply, + logit_scale_divide = logit_scale_divide, + logit_softcapping = logit_softcapping, + temperature = temperature, + ) + # This is needed to avoid race conditions with GPT OSS offload_embbed=True + # However, it seems that this line does not slow down or disrupt models. + torch.cuda.synchronize() + all_logprobs_list.append(logprobs_chunk) + logprobs = torch.cat(all_logprobs_list, dim = 0) entropies = None - if compute_entropy: - from trl.trainer.utils import entropy_from_logits - - entropies = entropy_from_logits(logits) os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "0" - # logits = logits[:, :-1, :] # (B, L-1, V), exclude the last logit: it corresponds to the next token pred - return logits.detach(), entropies # logps, entropies + + return logprobs.detach(), entropies # logps, entropies # input_ids = input_ids[:, -logits_to_keep:] # For transformers<=4.48, logits_to_keep argument isn't supported, so here we drop logits ourselves. # See https://github.com/huggingface/trl/issues/2770 @@ -708,14 +890,14 @@ def grpo_trainer_compute_loss(function_name, function): # ref_per_token_logps = per_token_logps = get_logps_func(model, input_ids, attention_mask, logits_to_keep) # else: # ref_per_token_logps = None - ref_hidden_states = inputs.get("ref_per_token_logps", None) + ref_logps = inputs.get("ref_per_token_logps", None) # per_token_kl = torch.exp(ref_per_token_logps - per_token_logps) - (ref_per_token_logps - per_token_logps) - 1 # x - x.detach() allows for preserving gradients from x advantages = inputs["advantages"] # per_token_loss = torch.exp(per_token_logps - per_token_logps.detach()) * advantages.unsqueeze(1) # per_token_loss = -(per_token_loss - self.beta * per_token_kl) # loss = ((per_token_loss * completion_mask).sum(dim=1) / completion_mask.sum(dim=1)).mean() - old_hidden_states = inputs.get("old_per_token_logps", None) + old_logps = inputs.get("old_per_token_logps", None) input_ids = input_ids[:, -logits_to_keep:] @@ -730,24 +912,13 @@ def grpo_trainer_compute_loss(function_name, function): if logit_scale_divide is None: logit_scale_divide = 0 + max_left_pad = inputs.get("max_left_pad", 0) if per_token_logps is not None: - if ref_hidden_states is not None: - ref_hidden_states = ref_hidden_states[ - :, :-1, : - ] # (B, L-1, V), exclude the last logit: it corresponds to the next token pred - if old_hidden_states is not None: - old_hidden_states = old_hidden_states[ - :, :-1, : - ] # (B, L-1, V), exclude the last logit: it corresponds to the next token pred - per_token_logps = per_token_logps[ - :, :-1, : - ] # (B, L-1, V), exclude the last logit: it corresponds to the next token pred - loss, completion_length, mean_kl, delta, flat_is_ratio = ( grpo_compute_loss_slow( - ref_hidden_states, + ref_logps, per_token_logps, - old_hidden_states, + old_logps, input_ids, completion_mask, self.beta, @@ -761,6 +932,7 @@ def grpo_trainer_compute_loss(function_name, function): max_completion_length = self.args.max_completion_length, delta = self.args.delta, temperature = self.args.temperature, + max_left_pad = max_left_pad, logit_softcapping = logit_softcapping, logit_scale_multiply = logit_scale_multiply, logit_scale_divide = logit_scale_divide, @@ -781,8 +953,8 @@ def grpo_trainer_compute_loss(function_name, function): logits_to_keep = logits_to_keep, completion_mask = completion_mask, advantages = advantages, - old_hidden_states = old_hidden_states, - ref_hidden_states = ref_hidden_states, + old_logps = old_logps, + ref_logps = ref_logps, n_chunks = self.args.unsloth_num_chunks, loss_type = self.args.loss_type, importance_sampling_level = self.importance_sampling_level, @@ -791,6 +963,7 @@ def grpo_trainer_compute_loss(function_name, function): max_completion_length = self.args.max_completion_length, delta = self.args.delta, temperature = self.args.temperature, + max_left_pad = max_left_pad, logit_softcapping = logit_softcapping, logit_scale_multiply = logit_scale_multiply, logit_scale_divide = logit_scale_divide, @@ -809,8 +982,8 @@ def grpo_trainer_compute_loss(function_name, function): logits_to_keep = logits_to_keep, completion_mask = completion_mask, advantages = advantages, - old_hidden_states = old_hidden_states, - ref_hidden_states = ref_hidden_states, + old_logps = old_logps, + ref_logps = ref_logps, n_chunks = self.args.unsloth_num_chunks, temperature = self.args.temperature, logit_softcapping = logit_softcapping, @@ -827,7 +1000,11 @@ def grpo_trainer_compute_loss(function_name, function): self._metrics["completion_length"].append(completion_length.item()) self._metrics["kl"].append(mean_kl.item()) - if self.use_vllm and delta is not None: + if ( + self.use_vllm + and delta is not None + and getattr(self, "vllm_importance_sampling_correction", False) + ): mean_delta = ( torch.mean(delta) if delta.numel() > 0 From 6edbfbc43511162335d8d0e601f8f858783227b1 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 15 Jan 2026 05:09:26 -0800 Subject: [PATCH 112/114] Update _utils.py --- unsloth/models/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index b0ad0a3082..6028635fb9 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2026.1.2" +__version__ = "2026.1.3" __all__ = [ "SUPPORTS_BFLOAT16", From ecd10f2e55e90cdafe4f71c9cbbfa58a4b89dc70 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 15 Jan 2026 07:00:25 -0800 Subject: [PATCH 113/114] Update pyproject.toml --- pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7fa249e64c..05fb690da5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ huggingfacenotorch = [ "sentencepiece>=0.2.0", "datasets>=3.4.1,!=4.0.*,!=4.1.0,<4.4.0", "accelerate>=0.34.1", - "peft>=0.7.1,!=0.11.0", + "peft>=0.18.0,!=0.11.0", "huggingface_hub>=0.34.0", "hf_transfer", "diffusers", @@ -60,7 +60,7 @@ huggingfacenotorch = [ ] huggingface = [ "unsloth[huggingfacenotorch]", - "unsloth_zoo>=2026.1.2", + "unsloth_zoo>=2026.1.3", "torchvision", "unsloth[triton]", ] @@ -523,7 +523,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3 ; ('linux' in sys_platform)", ] colab-new = [ - "unsloth_zoo>=2026.1.2", + "unsloth_zoo>=2026.1.3", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,<=4.57.3", @@ -542,7 +542,7 @@ colab-new = [ colab-no-deps = [ "accelerate>=0.34.1", "trl>=0.18.2,!=0.19.0,<=0.24.0", - "peft>=0.7.1", + "peft>=0.18.0", "xformers ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "protobuf", From 94b744ef5ea996bc6160cda0cafd1953827e43c1 Mon Sep 17 00:00:00 2001 From: Datta Nimmaturi Date: Tue, 20 Jan 2026 11:41:02 +0530 Subject: [PATCH 114/114] [transformers] [v5] remove unused hybridcache (#3910) * remote unused hybridcache * cleanup --- unsloth/models/vision.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 4e03e0a168..6835f2e986 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -106,7 +106,7 @@ PRE_COMPILE_INFERENCE = [ "gpt_oss", ] -from transformers import GenerationConfig, CompileConfig, HybridCache, AutoConfig +from transformers import GenerationConfig, CompileConfig, AutoConfig try: from transformers import PreTrainedConfig @@ -117,8 +117,6 @@ except: HAS_TORCH_DTYPE = "torch_dtype" in PretrainedConfig.__doc__ -from transformers import GenerationConfig, CompileConfig, HybridCache - _compile_config = CompileConfig( fullgraph = False, dynamic = None,