From de8c1cd9ad854300b0d477a1d129768ad19d9bdc Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 9 Dec 2025 01:02:26 -0800 Subject: [PATCH 01/96] Update _utils.py --- unsloth/models/_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index f0db65fdbf..e3814163ef 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -85,6 +85,7 @@ import re from dataclasses import dataclass, field import functools import textwrap +import logging import warnings, subprocess, inspect, psutil, os, math from unsloth_zoo.utils import Version, get_quant_type from importlib.metadata import version as importlib_version @@ -167,9 +168,9 @@ warnings.filterwarnings( ) warnings.filterwarnings(action = "ignore", category = RuntimeWarning, module = "multiprocess") warnings.filterwarnings(action = "ignore", category = UserWarning, module = "triton") -# Stop "Special tokens have been added in the vocabulary, ..." -import logging +warnings.filterwarnings(action = "ignore", category = UserWarning, module = "bitsandbytes") +# Stop "Special tokens have been added in the vocabulary, ..." logging.getLogger("transformers.tokenization_utils_base").setLevel(logging.CRITICAL + 1) From dfeecb9eeede1e3a0f2ac73bd88705a010e3b6bd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 10 Dec 2025 12:12:49 +0000 Subject: [PATCH 02/96] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/import_fixes.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 67d9bf286d..858e910c20 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -22,6 +22,7 @@ import logging UNSLOTH_ENABLE_LOGGING = os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") == "1" + def Version(version): try: new_version = str(version) @@ -30,13 +31,14 @@ def Version(version): raise Exception(str(e)) new_version = new_version.group(0).rstrip(".") if new_version != version: - new_version += ".1" # Add .1 for dev / alpha / beta / rc + new_version += ".1" # Add .1 for dev / alpha / beta / rc return TrueVersion(new_version) except: from inspect import getframeinfo, stack + caller = getframeinfo(stack()[1][0]) raise RuntimeError( - f"Unsloth: Could not get version for `{version}`\n"\ + f"Unsloth: Could not get version for `{version}`\n" f"File name = [{caller.filename}] Line number = [{caller.lineno}]" ) From a86363eca972118e2c6c4bb91c42810851fc72d6 Mon Sep 17 00:00:00 2001 From: oKatanaaa Date: Thu, 11 Dec 2025 03:21:02 +0000 Subject: [PATCH 03/96] fix: weights tying --- unsloth/models/llama.py | 48 ++++++++++++++++++++++++++++++++++++++++ unsloth/models/vision.py | 1 + 2 files changed, 49 insertions(+) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 4c9337ccf9..d38018ee1b 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -2601,6 +2601,7 @@ class FastLlamaModel: loftq_config = {}, temporary_location = "_unsloth_temporary_saved_buffers", qat_scheme = None, + ensure_weight_tying = False, **kwargs, ): if os.environ.get("UNSLOTH_USE_NEW_MODEL", "0") == "1": @@ -2630,6 +2631,7 @@ class FastLlamaModel: init_lora_weights = init_lora_weights, loftq_config = loftq_config, temporary_location = temporary_location, + ensure_weight_tying = ensure_weight_tying, **kwargs, ) if os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1": @@ -2953,6 +2955,7 @@ class FastLlamaModel: loftq_config = loftq_config, use_rslora = use_rslora, modules_to_save = modules_to_save, + ensure_weight_tying = ensure_weight_tying, **kwargs, ) if not SUPPORTS_LOFTQ: @@ -3002,6 +3005,51 @@ class FastLlamaModel: model = FastLlamaModel.patch_peft_model(model, use_gradient_checkpointing) + if ensure_weight_tying: + try: + input_embeddings = model.get_input_embeddings() + output_embeddings = model.get_output_embeddings() + + if input_embeddings is not None and output_embeddings is not None: + def _retie_parameter(target_module, source_module): + if not hasattr(source_module, "weight"): + return + weight = source_module.weight + # Remove existing registration to avoid "attribute already exists" + if "weight" in getattr(target_module, "_parameters", {}): + target_module._parameters.pop("weight") + if hasattr(target_module, "weight"): + try: + delattr(target_module, "weight") + except Exception: + pass + target_module.register_parameter("weight", weight) + + # Tie trainable copies created by ModulesToSaveWrapper first (these are used in forward) + if hasattr(input_embeddings, "modules_to_save") and hasattr( + output_embeddings, "modules_to_save" + ): + if hasattr(input_embeddings.modules_to_save, "default") and hasattr( + output_embeddings.modules_to_save, "default" + ): + _retie_parameter( + output_embeddings.modules_to_save.default, + input_embeddings.modules_to_save.default, + ) + + # Tie original_module references as well if present + if hasattr(input_embeddings, "original_module") and hasattr( + output_embeddings, "original_module" + ): + _retie_parameter( + output_embeddings.original_module, + input_embeddings.original_module, + ) + except Exception as e: + logger.warning_once( + f"Unsloth: Failed to ensure weight tying between embeddings and lm_head: {e}" + ) + if train_embed_tokens: print("Unsloth: Training embed_tokens in mixed precision to save VRAM") assert hasattr(model.get_input_embeddings(), "modules_to_save") diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index ed19f587cf..9f847f2837 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -930,6 +930,7 @@ class FastBaseModel: task_type = TaskType.CAUSAL_LM, temporary_location = "_unsloth_temporary_saved_buffers", qat_scheme = None, + ensure_weight_tying = False, **kwargs, ): if os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1": From 1837de275165b5307b057036c420f2778c6d1343 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 11 Dec 2025 03:31:41 +0000 Subject: [PATCH 04/96] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/llama.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index d38018ee1b..e0d8cbcf25 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -3011,6 +3011,7 @@ class FastLlamaModel: output_embeddings = model.get_output_embeddings() if input_embeddings is not None and output_embeddings is not None: + def _retie_parameter(target_module, source_module): if not hasattr(source_module, "weight"): return @@ -3029,9 +3030,9 @@ class FastLlamaModel: if hasattr(input_embeddings, "modules_to_save") and hasattr( output_embeddings, "modules_to_save" ): - if hasattr(input_embeddings.modules_to_save, "default") and hasattr( - output_embeddings.modules_to_save, "default" - ): + if hasattr( + input_embeddings.modules_to_save, "default" + ) and hasattr(output_embeddings.modules_to_save, "default"): _retie_parameter( output_embeddings.modules_to_save.default, input_embeddings.modules_to_save.default, From 10e8518ae3eee6582b8bd35c146a937ee79738b4 Mon Sep 17 00:00:00 2001 From: Datta Nimmaturi Date: Fri, 12 Dec 2025 17:03:39 +0530 Subject: [PATCH 05/96] [FIX] [Transformers] VLM input embeds fix for gradients (#3715) * Fix get_input_embeds call for VLMs * patch input_require_grads instead * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * cleanup old patch * cleanup old patch * cleanup * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Apply suggestion from @danielhanchen * use logger instead of prints * Move unsloth present set * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Daniel Han --- unsloth/__init__.py | 8 ++- unsloth/import_fixes.py | 105 ++++++++++++++++++++++++++++++---------- 2 files changed, 85 insertions(+), 28 deletions(-) diff --git a/unsloth/__init__.py b/unsloth/__init__.py index 47739fad15..5df43d3117 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -17,6 +17,9 @@ from packaging.version import Version import os, re, subprocess, inspect, functools import numpy as np +# Log Unsloth is being used +# We want logger in import_fixes and hence setting it here for zoo to be importable +os.environ["UNSLOTH_IS_PRESENT"] = "1" # Fix some issues before importing other packages from .import_fixes import ( fix_message_factory_issue, @@ -63,8 +66,6 @@ os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python" # "pinned_use_cuda_host_register:True,"\ # "pinned_num_register_threads:8" -# Log Unsloth is being used -os.environ["UNSLOTH_IS_PRESENT"] = "1" from importlib.metadata import version as importlib_version from importlib.metadata import PackageNotFoundError @@ -123,6 +124,7 @@ from .import_fixes import ( patch_ipykernel_hf_xet, patch_trackio, patch_datasets, + patch_enable_input_require_grads, ) fix_xformers_performance_issue() @@ -132,6 +134,7 @@ ignore_logger_messages() patch_ipykernel_hf_xet() patch_trackio() patch_datasets() +patch_enable_input_require_grads() del fix_xformers_performance_issue del fix_vllm_aimv2_issue @@ -140,6 +143,7 @@ del ignore_logger_messages del patch_ipykernel_hf_xet del patch_trackio del patch_datasets +del patch_enable_input_require_grads # Torch 2.4 has including_emulation if DEVICE_TYPE == "cuda": diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 858e910c20..a43be23194 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -19,8 +19,7 @@ from importlib.metadata import version as importlib_version from packaging.version import Version as TrueVersion import re import logging - -UNSLOTH_ENABLE_LOGGING = os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") == "1" +from unsloth_zoo.log import logger def Version(version): @@ -71,8 +70,7 @@ def fix_message_factory_issue(): return if not hasattr(google.protobuf.message_factory, "MessageFactory"): - if UNSLOTH_ENABLE_LOGGING: - print("Unsloth: Patching protobuf.MessageFactory as it doesn't exist") + logger.info("Unsloth: Patching protobuf.MessageFactory as it doesn't exist") google.protobuf.message_factory.MessageFactory = MessageFactory elif ( hasattr(google.protobuf.message_factory, "MessageFactory") @@ -82,8 +80,7 @@ def fix_message_factory_issue(): and not hasattr(google.protobuf.message_factory, "GetMessageClass") ): google.protobuf.message_factory.MessageFactory = MessageFactory - if UNSLOTH_ENABLE_LOGGING: - print("Unsloth: Patching protobuf.MessageFactory as it doesn't exist") + logger.info("Unsloth: Patching protobuf.MessageFactory as it doesn't exist") elif ( hasattr(google.protobuf.message_factory, "MessageFactory") and not hasattr( @@ -97,8 +94,7 @@ def fix_message_factory_issue(): return GetMessageClass(descriptor) google.protobuf.message_factory.MessageFactory.GetPrototype = GetPrototype - if UNSLOTH_ENABLE_LOGGING: - print("Unsloth: Patching protobuf.MessageFactory.GetPrototype") + logger.info("Unsloth: Patching protobuf.MessageFactory.GetPrototype") pass except: pass @@ -126,13 +122,11 @@ def fix_xformers_performance_issue(): f.seek(0) f.write(text) f.truncate() - if UNSLOTH_ENABLE_LOGGING: - print( - "Unsloth: Patching Xformers to fix some performance issues." - ) + logger.info( + "Unsloth: Patching Xformers to fix some performance issues." + ) except Exception as e: - if UNSLOTH_ENABLE_LOGGING: - print(f"Unsloth: Failed patching Xformers with error = {str(e)}") + logger.info(f"Unsloth: Failed patching Xformers with error = {str(e)}") # ValueError: 'aimv2' is already used by a Transformers config, pick another name. @@ -167,13 +161,11 @@ def fix_vllm_aimv2_issue(): f.seek(0) f.write(text) f.truncate() - if UNSLOTH_ENABLE_LOGGING: - print( - "Unsloth: Patching vLLM to fix `'aimv2' is already used by a Transformers config, pick another name.`" - ) + logger.info( + "Unsloth: Patching vLLM to fix `'aimv2' is already used by a Transformers config, pick another name.`" + ) except Exception as e: - if UNSLOTH_ENABLE_LOGGING: - print(f"Unsloth: Failed patching vLLM with error = {str(e)}") + logger.info(f"Unsloth: Failed patching vLLM with error = {str(e)}") def fix_vllm_guided_decoding_params(): @@ -274,8 +266,70 @@ def check_fbgemm_gpu_version(): raise ImportError( f"Unsloth: fbgemm_gpu_genai=={fbgemm_gpu_version} detected. It might cause unexpected issues like segmentation faults. Please uninstall the current one by doing `pip uninstall fbgemm-gpu` && `pip install fbgemm-gpu` to install fbgemm-gpu 1.4.0 or newer!" ) - elif UNSLOTH_ENABLE_LOGGING: - print(f"Unsloth: fbgemm_gpu_genai=={fbgemm_gpu_version} detected.") + logger.info(f"Unsloth: fbgemm_gpu_genai=={fbgemm_gpu_version} detected.") + + +def patch_enable_input_require_grads(): + """ + Patch transformers PreTrainedModel.enable_input_require_grads to handle vision models + that raise NotImplementedError from get_input_embeddings(). + + """ + import inspect + from transformers import PreTrainedModel + + # Check if the original function iterates over self.modules() instead of just returning the enable_input_require_grads + # Ref: https://github.com/huggingface/transformers/pull/41993/files#diff-6b72b98c4c2dcfc6cc606843917733f5d858374fbc22a735ff483bbc0c1e63eaL1979-R1996 + try: + original_source = inspect.getsource(PreTrainedModel.enable_input_require_grads) + except (OSError, TypeError): + return + + # Only patch if the new pattern exists (iterating over self.modules()) + if "for module in self.modules()" not in original_source: + return + + def _patched_enable_input_require_grads(self): + def make_inputs_require_grads(module, input, output): + output.requires_grad_(True) + + hooks = [] + seen_modules = set() + + for module in self.modules(): + if not ( + isinstance(module, PreTrainedModel) + and hasattr(module, "get_input_embeddings") + ): + continue + + try: + input_embeddings = module.get_input_embeddings() + except NotImplementedError: + # Vision models may not implement get_input_embeddings - skip them + # For GLM V4.6 for example, this skips only `self.visual` + continue + + if input_embeddings is None: + continue + + embedding_id = id(input_embeddings) + if embedding_id in seen_modules: + continue + + seen_modules.add(embedding_id) + hooks.append( + input_embeddings.register_forward_hook(make_inputs_require_grads) + ) + + self._require_grads_hooks = hooks + if hooks: + self._require_grads_hook = hooks[0] + + PreTrainedModel.enable_input_require_grads = _patched_enable_input_require_grads + logger.info( + "Unsloth: Patched enable_input_require_grads for vision model compatibility" + ) def torchvision_compatibility_check(): @@ -313,7 +367,6 @@ def torchvision_compatibility_check(): f"but found torchvision=={torchvision_version}. " f"Please refer to https://pytorch.org/get-started/previous-versions/ for more information." ) - elif UNSLOTH_ENABLE_LOGGING: - print( - f"Unsloth: torch=={torch_version} and torchvision=={torchvision_version} are compatible." - ) + logger.info( + f"Unsloth: torch=={torch_version} and torchvision=={torchvision_version} are compatible." + ) From 800a6d42e06df4d9fc81cb32af1a27076d95a0a6 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 12 Dec 2025 03:41:09 -0800 Subject: [PATCH 06/96] Update rope_embedding.py --- unsloth/kernels/rope_embedding.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.py index e93cbd1544..2adc9ecc5a 100644 --- a/unsloth/kernels/rope_embedding.py +++ b/unsloth/kernels/rope_embedding.py @@ -20,13 +20,6 @@ from ..device_type import DEVICE_COUNT from .utils import calculate_settings, torch_gpu_device, torch_device_stream -@triton.heuristics( - { - "BACKWARD_PASS": lambda args: bool(args["BACKWARD_PASS"]), - "HAS_ROPE_INDICES": lambda args: bool(args["HAS_ROPE_INDICES"]), - } -) -@triton.jit def _rope_embedding_QK( Q, Q_batch_stride, @@ -104,9 +97,17 @@ def _rope_embedding_QK( tl.store(k_ptr + half_head_dim + col_offsets, k1 * cos1 + k0 * sin1, mask = mask) -ROPE_GROUP_SIZE: int = 4 +_rope_embedding_QK = triton.jit(_rope_embedding_QK) +_rope_embedding_QK = triton.heuristics( + { + "BACKWARD_PASS": lambda args: bool(args["BACKWARD_PASS"]), + "HAS_ROPE_INDICES": lambda args: bool(args["HAS_ROPE_INDICES"]), + } +)(_rope_embedding_QK) +ROPE_GROUP_SIZE: int = 4 + def _rope_embedding( Q, Q_row_stride: tl.constexpr, From e478faef18442c1467d7452bb8bc1bada967756d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 12 Dec 2025 04:58:43 -0800 Subject: [PATCH 07/96] Fixes --- unsloth/import_fixes.py | 2 +- unsloth/models/rl_replacements.py | 8 +++++++- unsloth/trainer.py | 5 +++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index a43be23194..4cbe57cd95 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -282,7 +282,7 @@ def patch_enable_input_require_grads(): # Ref: https://github.com/huggingface/transformers/pull/41993/files#diff-6b72b98c4c2dcfc6cc606843917733f5d858374fbc22a735ff483bbc0c1e63eaL1979-R1996 try: original_source = inspect.getsource(PreTrainedModel.enable_input_require_grads) - except (OSError, TypeError): + except: return # Only patch if the new pattern exists (iterating over self.modules()) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 2cf3527c9b..7dab0d7307 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -26,7 +26,9 @@ import torch import inspect from collections import defaultdict from unsloth_zoo.rl_replacements import RL_REPLACEMENTS, left_pack_padding +from unsloth_zoo.utils import Version from unsloth_zoo.log import logger +import importlib.util from ..device_type import ( is_hip, get_device_type, @@ -942,11 +944,15 @@ def openenv_vllm_reload_weights(): # # The fix: Use wake_up() with no tags, which wakes everything. Unsloth's patched # CuMemAllocator.wake_up skips weights anyway, so this is safe. + if importlib.util.find_spec("trl") is None: + return + if Version(importlib_version("trl")) < Version("0.26.0"): + return try: import trl.experimental.openenv.utils as openenv_utils import trl.experimental.openenv as openenv except ImportError as e: - logger.warning(f"Unsloth: Failed to import trl openenv: {e}") + logger.info(f"Unsloth: Failed to import trl openenv: {e}") return src = inspect.getsource(openenv_utils.generate_rollout_completions) diff --git a/unsloth/trainer.py b/unsloth/trainer.py index 339af63f33..5cd1bfd08d 100644 --- a/unsloth/trainer.py +++ b/unsloth/trainer.py @@ -36,7 +36,7 @@ from unsloth_zoo.vision_utils import ( UnslothVisionDataCollator, ) from unsloth_zoo.hf_utils import get_transformers_model_type -from packaging.version import Version +from unsloth_zoo.utils import Version import dataclasses __all__ = [ @@ -315,10 +315,11 @@ def _patch_sft_trainer_auto_packing(trl_module): # We also disable vision language models for padding free collators blocked = ( - data_collator is not None + (data_collator is not None) or isinstance(processing_class, ProcessorMixin) or is_vlm or is_unsupported_model + or (os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1") # Disable padding free on forced logits ) requested_pack = bool(getattr(config_arg, "packing", False)) if blocked: From 0e34b86528db6973930ed48fad6636bb98dd78de Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 12 Dec 2025 05:01:43 -0800 Subject: [PATCH 08/96] Update _utils.py --- unsloth/models/_utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index bdb8f38a50..0377127860 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -413,6 +413,16 @@ try: except: pass +# Flax classes are deprecated and will be removed in Diffusers v1.0.0. +try: + from diffusers.utils import logger as diffusers_logger + + diffusers_logger.addFilter(HideLoggingMessage("are deprecated")) + del diffusers_logger +except: + pass + + # Errors out on # Some weights of Gemma3nForConditionalGeneration were not initialized from the model checkpoint from transformers.modeling_utils import logger as transformers_logger From 6cdbc674a437e160217f939b9e12ff6a9882a816 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 12 Dec 2025 05:10:45 -0800 Subject: [PATCH 09/96] Update import_fixes.py --- unsloth/import_fixes.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 4cbe57cd95..63bfd6e1e1 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -370,3 +370,41 @@ def torchvision_compatibility_check(): logger.info( f"Unsloth: torch=={torch_version} and torchvision=={torchvision_version} are compatible." ) + + +# Fix TRL OpenEnv 0.26 NameError: name 'SamplingParams' is not defined +def fix_openenv_no_vllm(): + if importlib.util.find_spec("trl") is None: + return + trl_location = importlib.util.find_spec("trl").origin + trl_location = os.path.split(trl_location)[0] + openenv = Path(trl_location) / "experimental" / "openenv" / "utils.py" + if not openenv.exists(): + return + try: + with open(openenv, "r+", encoding = "utf-8") as f: + text = f.read() + bad = ( + "if is_vllm_available():\n" + "from vllm import SamplingParams\n" + "from vllm.sampling_params import GuidedDecodingParams\n" + ) + if bad + "\n" + "\n" in text: + text = text.replace( + bad + "\n" + "\n", + bad + ( + "else:\n" + " from typing import Any\n"\ + " SamplingParams = Any\n"\ + " GuidedDecodingParams = Any\n" + "\n" + ) + ) + f.seek(0) + f.write(text) + f.truncate() + logger.info( + "Unsloth: Patching TRL OpenEnv to fix SamplingParams not defined" + ) + except Exception as e: + logger.info(f"Unsloth: Failed patching TRL OpenEnv with error = {str(e)}") From 33fa8b19fe2fe348c7bfe3cfa6ea0f978b450677 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 12 Dec 2025 05:11:12 -0800 Subject: [PATCH 10/96] Update rl_replacements.py --- unsloth/models/rl_replacements.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 7dab0d7307..7d4d520c1f 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -27,6 +27,7 @@ import inspect from collections import defaultdict from unsloth_zoo.rl_replacements import RL_REPLACEMENTS, left_pack_padding from unsloth_zoo.utils import Version +from importlib.metadata import version as importlib_version from unsloth_zoo.log import logger import importlib.util from ..device_type import ( From c8418a87e69490976217a42a4dd1244dd3804cd6 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 12 Dec 2025 05:20:09 -0800 Subject: [PATCH 11/96] fix_openenv_no_vllm --- pyproject.toml | 4 ++-- unsloth/__init__.py | 3 +++ unsloth/models/_utils.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8d91ff621d..c6e19b014e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ huggingfacenotorch = [ ] huggingface = [ "unsloth[huggingfacenotorch]", - "unsloth_zoo>=2025.12.3", + "unsloth_zoo>=2025.12.4", "torchvision", "unsloth[triton]", ] @@ -523,7 +523,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3 ; ('linux' in sys_platform)", ] colab-new = [ - "unsloth_zoo>=2025.12.3", + "unsloth_zoo>=2025.12.4", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,<=4.57.3", diff --git a/unsloth/__init__.py b/unsloth/__init__.py index 5df43d3117..e389074a1b 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -125,6 +125,7 @@ from .import_fixes import ( patch_trackio, patch_datasets, patch_enable_input_require_grads, + fix_openenv_no_vllm, ) fix_xformers_performance_issue() @@ -135,6 +136,7 @@ patch_ipykernel_hf_xet() patch_trackio() patch_datasets() patch_enable_input_require_grads() +fix_openenv_no_vllm() del fix_xformers_performance_issue del fix_vllm_aimv2_issue @@ -144,6 +146,7 @@ del patch_ipykernel_hf_xet del patch_trackio del patch_datasets del patch_enable_input_require_grads +del fix_openenv_no_vllm # Torch 2.4 has including_emulation if DEVICE_TYPE == "cuda": diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 0377127860..653b539b20 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.12.4" +__version__ = "2025.12.5" __all__ = [ "SUPPORTS_BFLOAT16", From 65aaa0b01ccc36b388083629af028a27433a22cc Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 12 Dec 2025 05:27:42 -0800 Subject: [PATCH 12/96] Fix --- unsloth/__init__.py | 4 ++-- unsloth/import_fixes.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/unsloth/__init__.py b/unsloth/__init__.py index e389074a1b..30bfae35bb 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -124,7 +124,7 @@ from .import_fixes import ( patch_ipykernel_hf_xet, patch_trackio, patch_datasets, - patch_enable_input_require_grads, + # patch_enable_input_require_grads, fix_openenv_no_vllm, ) @@ -135,7 +135,7 @@ ignore_logger_messages() patch_ipykernel_hf_xet() patch_trackio() patch_datasets() -patch_enable_input_require_grads() +# patch_enable_input_require_grads() fix_openenv_no_vllm() del fix_xformers_performance_issue diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 63bfd6e1e1..79fba855f5 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -386,8 +386,8 @@ def fix_openenv_no_vllm(): text = f.read() bad = ( "if is_vllm_available():\n" - "from vllm import SamplingParams\n" - "from vllm.sampling_params import GuidedDecodingParams\n" + " from vllm import SamplingParams\n" + " from vllm.sampling_params import GuidedDecodingParams\n" ) if bad + "\n" + "\n" in text: text = text.replace( From 3d6dcb63a86a5d0e195542e0a295e59080c03ece Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 12 Dec 2025 05:29:25 -0800 Subject: [PATCH 13/96] Update __init__.py --- unsloth/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unsloth/__init__.py b/unsloth/__init__.py index 30bfae35bb..e389074a1b 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -124,7 +124,7 @@ from .import_fixes import ( patch_ipykernel_hf_xet, patch_trackio, patch_datasets, - # patch_enable_input_require_grads, + patch_enable_input_require_grads, fix_openenv_no_vllm, ) @@ -135,7 +135,7 @@ ignore_logger_messages() patch_ipykernel_hf_xet() patch_trackio() patch_datasets() -# patch_enable_input_require_grads() +patch_enable_input_require_grads() fix_openenv_no_vllm() del fix_xformers_performance_issue From 4661c8b5327b02f4f1352eb073f4ab3d212b0c2a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 12 Dec 2025 05:31:36 -0800 Subject: [PATCH 14/96] Update __init__.py --- unsloth/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unsloth/__init__.py b/unsloth/__init__.py index e389074a1b..72d53f572e 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -18,8 +18,8 @@ import os, re, subprocess, inspect, functools import numpy as np # Log Unsloth is being used -# We want logger in import_fixes and hence setting it here for zoo to be importable os.environ["UNSLOTH_IS_PRESENT"] = "1" + # Fix some issues before importing other packages from .import_fixes import ( fix_message_factory_issue, @@ -46,7 +46,7 @@ if already_imported: # stacklevel=2 makes warning point to user's import line rather than this library code, # showing them exactly where to fix the import order in their script warnings.warn( - f"WARNING: Unsloth should be imported before {', '.join(already_imported)} " + f"WARNING: Unsloth should be imported before [{', '.join(already_imported)}] " f"to ensure all optimizations are applied. Your code may run slower or encounter " f"memory issues without these optimizations.\n\n" f"Please restructure your imports with 'import unsloth' at the top of your file.", From d2636f6e031483f0bc067f806826c20aa4fca77b Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 12 Dec 2025 05:34:29 -0800 Subject: [PATCH 15/96] Update __init__.py --- unsloth/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unsloth/__init__.py b/unsloth/__init__.py index 72d53f572e..9dd7b08b56 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -16,6 +16,7 @@ import warnings, importlib, sys from packaging.version import Version import os, re, subprocess, inspect, functools import numpy as np +print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) # Log Unsloth is being used os.environ["UNSLOTH_IS_PRESENT"] = "1" @@ -26,6 +27,7 @@ from .import_fixes import ( check_fbgemm_gpu_version, torchvision_compatibility_check, ) +print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) fix_message_factory_issue() check_fbgemm_gpu_version() From 1a59a96d82e9422fb2506c89ccb7ac46610aa43a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 12 Dec 2025 05:36:40 -0800 Subject: [PATCH 16/96] Update import_fixes.py --- unsloth/import_fixes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 79fba855f5..3ba79c2162 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -268,6 +268,7 @@ def check_fbgemm_gpu_version(): ) logger.info(f"Unsloth: fbgemm_gpu_genai=={fbgemm_gpu_version} detected.") +print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) def patch_enable_input_require_grads(): """ @@ -331,6 +332,7 @@ def patch_enable_input_require_grads(): "Unsloth: Patched enable_input_require_grads for vision model compatibility" ) +print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) def torchvision_compatibility_check(): if importlib.util.find_spec("torch") is None: From cd1739761a405985d6e0c65a146955762b0d6479 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 12 Dec 2025 05:38:24 -0800 Subject: [PATCH 17/96] Update import_fixes.py --- unsloth/import_fixes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 3ba79c2162..75dc12f5a1 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import sys import importlib.util from pathlib import Path from importlib.metadata import version as importlib_version From c05af38c22859c11415be8d429c00a6c4112ddcc Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 12 Dec 2025 05:40:56 -0800 Subject: [PATCH 18/96] Update import_fixes.py --- unsloth/import_fixes.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 75dc12f5a1..aa848a91b8 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -20,8 +20,9 @@ from importlib.metadata import version as importlib_version from packaging.version import Version as TrueVersion import re import logging +print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) from unsloth_zoo.log import logger - +print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) def Version(version): try: @@ -41,7 +42,7 @@ def Version(version): f"Unsloth: Could not get version for `{version}`\n" f"File name = [{caller.filename}] Line number = [{caller.lineno}]" ) - +print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) # Ignore logging messages class HideLoggingMessage(logging.Filter): @@ -52,7 +53,7 @@ class HideLoggingMessage(logging.Filter): def filter(self, x): return not (self.text in x.getMessage()) - +print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) # Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype' # MUST do this at the start primarily due to tensorflow causing issues @@ -99,7 +100,7 @@ def fix_message_factory_issue(): pass except: pass - +print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) # Fix Xformers performance issues since 0.0.25 def fix_xformers_performance_issue(): @@ -128,7 +129,7 @@ def fix_xformers_performance_issue(): ) except Exception as e: logger.info(f"Unsloth: Failed patching Xformers with error = {str(e)}") - +print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) # ValueError: 'aimv2' is already used by a Transformers config, pick another name. def fix_vllm_aimv2_issue(): @@ -167,7 +168,7 @@ def fix_vllm_aimv2_issue(): ) except Exception as e: logger.info(f"Unsloth: Failed patching vLLM with error = {str(e)}") - +print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) def fix_vllm_guided_decoding_params(): if importlib.util.find_spec("vllm") is None: @@ -183,7 +184,7 @@ def fix_vllm_guided_decoding_params(): vllm.sampling_params.GuidedDecodingParams = ( vllm.sampling_params.StructuredOutputsParams ) - +print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) def ignore_logger_messages(): # Ignore Environment variable `HF_TOKEN` is set @@ -194,7 +195,7 @@ def ignore_logger_messages(): del huggingface_hub_logger except: pass - +print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) def patch_ipykernel_hf_xet(): # HF-XET == 1.1.10 and ipykernel == 7.0.0 / 7.0.1 causes issues @@ -226,7 +227,7 @@ def patch_ipykernel_hf_xet(): from huggingface_hub.utils import disable_progress_bars disable_progress_bars() - +print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) def patch_trackio(): # Set some environment variables to customize the Trackio dashboard for experiment tracking @@ -238,7 +239,7 @@ def patch_trackio(): "https://raw.githubusercontent.com/unslothai/unsloth/main/images/unsloth%20logo%20white%20text.png" ) os.environ["TRACKIO_PLOT_ORDER"] = "train/reward" - +print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) def patch_datasets(): # Datasets 4.4.0 and 4.4.1 weirdly have some weird `_thread.RLock_recursion_count` issues @@ -253,7 +254,7 @@ def patch_datasets(): f"#### Unsloth: Using `datasets = {str(datasets_version)}` will cause recursion errors.\n" "Please downgrade datasets to `datasets==4.3.0" ) - +print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) def check_fbgemm_gpu_version(): if importlib.util.find_spec("fbgemm_gpu") is None: From 76788e0f8a199c32b27fcff8259596e0483231c7 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 12 Dec 2025 05:44:38 -0800 Subject: [PATCH 19/96] logger --- unsloth/__init__.py | 2 -- unsloth/import_fixes.py | 36 ++++++++++++++++++++---------------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/unsloth/__init__.py b/unsloth/__init__.py index 9dd7b08b56..72d53f572e 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -16,7 +16,6 @@ import warnings, importlib, sys from packaging.version import Version import os, re, subprocess, inspect, functools import numpy as np -print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) # Log Unsloth is being used os.environ["UNSLOTH_IS_PRESENT"] = "1" @@ -27,7 +26,6 @@ from .import_fixes import ( check_fbgemm_gpu_version, torchvision_compatibility_check, ) -print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) fix_message_factory_issue() check_fbgemm_gpu_version() diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index aa848a91b8..d90c9a8a07 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -13,16 +13,15 @@ # limitations under the License. import os -import sys import importlib.util from pathlib import Path from importlib.metadata import version as importlib_version from packaging.version import Version as TrueVersion import re import logging -print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) -from unsloth_zoo.log import logger -print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) +# Cannot import logger here since it'll import transformers +# from unsloth_zoo.log import logger + def Version(version): try: @@ -42,7 +41,7 @@ def Version(version): f"Unsloth: Could not get version for `{version}`\n" f"File name = [{caller.filename}] Line number = [{caller.lineno}]" ) -print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) + # Ignore logging messages class HideLoggingMessage(logging.Filter): @@ -53,7 +52,7 @@ class HideLoggingMessage(logging.Filter): def filter(self, x): return not (self.text in x.getMessage()) -print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) + # Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype' # MUST do this at the start primarily due to tensorflow causing issues @@ -71,6 +70,7 @@ def fix_message_factory_issue(): def GetPrototype(self, *args, **kwargs): return + from unsloth_zoo.log import logger if not hasattr(google.protobuf.message_factory, "MessageFactory"): logger.info("Unsloth: Patching protobuf.MessageFactory as it doesn't exist") google.protobuf.message_factory.MessageFactory = MessageFactory @@ -100,7 +100,7 @@ def fix_message_factory_issue(): pass except: pass -print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) + # Fix Xformers performance issues since 0.0.25 def fix_xformers_performance_issue(): @@ -108,6 +108,7 @@ def fix_xformers_performance_issue(): return xformers_version = importlib_version("xformers") if Version(xformers_version) < Version("0.0.29"): + from unsloth_zoo.log import logger xformers_location = importlib.util.find_spec("xformers").origin xformers_location = os.path.split(xformers_location)[0] cutlass = Path(xformers_location) / "ops" / "fmha" / "cutlass.py" @@ -129,7 +130,7 @@ def fix_xformers_performance_issue(): ) except Exception as e: logger.info(f"Unsloth: Failed patching Xformers with error = {str(e)}") -print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) + # ValueError: 'aimv2' is already used by a Transformers config, pick another name. def fix_vllm_aimv2_issue(): @@ -137,6 +138,7 @@ def fix_vllm_aimv2_issue(): return vllm_version = importlib_version("vllm") if Version(vllm_version) < Version("0.10.1"): + from unsloth_zoo.log import logger vllm_version = importlib.util.find_spec("vllm").origin vllm_version = os.path.split(vllm_version)[0] ovis_config = Path(vllm_version) / "transformers_utils" / "configs" / "ovis.py" @@ -168,7 +170,7 @@ def fix_vllm_aimv2_issue(): ) except Exception as e: logger.info(f"Unsloth: Failed patching vLLM with error = {str(e)}") -print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) + def fix_vllm_guided_decoding_params(): if importlib.util.find_spec("vllm") is None: @@ -184,7 +186,7 @@ def fix_vllm_guided_decoding_params(): vllm.sampling_params.GuidedDecodingParams = ( vllm.sampling_params.StructuredOutputsParams ) -print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) + def ignore_logger_messages(): # Ignore Environment variable `HF_TOKEN` is set @@ -195,7 +197,7 @@ def ignore_logger_messages(): del huggingface_hub_logger except: pass -print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) + def patch_ipykernel_hf_xet(): # HF-XET == 1.1.10 and ipykernel == 7.0.0 / 7.0.1 causes issues @@ -227,7 +229,7 @@ def patch_ipykernel_hf_xet(): from huggingface_hub.utils import disable_progress_bars disable_progress_bars() -print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) + def patch_trackio(): # Set some environment variables to customize the Trackio dashboard for experiment tracking @@ -239,7 +241,7 @@ def patch_trackio(): "https://raw.githubusercontent.com/unslothai/unsloth/main/images/unsloth%20logo%20white%20text.png" ) os.environ["TRACKIO_PLOT_ORDER"] = "train/reward" -print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) + def patch_datasets(): # Datasets 4.4.0 and 4.4.1 weirdly have some weird `_thread.RLock_recursion_count` issues @@ -254,7 +256,7 @@ def patch_datasets(): f"#### Unsloth: Using `datasets = {str(datasets_version)}` will cause recursion errors.\n" "Please downgrade datasets to `datasets==4.3.0" ) -print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) + def check_fbgemm_gpu_version(): if importlib.util.find_spec("fbgemm_gpu") is None: @@ -268,9 +270,9 @@ def check_fbgemm_gpu_version(): raise ImportError( f"Unsloth: fbgemm_gpu_genai=={fbgemm_gpu_version} detected. It might cause unexpected issues like segmentation faults. Please uninstall the current one by doing `pip uninstall fbgemm-gpu` && `pip install fbgemm-gpu` to install fbgemm-gpu 1.4.0 or newer!" ) + from unsloth_zoo.log import logger logger.info(f"Unsloth: fbgemm_gpu_genai=={fbgemm_gpu_version} detected.") -print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) def patch_enable_input_require_grads(): """ @@ -330,11 +332,11 @@ def patch_enable_input_require_grads(): self._require_grads_hook = hooks[0] PreTrainedModel.enable_input_require_grads = _patched_enable_input_require_grads + from unsloth_zoo.log import logger logger.info( "Unsloth: Patched enable_input_require_grads for vision model compatibility" ) -print([mod for mod in ["trl", "transformers", "peft"] if mod in sys.modules]) def torchvision_compatibility_check(): if importlib.util.find_spec("torch") is None: @@ -371,6 +373,7 @@ def torchvision_compatibility_check(): f"but found torchvision=={torchvision_version}. " f"Please refer to https://pytorch.org/get-started/previous-versions/ for more information." ) + from unsloth_zoo.log import logger logger.info( f"Unsloth: torch=={torch_version} and torchvision=={torchvision_version} are compatible." ) @@ -385,6 +388,7 @@ def fix_openenv_no_vllm(): openenv = Path(trl_location) / "experimental" / "openenv" / "utils.py" if not openenv.exists(): return + from unsloth_zoo.log import logger try: with open(openenv, "r+", encoding = "utf-8") as f: text = f.read() From 6450e3c8bc0954d7b2f2a5ba1f95b5360c0e3074 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 12 Dec 2025 05:46:50 -0800 Subject: [PATCH 20/96] Update __init__.py --- unsloth/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/unsloth/__init__.py b/unsloth/__init__.py index 72d53f572e..a0ca276cde 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -20,6 +20,10 @@ import numpy as np # Log Unsloth is being used os.environ["UNSLOTH_IS_PRESENT"] = "1" +# Check if modules that need patching are already imported +critical_modules = ["trl", "transformers", "peft"] +already_imported = [mod for mod in critical_modules if mod in sys.modules] + # Fix some issues before importing other packages from .import_fixes import ( fix_message_factory_issue, @@ -34,10 +38,6 @@ del fix_message_factory_issue del check_fbgemm_gpu_version del torchvision_compatibility_check -# Check if modules that need patching are already imported -critical_modules = ["trl", "transformers", "peft"] -already_imported = [mod for mod in critical_modules if mod in sys.modules] - # This check is critical because Unsloth optimizes these libraries by modifying # their code at import time. If they're imported first, the original (slower, # more memory-intensive) implementations will be used instead of Unsloth's From 31cc537543520cc4206cf9f4fc2b772b056c7600 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 Dec 2025 13:48:24 +0000 Subject: [PATCH 21/96] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/import_fixes.py | 16 ++++++++++++---- unsloth/kernels/rope_embedding.py | 1 + unsloth/trainer.py | 4 +++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index d90c9a8a07..a78b5451ea 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -71,6 +71,7 @@ def fix_message_factory_issue(): return from unsloth_zoo.log import logger + if not hasattr(google.protobuf.message_factory, "MessageFactory"): logger.info("Unsloth: Patching protobuf.MessageFactory as it doesn't exist") google.protobuf.message_factory.MessageFactory = MessageFactory @@ -109,6 +110,7 @@ def fix_xformers_performance_issue(): xformers_version = importlib_version("xformers") if Version(xformers_version) < Version("0.0.29"): from unsloth_zoo.log import logger + xformers_location = importlib.util.find_spec("xformers").origin xformers_location = os.path.split(xformers_location)[0] cutlass = Path(xformers_location) / "ops" / "fmha" / "cutlass.py" @@ -139,6 +141,7 @@ def fix_vllm_aimv2_issue(): vllm_version = importlib_version("vllm") if Version(vllm_version) < Version("0.10.1"): from unsloth_zoo.log import logger + vllm_version = importlib.util.find_spec("vllm").origin vllm_version = os.path.split(vllm_version)[0] ovis_config = Path(vllm_version) / "transformers_utils" / "configs" / "ovis.py" @@ -271,6 +274,7 @@ def check_fbgemm_gpu_version(): f"Unsloth: fbgemm_gpu_genai=={fbgemm_gpu_version} detected. It might cause unexpected issues like segmentation faults. Please uninstall the current one by doing `pip uninstall fbgemm-gpu` && `pip install fbgemm-gpu` to install fbgemm-gpu 1.4.0 or newer!" ) from unsloth_zoo.log import logger + logger.info(f"Unsloth: fbgemm_gpu_genai=={fbgemm_gpu_version} detected.") @@ -333,6 +337,7 @@ def patch_enable_input_require_grads(): PreTrainedModel.enable_input_require_grads = _patched_enable_input_require_grads from unsloth_zoo.log import logger + logger.info( "Unsloth: Patched enable_input_require_grads for vision model compatibility" ) @@ -374,6 +379,7 @@ def torchvision_compatibility_check(): f"Please refer to https://pytorch.org/get-started/previous-versions/ for more information." ) from unsloth_zoo.log import logger + logger.info( f"Unsloth: torch=={torch_version} and torchvision=={torchvision_version} are compatible." ) @@ -389,6 +395,7 @@ def fix_openenv_no_vllm(): if not openenv.exists(): return from unsloth_zoo.log import logger + try: with open(openenv, "r+", encoding = "utf-8") as f: text = f.read() @@ -400,13 +407,14 @@ def fix_openenv_no_vllm(): if bad + "\n" + "\n" in text: text = text.replace( bad + "\n" + "\n", - bad + ( + bad + + ( "else:\n" - " from typing import Any\n"\ - " SamplingParams = Any\n"\ + " from typing import Any\n" + " SamplingParams = Any\n" " GuidedDecodingParams = Any\n" "\n" - ) + ), ) f.seek(0) f.write(text) diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.py index 2adc9ecc5a..a032e0f7fc 100644 --- a/unsloth/kernels/rope_embedding.py +++ b/unsloth/kernels/rope_embedding.py @@ -108,6 +108,7 @@ _rope_embedding_QK = triton.heuristics( ROPE_GROUP_SIZE: int = 4 + def _rope_embedding( Q, Q_row_stride: tl.constexpr, diff --git a/unsloth/trainer.py b/unsloth/trainer.py index 5cd1bfd08d..c0b2dd03b6 100644 --- a/unsloth/trainer.py +++ b/unsloth/trainer.py @@ -319,7 +319,9 @@ def _patch_sft_trainer_auto_packing(trl_module): or isinstance(processing_class, ProcessorMixin) or is_vlm or is_unsupported_model - or (os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1") # Disable padding free on forced logits + or ( + os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1" + ) # Disable padding free on forced logits ) requested_pack = bool(getattr(config_arg, "packing", False)) if blocked: From ad19840a78b4fe31ec3bcce80a10b8daa93070fb Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 12 Dec 2025 05:51:31 -0800 Subject: [PATCH 22/96] Update __init__.py --- unsloth/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/__init__.py b/unsloth/__init__.py index a0ca276cde..007b952200 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -73,7 +73,7 @@ from importlib.metadata import PackageNotFoundError # Check for unsloth_zoo try: unsloth_zoo_version = importlib_version("unsloth_zoo") - if Version(unsloth_zoo_version) < Version("2025.12.3"): + if Version(unsloth_zoo_version) < Version("2025.12.4"): print( "Unsloth: Please update Unsloth and Unsloth-Zoo to the latest version!\n" "Do this via `pip install --upgrade --force-reinstall --no-cache-dir --no-deps unsloth unsloth_zoo`" From 7403104b0c05c0794bd8f74342624a22c930a535 Mon Sep 17 00:00:00 2001 From: oKatanaaa Date: Sat, 13 Dec 2025 00:02:48 +0000 Subject: [PATCH 23/96] fix: add a log instead of silent exception --- unsloth/models/llama.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index e0d8cbcf25..6e47907166 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -3022,8 +3022,11 @@ class FastLlamaModel: if hasattr(target_module, "weight"): try: delattr(target_module, "weight") - except Exception: - pass + except Exception as exc: + logger.warning_once( + f"Unsloth: Could not delete existing weight attr during retie on " + f"{type(target_module).__name__}: {exc}" + ) target_module.register_parameter("weight", weight) # Tie trainable copies created by ModulesToSaveWrapper first (these are used in forward) From 2a2716f66d882b1820f9344066461a8fd318a9ae Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 16 Dec 2025 19:48:41 -0800 Subject: [PATCH 24/96] Update import_fixes.py --- unsloth/import_fixes.py | 67 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 3da82c8eb6..458221cf18 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -422,3 +422,70 @@ def fix_openenv_no_vllm(): ) except Exception as e: logger.info(f"Unsloth: Failed patching TRL OpenEnv with error = {str(e)}") + + +# Fix Exeuctorch needing get_mapped_key +def fix_executorch(): + if importlib.util.find_spec("executorch") is None: + print(1) + executorch_location = importlib.util.find_spec("executorch").origin + if executorch_location is None: + executorch_location = importlib.util.find_spec("executorch").submodule_search_locations[0] + else: + executorch_location = os.path.split(executorch_location)[0] + executorch = Path(executorch_location) / "examples" / "models" / "__init__.py" + if not executorch.exists(): + return + + try: + what = r''' + import sys + import types + import re + from typing import Any, Optional + def get_mapped_key(key: str, mapping_dict: dict[str, str]) -> str: + try: + # Checks if there is a layer # in the key + if any(k.isdigit() for k in key.split(".")): + # Replace layer number with "{}" to create key for lookup + abstract_key = re.sub(r"(\.\d+)", ".{}", key) + layer_num = re.search(r"\d+", key).group(0) + new_key = mapping_dict[abstract_key] + new_key = new_key.format(layer_num) + else: + new_key = mapping_dict[key] + except KeyError as e: + raise Exception( + f'Error converting the state dict. Found unexpected key: "{key}". ' + "Please make sure you're loading a checkpoint with the right format. " + ) from e + + return new_key + + torchtune = types.ModuleType("torchtune") + torchtune.__path__ = [] + models = types.ModuleType("torchtune.models") + models.__path__ = [] + convert_weights = types.ModuleType("torchtune.models.convert_weights") + convert_weights.get_mapped_key = get_mapped_key + torchtune.models = models + models.convert_weights = convert_weights + sys.modules["torchtune"] = torchtune + sys.modules["torchtune.models"] = models + sys.modules["torchtune.models.convert_weights"] = convert_weights + ''' + what = textwrap.dedent(what) + + with open(executorch, "r+", encoding = "utf-8") as f: + text = f.read() + bad = "from enum import Enum\n" + if bad in text: + text = text.replace(bad + "\n", bad + "\n" + what) + f.seek(0) + f.write(text) + f.truncate() + logger.info( + "Unsloth: Patching Executorch to fix get_mapped_key" + ) + except Exception as e: + logger.info(f"Unsloth: Failed Executorch with error = {str(e)}") From 9cda89cedebb3857226b800f8a20adbf5fd7f127 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 16 Dec 2025 19:49:19 -0800 Subject: [PATCH 25/96] Update __init__.py --- unsloth/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/unsloth/__init__.py b/unsloth/__init__.py index 007b952200..bf3de82dc0 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -126,6 +126,7 @@ from .import_fixes import ( patch_datasets, patch_enable_input_require_grads, fix_openenv_no_vllm, + fix_executorch, ) fix_xformers_performance_issue() @@ -137,6 +138,7 @@ patch_trackio() patch_datasets() patch_enable_input_require_grads() fix_openenv_no_vllm() +fix_executorch() del fix_xformers_performance_issue del fix_vllm_aimv2_issue @@ -147,6 +149,7 @@ del patch_trackio del patch_datasets del patch_enable_input_require_grads del fix_openenv_no_vllm +del fix_executorch # Torch 2.4 has including_emulation if DEVICE_TYPE == "cuda": From 7e2a61aab4e84eb225579db6bab8426c50de2a68 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 16 Dec 2025 19:51:10 -0800 Subject: [PATCH 26/96] Update import_fixes.py --- unsloth/import_fixes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 458221cf18..1577a8384e 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -427,7 +427,7 @@ def fix_openenv_no_vllm(): # Fix Exeuctorch needing get_mapped_key def fix_executorch(): if importlib.util.find_spec("executorch") is None: - print(1) + return executorch_location = importlib.util.find_spec("executorch").origin if executorch_location is None: executorch_location = importlib.util.find_spec("executorch").submodule_search_locations[0] From 15303b7f359c28e21fe47a3164eb75f30ac9cbcc Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 16 Dec 2025 19:59:04 -0800 Subject: [PATCH 27/96] Update import_fixes.py --- unsloth/import_fixes.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 1577a8384e..09c8637992 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -115,8 +115,11 @@ def fix_xformers_performance_issue(): return xformers_version = importlib_version("xformers") if Version(xformers_version) < Version("0.0.29"): - xformers_location = importlib.util.find_spec("xformers").origin - xformers_location = os.path.split(xformers_location)[0] + xformers_location = importlib.util.find_spec("xformers") + if xformers_location is None: + xformers_location = importlib.util.find_spec("xformers").submodule_search_locations[0] + else: + xformers_location = os.path.split(xformers_location.origin)[0] cutlass = Path(xformers_location) / "ops" / "fmha" / "cutlass.py" try: if cutlass.exists(): @@ -144,8 +147,11 @@ def fix_vllm_aimv2_issue(): return vllm_version = importlib_version("vllm") if Version(vllm_version) < Version("0.10.1"): - vllm_version = importlib.util.find_spec("vllm").origin - vllm_version = os.path.split(vllm_version)[0] + vllm_version = importlib.util.find_spec("vllm") + if vllm_version is None: + vllm_version = importlib.util.find_spec("vllm").submodule_search_locations[0] + else: + vllm_version = os.path.split(vllm_version.origin)[0] ovis_config = Path(vllm_version) / "transformers_utils" / "configs" / "ovis.py" try: if ovis_config.exists(): @@ -388,8 +394,11 @@ def torchvision_compatibility_check(): def fix_openenv_no_vllm(): if importlib.util.find_spec("trl") is None: return - trl_location = importlib.util.find_spec("trl").origin - trl_location = os.path.split(trl_location)[0] + trl_location = importlib.util.find_spec("trl") + if trl_location is None: + trl_location = importlib.util.find_spec("trl").submodule_search_locations[0] + else: + trl_location = os.path.split(trl_location.origin)[0] openenv = Path(trl_location) / "experimental" / "openenv" / "utils.py" if not openenv.exists(): return @@ -428,11 +437,11 @@ def fix_openenv_no_vllm(): def fix_executorch(): if importlib.util.find_spec("executorch") is None: return - executorch_location = importlib.util.find_spec("executorch").origin + executorch_location = importlib.util.find_spec("executorch") if executorch_location is None: executorch_location = importlib.util.find_spec("executorch").submodule_search_locations[0] else: - executorch_location = os.path.split(executorch_location)[0] + executorch_location = os.path.split(executorch_location.origin)[0] executorch = Path(executorch_location) / "examples" / "models" / "__init__.py" if not executorch.exists(): return From 50a777240906e511559cd0ea9e4a7cf3cfae4246 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 16 Dec 2025 20:06:12 -0800 Subject: [PATCH 28/96] Update import_fixes.py --- unsloth/import_fixes.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 09c8637992..23bb4789ca 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -115,11 +115,11 @@ def fix_xformers_performance_issue(): return xformers_version = importlib_version("xformers") if Version(xformers_version) < Version("0.0.29"): - xformers_location = importlib.util.find_spec("xformers") + xformers_location = importlib.util.find_spec("xformers").origin if xformers_location is None: xformers_location = importlib.util.find_spec("xformers").submodule_search_locations[0] else: - xformers_location = os.path.split(xformers_location.origin)[0] + xformers_location = os.path.split(xformers_location)[0] cutlass = Path(xformers_location) / "ops" / "fmha" / "cutlass.py" try: if cutlass.exists(): @@ -147,11 +147,11 @@ def fix_vllm_aimv2_issue(): return vllm_version = importlib_version("vllm") if Version(vllm_version) < Version("0.10.1"): - vllm_version = importlib.util.find_spec("vllm") + vllm_version = importlib.util.find_spec("vllm").origin if vllm_version is None: vllm_version = importlib.util.find_spec("vllm").submodule_search_locations[0] else: - vllm_version = os.path.split(vllm_version.origin)[0] + vllm_version = os.path.split(vllm_version)[0] ovis_config = Path(vllm_version) / "transformers_utils" / "configs" / "ovis.py" try: if ovis_config.exists(): @@ -394,11 +394,11 @@ def torchvision_compatibility_check(): def fix_openenv_no_vllm(): if importlib.util.find_spec("trl") is None: return - trl_location = importlib.util.find_spec("trl") + trl_location = importlib.util.find_spec("trl").origin if trl_location is None: trl_location = importlib.util.find_spec("trl").submodule_search_locations[0] else: - trl_location = os.path.split(trl_location.origin)[0] + trl_location = os.path.split(trl_location)[0] openenv = Path(trl_location) / "experimental" / "openenv" / "utils.py" if not openenv.exists(): return @@ -437,11 +437,11 @@ def fix_openenv_no_vllm(): def fix_executorch(): if importlib.util.find_spec("executorch") is None: return - executorch_location = importlib.util.find_spec("executorch") + executorch_location = importlib.util.find_spec("executorch").origin if executorch_location is None: executorch_location = importlib.util.find_spec("executorch").submodule_search_locations[0] else: - executorch_location = os.path.split(executorch_location.origin)[0] + executorch_location = os.path.split(executorch_location)[0] executorch = Path(executorch_location) / "examples" / "models" / "__init__.py" if not executorch.exists(): return From 93b0d64060f95565e52a9fe7b44b7f2e4b25ee70 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 16 Dec 2025 20:15:22 -0800 Subject: [PATCH 29/96] Update import_fixes.py --- unsloth/import_fixes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 23bb4789ca..3b10ec26fb 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -19,6 +19,7 @@ from importlib.metadata import version as importlib_version from packaging.version import Version as TrueVersion import re import logging +import textwrap # We cannot do from unsloth_zoo.log import logger since FBGEMM might cause seg faults. UNSLOTH_ENABLE_LOGGING = os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") in ("1", "True", "true",) From 6474510d9673601af582f0cc756c6b721c57d544 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 17 Dec 2025 04:26:02 +0000 Subject: [PATCH 30/96] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/import_fixes.py | 38 +++++++++++++++++++++++++------------- unsloth/models/rl.py | 2 +- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 3b10ec26fb..a93a6c917f 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -22,14 +22,22 @@ import logging import textwrap # We cannot do from unsloth_zoo.log import logger since FBGEMM might cause seg faults. -UNSLOTH_ENABLE_LOGGING = os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") in ("1", "True", "true",) +UNSLOTH_ENABLE_LOGGING = os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") in ( + "1", + "True", + "true", +) logger = logging.getLogger(__name__) if UNSLOTH_ENABLE_LOGGING: - logging.basicConfig(level = logging.INFO, format = '[%(name)s|%(levelname)s]%(message)s') + logging.basicConfig( + level = logging.INFO, format = "[%(name)s|%(levelname)s]%(message)s" + ) logger.setLevel(logging.INFO) else: - logging.basicConfig(level = logging.WARNING, format = '[%(name)s|%(levelname)s]%(message)s') - logger.setLevel(logging.WARNING) + logging.basicConfig( + level = logging.WARNING, format = "[%(name)s|%(levelname)s]%(message)s" + ) + logger.setLevel(logging.WARNING) def Version(version): @@ -118,7 +126,9 @@ def fix_xformers_performance_issue(): if Version(xformers_version) < Version("0.0.29"): xformers_location = importlib.util.find_spec("xformers").origin if xformers_location is None: - xformers_location = importlib.util.find_spec("xformers").submodule_search_locations[0] + xformers_location = importlib.util.find_spec( + "xformers" + ).submodule_search_locations[0] else: xformers_location = os.path.split(xformers_location)[0] cutlass = Path(xformers_location) / "ops" / "fmha" / "cutlass.py" @@ -150,7 +160,9 @@ def fix_vllm_aimv2_issue(): if Version(vllm_version) < Version("0.10.1"): vllm_version = importlib.util.find_spec("vllm").origin if vllm_version is None: - vllm_version = importlib.util.find_spec("vllm").submodule_search_locations[0] + vllm_version = importlib.util.find_spec("vllm").submodule_search_locations[ + 0 + ] else: vllm_version = os.path.split(vllm_version)[0] ovis_config = Path(vllm_version) / "transformers_utils" / "configs" / "ovis.py" @@ -440,7 +452,9 @@ def fix_executorch(): return executorch_location = importlib.util.find_spec("executorch").origin if executorch_location is None: - executorch_location = importlib.util.find_spec("executorch").submodule_search_locations[0] + executorch_location = importlib.util.find_spec( + "executorch" + ).submodule_search_locations[0] else: executorch_location = os.path.split(executorch_location)[0] executorch = Path(executorch_location) / "examples" / "models" / "__init__.py" @@ -448,7 +462,7 @@ def fix_executorch(): return try: - what = r''' + what = r""" import sys import types import re @@ -483,9 +497,9 @@ def fix_executorch(): sys.modules["torchtune"] = torchtune sys.modules["torchtune.models"] = models sys.modules["torchtune.models.convert_weights"] = convert_weights - ''' + """ what = textwrap.dedent(what) - + with open(executorch, "r+", encoding = "utf-8") as f: text = f.read() bad = "from enum import Enum\n" @@ -494,8 +508,6 @@ def fix_executorch(): f.seek(0) f.write(text) f.truncate() - logger.info( - "Unsloth: Patching Executorch to fix get_mapped_key" - ) + logger.info("Unsloth: Patching Executorch to fix get_mapped_key") except Exception as e: logger.info(f"Unsloth: Failed Executorch with error = {str(e)}") diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 3fd180bb27..31316e45b7 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -741,7 +741,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): "generation_kwargs": {}, "bf16": False, "fp16": False, - "report_to" : "none", + "report_to": "none", "include_tokens_per_second": False, "include_num_input_tokens_seen": False, "auto_find_batch_size": False, # Auto /2 batch size - too many people complained so removing From c3c3c4b33280db247d19e994080a5a4deea48fb3 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 16 Dec 2025 20:34:34 -0800 Subject: [PATCH 31/96] Update import_fixes.py --- unsloth/import_fixes.py | 51 +++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 3b10ec26fb..afbc6f5a96 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -112,13 +112,14 @@ def fix_message_factory_issue(): # Fix Xformers performance issues since 0.0.25 def fix_xformers_performance_issue(): - if importlib.util.find_spec("xformers") is None: + spec = importlib.util.find_spec("xformers") + if spec is None: return xformers_version = importlib_version("xformers") if Version(xformers_version) < Version("0.0.29"): - xformers_location = importlib.util.find_spec("xformers").origin + xformers_location = spec.origin if xformers_location is None: - xformers_location = importlib.util.find_spec("xformers").submodule_search_locations[0] + xformers_location = spec.submodule_search_locations[0] else: xformers_location = os.path.split(xformers_location)[0] cutlass = Path(xformers_location) / "ops" / "fmha" / "cutlass.py" @@ -144,13 +145,14 @@ def fix_xformers_performance_issue(): # ValueError: 'aimv2' is already used by a Transformers config, pick another name. def fix_vllm_aimv2_issue(): - if importlib.util.find_spec("vllm") is None: + spec = importlib.util.find_spec("vllm") + if spec is None: return vllm_version = importlib_version("vllm") if Version(vllm_version) < Version("0.10.1"): - vllm_version = importlib.util.find_spec("vllm").origin + vllm_version = spec.origin if vllm_version is None: - vllm_version = importlib.util.find_spec("vllm").submodule_search_locations[0] + vllm_version = spec.submodule_search_locations[0] else: vllm_version = os.path.split(vllm_version)[0] ovis_config = Path(vllm_version) / "transformers_utils" / "configs" / "ovis.py" @@ -393,11 +395,12 @@ def torchvision_compatibility_check(): # Fix TRL OpenEnv 0.26 NameError: name 'SamplingParams' is not defined def fix_openenv_no_vllm(): - if importlib.util.find_spec("trl") is None: + spec = importlib.util.find_spec("trl") + if spec is None: return - trl_location = importlib.util.find_spec("trl").origin + trl_location = spec.origin if trl_location is None: - trl_location = importlib.util.find_spec("trl").submodule_search_locations[0] + trl_location = spec.submodule_search_locations[0] else: trl_location = os.path.split(trl_location)[0] openenv = Path(trl_location) / "experimental" / "openenv" / "utils.py" @@ -412,18 +415,15 @@ def fix_openenv_no_vllm(): " from vllm import SamplingParams\n" " from vllm.sampling_params import GuidedDecodingParams\n" ) - if bad + "\n" + "\n" in text: - text = text.replace( - bad + "\n" + "\n", - bad - + ( - "else:\n" - " from typing import Any\n" - " SamplingParams = Any\n" - " GuidedDecodingParams = Any\n" - "\n" - ), - ) + replace_with = bad + ( + "else:\n" + " from typing import Any\n" + " SamplingParams = Any\n" + " GuidedDecodingParams = Any\n" + "\n" + ) + if bad + "\n" + "\n" in text and replace_with not in text: + text = text.replace(bad + "\n" + "\n", replace_with) f.seek(0) f.write(text) f.truncate() @@ -436,11 +436,12 @@ def fix_openenv_no_vllm(): # Fix Exeuctorch needing get_mapped_key def fix_executorch(): - if importlib.util.find_spec("executorch") is None: + spec = importlib.util.find_spec("executorch") + if spec is None: return - executorch_location = importlib.util.find_spec("executorch").origin + executorch_location = spec.origin if executorch_location is None: - executorch_location = importlib.util.find_spec("executorch").submodule_search_locations[0] + executorch_location = spec.submodule_search_locations[0] else: executorch_location = os.path.split(executorch_location)[0] executorch = Path(executorch_location) / "examples" / "models" / "__init__.py" @@ -489,7 +490,7 @@ def fix_executorch(): with open(executorch, "r+", encoding = "utf-8") as f: text = f.read() bad = "from enum import Enum\n" - if bad in text: + if bad in text and what not in text: text = text.replace(bad + "\n", bad + "\n" + what) f.seek(0) f.write(text) From a2bbd9368f3e0d92aabdbc80f837b8ac1b99ad14 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 16 Dec 2025 20:52:09 -0800 Subject: [PATCH 32/96] Update unsloth/import_fixes.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- unsloth/import_fixes.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index cffbb8ef3a..2c4dbcffb0 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -158,12 +158,12 @@ def fix_vllm_aimv2_issue(): return vllm_version = importlib_version("vllm") if Version(vllm_version) < Version("0.10.1"): - vllm_version = spec.origin - if vllm_version is None: - vllm_version = spec.submodule_search_locations[0] + vllm_location = spec.origin + if vllm_location is None: + vllm_location = spec.submodule_search_locations[0] else: - vllm_version = os.path.split(vllm_version)[0] - ovis_config = Path(vllm_version) / "transformers_utils" / "configs" / "ovis.py" + vllm_location = os.path.split(vllm_location)[0] + ovis_config = Path(vllm_location) / "transformers_utils" / "configs" / "ovis.py" try: if ovis_config.exists(): with open(ovis_config, "r+", encoding = "utf-8") as f: From f739af754dea08b5650938c9d3e49c9ef6e52843 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 16 Dec 2025 23:15:31 -0800 Subject: [PATCH 33/96] Update save.py --- unsloth/save.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/unsloth/save.py b/unsloth/save.py index 01887321cf..640a7ffe14 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -3037,7 +3037,9 @@ def patch_saving_functions(model, vision = False): model.save_pretrained_merged = types.MethodType( unsloth_generic_save_pretrained_merged, model ) - model.push_to_hub_gguf = types.MethodType(unsloth_push_to_hub_gguf, model) + model.push_to_hub_gguf = types.MethodType( + unsloth_push_to_hub_gguf, model + ) model.save_pretrained_gguf = types.MethodType( unsloth_save_pretrained_gguf, model ) @@ -3058,7 +3060,9 @@ def patch_saving_functions(model, vision = False): model.save_pretrained_merged = types.MethodType( unsloth_generic_save_pretrained_merged, model ) - model.push_to_hub_gguf = types.MethodType(unsloth_push_to_hub_gguf, model) + model.push_to_hub_gguf = types.MethodType( + unsloth_push_to_hub_gguf, model + ) model.save_pretrained_gguf = types.MethodType( unsloth_save_pretrained_gguf, model ) From c0b128617a7c5d5b3b3c3d644d637775b2ee8066 Mon Sep 17 00:00:00 2001 From: Datta Nimmaturi Date: Wed, 17 Dec 2025 14:37:21 +0530 Subject: [PATCH 34/96] [fbgemm] Silence tma fbgemm (#3735) * Silence fbgemm TMA print Also safer .push_to_hub * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- unsloth/import_fixes.py | 30 ++++++++++++++++++++++++++++++ unsloth/save.py | 6 +++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 2c4dbcffb0..308bd92db7 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -71,6 +71,36 @@ class HideLoggingMessage(logging.Filter): return not (self.text in x.getMessage()) +class HidePrintMessage: + __slots__ = ("_original_stream", "_hidden_texts") + + def __init__(self, original_stream): + self._original_stream = original_stream + self._hidden_texts = [] + + def add_filter(self, text): + self._hidden_texts.append(text) + + def write(self, message): + if not any(text in message for text in self._hidden_texts): + self._original_stream.write(message) + + def flush(self): + self._original_stream.flush() + + def __getattr__(self, name): + return getattr(self._original_stream, name) + + +if os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") != "1": + import sys + + # Apply to stderr for FBGEMM + sys.stderr = HidePrintMessage(sys.stderr) + # https://github.com/pytorch/FBGEMM/blob/d99cd96490ec4aabac2ee95b1e76ea4dcfcfa628/fbgemm_gpu/experimental/gemm/triton_gemm/utils.py#L43-L52 + sys.stderr.add_filter("TMA benchmarks will be running") + + # Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype' # MUST do this at the start primarily due to tensorflow causing issues def fix_message_factory_issue(): diff --git a/unsloth/save.py b/unsloth/save.py index 01887321cf..d3b20f117c 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -3010,7 +3010,11 @@ def patch_saving_functions(model, vision = False): original_model = model while True: - if original_model.push_to_hub.__name__ != "unsloth_push_to_hub": + # Check if push_to_hub exists before accessing its __name__ + if ( + hasattr(original_model, "push_to_hub") + and original_model.push_to_hub.__name__ != "unsloth_push_to_hub" + ): original_model.original_push_to_hub = original_model.push_to_hub original_model.push_to_hub = types.MethodType( unsloth_push_to_hub, original_model From ea65a3f19c16e9c30eba121e3e91327dc1b8e975 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 17 Dec 2025 01:51:29 -0800 Subject: [PATCH 35/96] Update loader.py --- unsloth/models/loader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index e1c13315f7..bfa94d86d7 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -739,6 +739,8 @@ class FastModel(FastBaseModel): "compatible with `full_finetuning=True`. If you wish to use QAT with LoRA, " "please pass in `qat_scheme` in `FastLanguageModel.get_peft_model(...)` instead." ) + if qat_scheme == "phone-deployment": + qat_scheme = "int8-int4" # Check if 4bit is allowed specifically for AMD if not ALLOW_BITSANDBYTES and not use_exact_model_name: if load_in_4bit or load_in_8bit or model_name.lower().endswith("-bnb-4bit"): From 88e1930a1893408f08a0aa610be6b3400e5bc459 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 17 Dec 2025 02:21:47 -0800 Subject: [PATCH 36/96] Update save.py --- unsloth/save.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/unsloth/save.py b/unsloth/save.py index f5ea8d7d8f..5fa2df7b18 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -2745,6 +2745,17 @@ def _unsloth_save_torchao_with_attached_config( """Save a QAT-trained model by converting fake-quantized weights to real quantized weights.""" # Convert QAT fake-quantized weights to real quantized weights _convert_torchao_model(model) + # PEFT models also might come here, so parse it + if isinstance(model, PeftModelForCausalLM): + _unsloth_save_torchao_with_given_config( + model = model, + save_directory = save_directory, + tokenizer = tokenizer, + torchao_config = model.config.quantization_config, + push_to_hub = push_to_hub, + token = token, + ) + return # TorchAO does not support safe_serialization reliably safe_serialization = False @@ -2897,7 +2908,7 @@ def unsloth_save_pretrained_torchao( ) if torchao_config is not None: - # PTQ path: user provided a config, model must NOT have QAT config + # PTQ path: user provided a config, model must NOT have QAT config unless PEFT assert not has_qat_config, ( "Unsloth: You passed `torchao_config` but this model was trained with `qat_scheme`. " "For QAT models, do not pass `torchao_config` - the quantization config is already " From 4063b87c012caa1de85c1daa55b0113a2fe9027c Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 17 Dec 2025 02:28:03 -0800 Subject: [PATCH 37/96] Update save.py --- unsloth/save.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/unsloth/save.py b/unsloth/save.py index 5fa2df7b18..c5099a5891 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -2817,7 +2817,10 @@ def _unsloth_save_torchao_with_given_config( ) from torchao import quantize_ - quantization_config = TorchAoConfig(quant_type = torchao_config) + if isinstance(torchao_config, TorchAoConfig): + quantization_config = torchao_config + else: + quantization_config = TorchAoConfig(quant_type = torchao_config) # Determine if this is a VLM is_vlm = False From 87b459f1fd01b574ac5793d9f9da96b789f22000 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 17 Dec 2025 02:38:05 -0800 Subject: [PATCH 38/96] Update _utils.py --- unsloth/models/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 653b539b20..5b2cc681b0 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.12.5" +__version__ = "2025.12.6" __all__ = [ "SUPPORTS_BFLOAT16", From 1da74ab342435fe18d5a6d77d84f35414ed96bee Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 17 Dec 2025 02:54:15 -0800 Subject: [PATCH 39/96] Update _utils.py --- unsloth/models/_utils.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 5b2cc681b0..6f6d693b4f 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -413,16 +413,6 @@ try: except: pass -# Flax classes are deprecated and will be removed in Diffusers v1.0.0. -try: - from diffusers.utils import logger as diffusers_logger - - diffusers_logger.addFilter(HideLoggingMessage("are deprecated")) - del diffusers_logger -except: - pass - - # Errors out on # Some weights of Gemma3nForConditionalGeneration were not initialized from the model checkpoint from transformers.modeling_utils import logger as transformers_logger From f34d08ec6ab1ce2b6678bf683bc2e90412697a5e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 17 Dec 2025 03:25:40 -0800 Subject: [PATCH 40/96] Diffusers warnings --- unsloth/__init__.py | 3 +++ unsloth/import_fixes.py | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/unsloth/__init__.py b/unsloth/__init__.py index bf3de82dc0..d10a0f8030 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -29,14 +29,17 @@ from .import_fixes import ( fix_message_factory_issue, check_fbgemm_gpu_version, torchvision_compatibility_check, + fix_diffusers_warnings, ) fix_message_factory_issue() check_fbgemm_gpu_version() torchvision_compatibility_check() +fix_diffusers_warnings() del fix_message_factory_issue del check_fbgemm_gpu_version del torchvision_compatibility_check +del fix_diffusers_warnings # This check is critical because Unsloth optimizes these libraries by modifying # their code at import time. If they're imported first, the original (slower, diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 308bd92db7..f3aae7f523 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -536,3 +536,8 @@ def fix_executorch(): logger.info("Unsloth: Patching Executorch to fix get_mapped_key") except Exception as e: logger.info(f"Unsloth: Failed Executorch with error = {str(e)}") + + +def fix_diffusers_warnings(): + # Silence Flax classes are deprecated and will be removed in Diffusers v1.0.0. + os.environ["DIFFUSERS_VERBOSITY"] = "error" From 35a619aaf551edca7fe67df8a372c614817ee753 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 17 Dec 2025 03:26:19 -0800 Subject: [PATCH 41/96] Update pyproject.toml --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c6e19b014e..cb3f8f3fa4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ huggingfacenotorch = [ ] huggingface = [ "unsloth[huggingfacenotorch]", - "unsloth_zoo>=2025.12.4", + "unsloth_zoo>=2025.12.5", "torchvision", "unsloth[triton]", ] @@ -523,7 +523,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3 ; ('linux' in sys_platform)", ] colab-new = [ - "unsloth_zoo>=2025.12.4", + "unsloth_zoo>=2025.12.5", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,<=4.57.3", From 2d00f37f69b3e22223fda4742d6bb7ca4a081998 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 17 Dec 2025 11:29:39 +0000 Subject: [PATCH 42/96] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/save.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/unsloth/save.py b/unsloth/save.py index c5099a5891..24303aba52 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -3055,9 +3055,7 @@ def patch_saving_functions(model, vision = False): model.save_pretrained_merged = types.MethodType( unsloth_generic_save_pretrained_merged, model ) - model.push_to_hub_gguf = types.MethodType( - unsloth_push_to_hub_gguf, model - ) + model.push_to_hub_gguf = types.MethodType(unsloth_push_to_hub_gguf, model) model.save_pretrained_gguf = types.MethodType( unsloth_save_pretrained_gguf, model ) @@ -3078,9 +3076,7 @@ def patch_saving_functions(model, vision = False): model.save_pretrained_merged = types.MethodType( unsloth_generic_save_pretrained_merged, model ) - model.push_to_hub_gguf = types.MethodType( - unsloth_push_to_hub_gguf, model - ) + model.push_to_hub_gguf = types.MethodType(unsloth_push_to_hub_gguf, model) model.save_pretrained_gguf = types.MethodType( unsloth_save_pretrained_gguf, model ) From 3f2f589de993842ba75c8c3d588409e3dddb40d3 Mon Sep 17 00:00:00 2001 From: Datta Nimmaturi Date: Thu, 18 Dec 2025 17:37:12 +0530 Subject: [PATCH 43/96] [hf_hub] Token login (#3739) * login on token * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * cleanup old code * safer imports * cleanup * Return token after login * correct return types * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Apply suggestion from @danielhanchen * add back imports * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * finish return token --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Daniel Han --- unsloth/models/_utils.py | 21 +++++++++++++++++++++ unsloth/models/llama.py | 3 +-- unsloth/models/loader.py | 23 +++-------------------- unsloth/models/vision.py | 3 +-- 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 6f6d693b4f..0d0f90cf40 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -72,6 +72,7 @@ __all__ = [ "patch_hf_quantizer", "verify_fp8_support_if_applicable", "_get_inference_mode_context_manager", + "hf_login", ] import torch @@ -2344,3 +2345,23 @@ def _get_inference_mode_context_manager(model: torch.nn.Module): return torch.no_grad() else: return torch.inference_mode() + + +def hf_login(token: Optional[str] = None) -> Optional[str]: + if token is None: + try: + from huggingface_hub import get_token + + token = get_token() + if token is None: + return None + except: + return None + try: + from huggingface_hub import login + + login(token = token) + return token + except Exception as e: + logger.info(f"Failed to login to huggingface using token with error: {e}") + return token diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 4c9337ccf9..1d7695b9aa 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -2130,8 +2130,7 @@ class FastLlamaModel: "Unsloth: `unsloth_vllm_standby` is True, but environment variable `UNSLOTH_VLLM_STANDBY` is not set to 1!" ) - if token is None: - token = get_token() + token = hf_login(token) if model_patcher is None: model_patcher = FastLlamaModel SUPPORTS_BFLOAT16 = is_bfloat16_supported() diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index bfa94d86d7..b13775076c 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -20,6 +20,7 @@ from ._utils import ( HAS_FLASH_ATTENTION_SOFTCAPPING, USE_MODELSCOPE, get_transformers_model_type, + hf_login, ) from .granite import FastGraniteModel from .llama import FastLlamaModel, logger @@ -151,15 +152,7 @@ class FastLanguageModel(FastLlamaModel): **kwargs, ): # Login to allow private models - if token is None: - token = get_token() - if token is not None: - try: - from huggingface_hub import login - - login(token = token) - except: - pass + token = hf_login(token) if load_in_8bit or full_finetuning or qat_scheme is not None: return FastModel.from_pretrained( model_name = model_name, @@ -195,8 +188,6 @@ class FastLanguageModel(FastLlamaModel): **kwargs, ) - if token is None: - token = get_token() if isinstance(dtype, str) and dtype in ["float16", "bfloat16"]: dtype = getattr(torch, dtype) assert ( @@ -682,16 +673,8 @@ class FastModel(FastBaseModel): *args, **kwargs, ): - if token is None: - token = get_token() # Login to allow private models - if token is not None: - try: - from huggingface_hub import login - - login(token = token) - except: - pass + token = hf_login(token) if whisper_language is not None: assert type(whisper_language) is str if whisper_task is not None: diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index ed19f587cf..a10d65f3fb 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -390,8 +390,7 @@ class FastBaseModel: "Unsloth: WARNING `trust_remote_code` is True.\n" "Are you certain you want to do remote code execution?" ) - if token is None: - token = get_token() + token = hf_login(token) SUPPORTS_BFLOAT16 = is_bfloat16_supported() if DEVICE_TYPE == "cuda": From 532707a580ae779d45cd552f41d23aa70f2286bd Mon Sep 17 00:00:00 2001 From: Datta Nimmaturi Date: Sat, 20 Dec 2025 08:38:28 +0530 Subject: [PATCH 44/96] Do not overwrite slots (#3752) * Do not overwrite slots * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- unsloth/import_fixes.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index f3aae7f523..efc7a7f4cd 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -72,8 +72,6 @@ class HideLoggingMessage(logging.Filter): class HidePrintMessage: - __slots__ = ("_original_stream", "_hidden_texts") - def __init__(self, original_stream): self._original_stream = original_stream self._hidden_texts = [] From aecd8f0b7efa15160a57b74b2a7ccdd2a5a847a9 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 23 Dec 2025 04:46:43 -0800 Subject: [PATCH 45/96] Update save.py --- unsloth/save.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/unsloth/save.py b/unsloth/save.py index f9c677f5f7..c4ee322937 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -1429,7 +1429,7 @@ language: - **License:** apache-2.0 - **Finetuned from model :** {base_model} -This {model_type} model was trained 2x faster with [Unsloth](https://github.com/unslothai/unsloth) and Huggingface's TRL library. +This {model_type} model was trained 2x faster with [Unsloth](https://github.com/unslothai/unsloth) [](https://github.com/unslothai/unsloth) """ @@ -2234,13 +2234,13 @@ tags: {"- vision-language-model" if is_vlm else ""} --- -# {repo_id.split("/")[-1]} - GGUF +# {repo_id.split("/")[-1]} : GGUF This model was finetuned and converted to GGUF format using [Unsloth](https://github.com/unslothai/unsloth). **Example usage**: -- For text only LLMs: **llama-cli** **--hf** repo_id/model_name **-p** "why is the sky blue?" -- For multimodal models: **llama-mtmd-cli** **-m** model_name.gguf **--mmproj** mmproj_file.gguf +- For text only LLMs: `./llama.cpp/llama-cli -hf {repo_id} --jinja` +- For multimodal models: `./llama.cpp/llama-mtmd-cli -hf {repo_id} --jinja` ## Available Model files: """ @@ -2281,6 +2281,11 @@ This model was finetuned and converted to GGUF format using [Unsloth](https://gi "The model's BOS token behavior was adjusted for GGUF compatibility.\n" ) + readme_content += ( + 'This was trained 2x faster with [Unsloth](https://github.com/unslothai/unsloth)\n' + '[](https://github.com/unslothai/unsloth)\n' + ) + readme_path = os.path.join(actual_save_directory, "README.md") with open(readme_path, "w") as f: f.write(readme_content) From 5a39829e4d97b4838fd1b80ac7074fc728a41ebf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 23 Dec 2025 12:51:08 +0000 Subject: [PATCH 46/96] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/save.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/save.py b/unsloth/save.py index c4ee322937..3a275cf0c3 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -2282,7 +2282,7 @@ This model was finetuned and converted to GGUF format using [Unsloth](https://gi ) readme_content += ( - 'This was trained 2x faster with [Unsloth](https://github.com/unslothai/unsloth)\n' + "This was trained 2x faster with [Unsloth](https://github.com/unslothai/unsloth)\n" '[](https://github.com/unslothai/unsloth)\n' ) From 08f1716a70ae932f8299421d020fa44fa2de6f2e Mon Sep 17 00:00:00 2001 From: Strahinja Stamenkovic Date: Fri, 26 Dec 2025 03:43:59 +0100 Subject: [PATCH 47/96] Add missing import of inspect (#3778) * Add missing import of inspect * Update device_type.py --- unsloth/device_type.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unsloth/device_type.py b/unsloth/device_type.py index 68038de679..0f924bfdfd 100644 --- a/unsloth/device_type.py +++ b/unsloth/device_type.py @@ -24,6 +24,7 @@ __all__ = [ import torch import functools +import inspect from unsloth_zoo.utils import Version From 181b76420efde3a0a0e0a4e5f6dd598523027a2e Mon Sep 17 00:00:00 2001 From: Fizza Mukhtar Date: Thu, 25 Dec 2025 18:46:13 -0800 Subject: [PATCH 48/96] Clarify NotImplementedError for fast_inference with full_finetuning (#3768) * Improve error message for fast_inference and full_finetuning * Refine error message string formatting * Update unsloth/models/vision.py --------- Co-authored-by: Daniel Han --- unsloth/models/vision.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index b78b190bcb..e1cf8f6f82 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -718,9 +718,13 @@ class FastBaseModel: if full_finetuning: max_lora_rank = max(get_lora_supported_ranks()) raise NotImplementedError( - f"Unsloth: `fast_inference = True` does not yet support `full_finetuning = True`.\n" - f"Use LoRA rank `r = {max_lora_rank}` as the closest replacement for full finetuning with Unsloth for RL." + "Unsloth: `fast_inference=True` cannot be used together with `full_finetuning=True`.\n" + "Reason: fast_inference is optimized for inference-only workflows and " + "does not currently support full fine-tuning.\n" + "Workaround: disable fast_inference, or use parameter-efficient fine-tuning " + f"(e.g. LoRA with rank r={max_lora_rank})." ) + model_config.model_name = model_name if fast_inference: From b314dca22dc5bedc80235d2c712d2cbfed2add89 Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Sat, 27 Dec 2025 00:49:19 -0800 Subject: [PATCH 49/96] Update README for new unsloth.ai/docs.md --- README.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 43c09381fc..7cd9d0bba4 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ - + ### Train gpt-oss, DeepSeek, Gemma, Qwen & Llama 2x faster with 70% less VRAM! @@ -18,7 +18,7 @@ ## ✨ Train for Free -Notebooks are beginner friendly. Read our [guide](https://docs.unsloth.ai/get-started/fine-tuning-guide). Add dataset, run, then export your trained model to GGUF, llama.cpp, Ollama, vLLM, SGLang or Hugging Face. +Notebooks are beginner friendly. Read our [guide](https://docs.unsloth.ai/get-started/fine-tuning-guide). Add dataset, run, then deploy your trained model. | Model | Free Notebooks | Performance | Memory use | |-----------|---------|--------|----------| @@ -34,9 +34,9 @@ Notebooks are beginner friendly. Read our [guide](https://docs.unsloth.ai/get-st | **Llama 3.2 Conversational** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3.2_(1B_and_3B)-Conversational.ipynb) | 2x faster | 70% less | | **Orpheus-TTS (3B)** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Orpheus_(3B)-TTS.ipynb) | 1.5x faster | 50% less | -- See all our notebooks for: [Kaggle](https://github.com/unslothai/notebooks?tab=readme-ov-file#-kaggle-notebooks), [GRPO](https://docs.unsloth.ai/get-started/unsloth-notebooks#grpo-reasoning-rl-notebooks), **[TTS](https://docs.unsloth.ai/get-started/unsloth-notebooks#text-to-speech-tts-notebooks)** & [Vision](https://docs.unsloth.ai/get-started/unsloth-notebooks#vision-multimodal-notebooks) -- See [all our models](https://docs.unsloth.ai/get-started/all-our-models) and [all our notebooks](https://docs.unsloth.ai/get-started/unsloth-notebooks) -- See detailed documentation for Unsloth [here](https://docs.unsloth.ai/) +- See all our notebooks for: [Kaggle](https://github.com/unslothai/notebooks?tab=readme-ov-file#-kaggle-notebooks), [GRPO](https://unsloth.ai/docs/get-started/unsloth-notebooks#grpo-reasoning-rl-notebooks), [TTS](https://unsloth.ai/docs/get-started/unsloth-notebooks#text-to-speech-tts-notebooks) & [Vision](https://unsloth.ai/docs/get-started/unsloth-notebooks#vision-multimodal-notebooks) +- See [all our models](https://unsloth.ai/docs/get-started/unsloth-model-catalog) and [all our notebooks](https://unsloth.ai/docs/get-started/unsloth-notebooks) +- See detailed documentation for Unsloth [here](https://unsloth.ai/docs) ## ⚡ Quickstart ### Linux or WSL @@ -46,9 +46,9 @@ pip install unsloth ### Windows For Windows, `pip install unsloth` works only if you have Pytorch installed. Read our [Windows Guide](https://docs.unsloth.ai/get-started/installing-+-updating/windows-installation). ### Docker -Use our official [Unsloth Docker image](https://hub.docker.com/r/unsloth/unsloth) ```unsloth/unsloth``` container. Read our [Docker Guide](https://docs.unsloth.ai/get-started/install-and-update/docker). +Use our official [Unsloth Docker image](https://hub.docker.com/r/unsloth/unsloth) ```unsloth/unsloth``` container. Read our [Docker Guide](https://unsloth.ai/docs/get-started/install-and-update/docker). ### Blackwell & DGX Spark -For RTX 50x, B200, 6000 GPUs: `pip install unsloth`. Read our [Blackwell Guide](https://docs.unsloth.ai/basics/training-llms-with-blackwell-rtx-50-series-and-unsloth) and [DGX Spark Guide](https://docs.unsloth.ai/new/fine-tuning-llms-with-nvidia-dgx-spark-and-unsloth) for more details. +For RTX 50x, B200, 6000 GPUs: `pip install unsloth`. Read our [Blackwell Guide](https://unsloth.ai/docs/basics/fine-tuning-llms-with-blackwell-rtx-50-series-and-unsloth) and [DGX Spark Guide](https://unsloth.ai/docs/basics/fine-tuning-llms-with-nvidia-dgx-spark-and-unsloth) for more details. ## 🦥 Unsloth News - New RoPE & MLP **Triton Kernels** & **Padding Free + Packing**: 3x faster training & 30% less VRAM. [Blog](https://docs.unsloth.ai/new/3x-faster-training-packing) @@ -98,6 +98,7 @@ For RTX 50x, B200, 6000 GPUs: `pip install unsloth`. Read our [Blackwell Guide]( - Supports **all models** including [TTS](https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning), multimodal, [BERT](https://docs.unsloth.ai/get-started/unsloth-notebooks#other-important-notebooks) and more! Any model that works in transformers, works in Unsloth. - The most efficient library for [Reinforcement Learning (RL)](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide), using 80% less VRAM. Supports GRPO, GSPO, DrGRPO, DAPO etc. - **0% loss in accuracy** - no approximation methods - all exact. +- Export and [deploy your model](https://unsloth.ai/docs/basics/inference-and-deployment) to GGUF, llama.cpp, vLLM, SGLang and Hugging Face. - Supports NVIDIA (since 2018), [AMD](https://docs.unsloth.ai/get-started/install-and-update/amd) and Intel GPUs. Minimum CUDA Capability 7.0 (V100, T4, Titan V, RTX 20, 30, 40x, A100, H100, L40 etc) - Works on **Linux**, WSL and **Windows** - All kernels written in [OpenAI's Triton](https://openai.com/index/triton/) language. Manual backprop engine. @@ -283,7 +284,7 @@ docker run -d -e JUPYTER_PASSWORD="mypassword" \ Access Jupyter Lab at `http://localhost:8888` and start fine-tuning! ## 📜 Documentation -- Go to our official [Documentation](https://docs.unsloth.ai) for [running models](https://docs.unsloth.ai/basics/running-and-saving-models), [saving to GGUF](https://docs.unsloth.ai/basics/running-and-saving-models/saving-to-gguf), [checkpointing](https://docs.unsloth.ai/basics/finetuning-from-last-checkpoint), [evaluation](https://docs.unsloth.ai/get-started/fine-tuning-llms-guide#evaluation) and more! +- Go to our official [Documentation](https://unsloth.ai/docs) for [running models](https://docs.unsloth.ai/basics/running-and-saving-models), [saving to GGUF](https://docs.unsloth.ai/basics/running-and-saving-models/saving-to-gguf), [checkpointing](https://docs.unsloth.ai/basics/finetuning-from-last-checkpoint), [evaluation](https://docs.unsloth.ai/get-started/fine-tuning-llms-guide#evaluation) and more! - Read our Guides for: [Fine-tuning](https://docs.unsloth.ai/get-started/fine-tuning-llms-guide), [Reinforcement Learning](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide), [Text-to-Speech (TTS)](https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning), [Vision](https://docs.unsloth.ai/basics/vision-fine-tuning) and [any model](https://docs.unsloth.ai/models/tutorials-how-to-fine-tune-and-run-llms). - We support Huggingface's transformers, TRL, Trainer, Seq2SeqTrainer and Pytorch code. From 58235ee1927d2dd448762ff9fd1010e991435370 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 28 Dec 2025 19:57:43 -0800 Subject: [PATCH 50/96] Update FUNDING.yml (#3792) --- .github/FUNDING.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 4ebb6df3d0..ae5dade42d 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -3,7 +3,7 @@ github: unslothai patreon: # Replace with a single Patreon username open_collective: # Replace with a single Open Collective username -ko_fi: unsloth +ko_fi: # unsloth tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry liberapay: # Replace with a single Liberapay username From c0c21a1e227823dac76e1cfc08856ac3def1015b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alk=C4=B1n=20=C3=9Cnl=C3=BC?= Date: Mon, 29 Dec 2025 08:18:02 +0300 Subject: [PATCH 51/96] fix(trainer): import psutil to prevent NameError in _prepare_dataset (#3780) * fix(trainer): import psutil to prevent NameError in _prepare_dataset Fixes #3777 * Update rl.py --------- Co-authored-by: Daniel Han --- unsloth/models/rl.py | 1 + unsloth/tokenizer_utils.py | 1 + unsloth/trainer.py | 1 + 3 files changed, 3 insertions(+) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 4ea36519d9..003a0e7f1b 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -227,6 +227,7 @@ import numpy as np from contextlib import nullcontext from torch.nn import functional as F import inspect +import psutil from transformers import DataCollatorForSeq2Seq, DataCollatorForLanguageModeling as TransformersDataCollatorForLanguageModeling from transformers.training_args import ParallelMode diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 99651643a8..0136e3498e 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -25,6 +25,7 @@ import collections import numpy as np import gc import subprocess +import psutil from unsloth_zoo.tokenizer_utils import ( mean_of_trained_tokens, diff --git a/unsloth/trainer.py b/unsloth/trainer.py index c0b2dd03b6..0d98cff305 100644 --- a/unsloth/trainer.py +++ b/unsloth/trainer.py @@ -14,6 +14,7 @@ import logging import os +import psutil import warnings from dataclasses import dataclass, field from typing import Optional From ab815692a9c80d9737e3b0d67927363e6da3b527 Mon Sep 17 00:00:00 2001 From: Francesco Bertolotti Date: Mon, 29 Dec 2025 06:21:48 +0100 Subject: [PATCH 52/96] fastrope fix for zero strided tensors (#3782) Co-authored-by: Francesco Bertolotti --- unsloth/kernels/rope_embedding.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.py index a032e0f7fc..fcc9cb923b 100644 --- a/unsloth/kernels/rope_embedding.py +++ b/unsloth/kernels/rope_embedding.py @@ -312,8 +312,8 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function): _, n_heads_K, _, _ = K.shape # Inplace rotary embedding is generally fine - Q_out = Q.clone() if not Q.is_contiguous else Q - K_out = K.clone() if not K.is_contiguous else K + Q_out = Q.clone() if not Q.is_contiguous() else Q + K_out = K.clone() if not K.is_contiguous() else K if has_indices: # TRL's rotary indices are always in int32, so casting is just for safety @@ -383,21 +383,21 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function): else ctx.cos.new_empty(1, dtype = torch.int32) ) + # Inplace rotary embedding is generally fine + dQ_out = dQ.clone() if not dQ.is_contiguous() else dQ + dK_out = dK.clone() if not dK.is_contiguous() else dK + Q_batch_stride, Q_head_stride, Q_seq_stride = ( - dQ.stride(0), - dQ.stride(1), - dQ.stride(2), + dQ_out.stride(0), + dQ_out.stride(1), + dQ_out.stride(2), ) K_batch_stride, K_head_stride, K_seq_stride = ( - dK.stride(0), - dK.stride(1), - dK.stride(2), + dK_out.stride(0), + dK_out.stride(1), + dK_out.stride(2), ) - # Inplace rotary embedding is generally fine - dQ_out = dQ.clone() if not dQ.is_contiguous else dQ - dK_out = dK.clone() if not dK.is_contiguous else dK - with torch_gpu_device(dQ.device): _rope_embedding_QK[(batch * ctx.seq_len, ctx.n_heads_Q)]( dQ_out, From c8b0bada94f55ab93848dbff28dcdec22b7cec31 Mon Sep 17 00:00:00 2001 From: Fizza Mukhtar Date: Sun, 28 Dec 2025 21:23:51 -0800 Subject: [PATCH 53/96] Fix crash when trl.experimental.openenv is unavailable (#3787) * Guard optional trl.experimental.openenv usage in RL patches * Simplify optional trl.openenv import handling * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- unsloth/models/rl_replacements.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 7d4d520c1f..3dfeea6871 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -949,11 +949,15 @@ def openenv_vllm_reload_weights(): return if Version(importlib_version("trl")) < Version("0.26.0"): return + try: import trl.experimental.openenv.utils as openenv_utils import trl.experimental.openenv as openenv except ImportError as e: logger.info(f"Unsloth: Failed to import trl openenv: {e}") + logger.info( + "Unsloth: trl.experimental.openenv not available — skipping RL openenv patches." + ) return src = inspect.getsource(openenv_utils.generate_rollout_completions) From fe82f5f3663eb75e106fa17f6bc65141265fd5cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=82=86=E3=82=8A?= Date: Mon, 29 Dec 2025 13:30:55 +0800 Subject: [PATCH 54/96] Fix Boolean value of Tensor ambiguity error in mistral.py (#3790) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix is_contiguous() method call and remove duplicate imports - Fix bug in rope_embedding.py where is_contiguous was used without parentheses, causing the method object (always truthy) to be evaluated instead of calling the method. This fixes issue #3781 where fast rope backpropagation was broken for zero strided/non-contiguous tensors. - Remove duplicate `import torch` in rl.py (lines 20 and 25) - Remove duplicate `import functools` and `import types` in vision.py 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 * Fix Boolean value of Tensor ambiguity error in mistral.py Replace `or` operator with explicit `is None` check when getting n_items from kwargs. The `or` operator fails when the value is a Tensor because Python cannot determine the boolean value of a multi-element tensor. Fixes #3766 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 * Update rope_embedding.py --------- Co-authored-by: yurekami Co-authored-by: Claude Opus 4.5 Co-authored-by: Daniel Han --- unsloth/models/mistral.py | 12 +++++++----- unsloth/models/rl.py | 1 - unsloth/models/vision.py | 2 -- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py index 0eed45c5cd..5e893d2b6f 100644 --- a/unsloth/models/mistral.py +++ b/unsloth/models/mistral.py @@ -307,9 +307,9 @@ def MistralForCausalLM_fast_forward( RETURN_LOGITS = False if not RETURN_LOGITS and labels is not None: - n_items = kwargs.get("num_items_in_batch", None) or kwargs.get( - "n_items", None - ) + n_items = kwargs.get("num_items_in_batch", None) + if n_items is None: + n_items = kwargs.get("n_items", None) logit_softcapping = getattr(self.config, "final_logit_softcapping", 0) # loss = fused_linear_cross_entropy( @@ -363,11 +363,13 @@ def MistralForCausalLM_fast_forward( shift_labels, kwargs.get("packed_seq_lengths"), ) + n_items = kwargs.get("num_items_in_batch", None) + if n_items is None: + n_items = kwargs.get("n_items", None) loss = fast_cross_entropy_loss( logits = shift_logits, labels = shift_labels, - n_items = kwargs.get("num_items_in_batch", None) - or kwargs.get("n_items", None), + n_items = n_items, ) if not return_dict: diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 003a0e7f1b..03f2c44701 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -22,7 +22,6 @@ from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union import inspect import os import re -import torch from unsloth_zoo.compiler import create_new_function from unsloth_zoo.log import logger from unsloth_zoo.logging_utils import PatchRLStatistics diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index e1cf8f6f82..36cfbf0b17 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -68,11 +68,9 @@ import functools import os import gc import math -import functools from typing import Optional, Tuple, List, Union import re, inspect, sys import contextlib -import types try: from huggingface_hub.utils import get_token From 9fedb1c11df2c4b7d1096962d2cf16d74372c80a Mon Sep 17 00:00:00 2001 From: lif <1835304752@qq.com> Date: Mon, 29 Dec 2025 15:17:58 +0800 Subject: [PATCH 55/96] fix: add support for init_lora_weights="corda" in get_peft_model (#3794) Add "corda" as an allowed value for the init_lora_weights parameter in FastLanguageModel.get_peft_model() and FastBaseModel.get_peft_model(). This enables users to use CorDA (Correlation-aware Decomposed Adaptation) initialization from PEFT, which provides an alternative LoRA initialization strategy for improved finetuning performance. Fixes #3693 Signed-off-by: majiayu000 <1835304752@qq.com> --- unsloth/models/_utils.py | 3 ++- unsloth/models/llama.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index abc8380562..ccb547f58e 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -1981,9 +1981,10 @@ def validate_loftq_config(loftq_config, lora_dropout, bias, init_lora_weights, m type(init_lora_weights) is bool or init_lora_weights == "gaussian" or init_lora_weights == "loftq" + or init_lora_weights == "corda" ): raise ValueError( - 'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq"].' + 'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq", "corda"].' ) if init_lora_weights == "loftq": diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 1d7695b9aa..762445b5e8 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -2779,9 +2779,10 @@ class FastLlamaModel: type(init_lora_weights) is bool or init_lora_weights == "gaussian" or init_lora_weights == "loftq" + or init_lora_weights == "corda" ): raise ValueError( - 'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq"].' + 'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq", "corda"].' ) if init_lora_weights == "loftq": From c452eb13f54dc572bb0b12c63ae47746543c4654 Mon Sep 17 00:00:00 2001 From: Fizza-Mukhtar Date: Tue, 30 Dec 2025 07:08:10 -0800 Subject: [PATCH 56/96] Fix 3D tensor support for bitsandbytes 8-bit matmul in forward pass --- unsloth/kernels/fast_lora.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index 60d0c318c3..16b679d005 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -379,9 +379,22 @@ class LoRA_QKV(torch.autograd.Function): ): dtype = X.dtype + # bitsandbytes 8-bit matmul expects 2D inputs. + # TorchInductor/AOTAutograd fails on 3D tensors during backward, + # so we explicitly flatten the sequence dimension. + orig_shape = X.shape + if X.dim() == 3: + X = X.view(-1, X.shape[-1]) + Q = matmul_lora(X, QW, QW_quant, QA, QB, QS) K = matmul_lora(X, KW, KW_quant, KA, KB, KS) V = matmul_lora(X, VW, VW_quant, VA, VB, VS) + + # Restore original shape after matmul + if len(orig_shape) == 3: + Q = Q.view(orig_shape[0], orig_shape[1], -1) + K = K.view(orig_shape[0], orig_shape[1], -1) + V = V.view(orig_shape[0], orig_shape[1], -1) ctx.custom_saved_tensors = ( QW, From f2e87251c721482d05bf8dd21452e4eb5c20ba02 Mon Sep 17 00:00:00 2001 From: Fizza-Mukhtar Date: Tue, 30 Dec 2025 07:56:01 -0800 Subject: [PATCH 57/96] Fix 3D tensor support for bitsandbytes 8-bit matmul in forward pass --- unsloth/kernels/fast_lora.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index 16b679d005..fbb18c3a15 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -383,12 +383,12 @@ class LoRA_QKV(torch.autograd.Function): # TorchInductor/AOTAutograd fails on 3D tensors during backward, # so we explicitly flatten the sequence dimension. orig_shape = X.shape + X_for_matmul = X if X.dim() == 3: - X = X.view(-1, X.shape[-1]) - - Q = matmul_lora(X, QW, QW_quant, QA, QB, QS) - K = matmul_lora(X, KW, KW_quant, KA, KB, KS) - V = matmul_lora(X, VW, VW_quant, VA, VB, VS) + X_for_matmul = X.view(-1, X.shape[-1]) + Q = matmul_lora(X_for_matmul, QW, QW_quant, QA, QB, QS) + K = matmul_lora(X_for_matmul, KW, KW_quant, KA, KB, KS) + V = matmul_lora(X_for_matmul, VW, VW_quant, VA, VB, VS) # Restore original shape after matmul if len(orig_shape) == 3: From e43e67cb18e3f9842ca34416db8f42b21f7154f2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 30 Dec 2025 15:58:40 +0000 Subject: [PATCH 58/96] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/kernels/fast_lora.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index fbb18c3a15..f1c0e298d9 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -389,7 +389,7 @@ class LoRA_QKV(torch.autograd.Function): Q = matmul_lora(X_for_matmul, QW, QW_quant, QA, QB, QS) K = matmul_lora(X_for_matmul, KW, KW_quant, KA, KB, KS) V = matmul_lora(X_for_matmul, VW, VW_quant, VA, VB, VS) - + # Restore original shape after matmul if len(orig_shape) == 3: Q = Q.view(orig_shape[0], orig_shape[1], -1) From b21b4e6252a8ee2381952d26d21fe023ad14c0d9 Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Tue, 30 Dec 2025 15:14:27 -0800 Subject: [PATCH 59/96] Refresh of Unsloth README.md with https://unsloth.ai/docs --- README.md | 115 +++++++++++++++++++++++++----------------------------- 1 file changed, 53 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index 7cd9d0bba4..ae1fccfbba 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@
- + unsloth logo @@ -18,7 +18,7 @@ ## ✨ Train for Free -Notebooks are beginner friendly. Read our [guide](https://docs.unsloth.ai/get-started/fine-tuning-guide). Add dataset, run, then deploy your trained model. +Notebooks are beginner friendly. Read our [guide](https://unsloth.ai/docs/get-started/fine-tuning-llms-guide). Add dataset, run, then deploy your trained model. | Model | Free Notebooks | Performance | Memory use | |-----------|---------|--------|----------| @@ -44,33 +44,35 @@ Notebooks are beginner friendly. Read our [guide](https://docs.unsloth.ai/get-st pip install unsloth ``` ### Windows -For Windows, `pip install unsloth` works only if you have Pytorch installed. Read our [Windows Guide](https://docs.unsloth.ai/get-started/installing-+-updating/windows-installation). +For Windows, `pip install unsloth` works only if you have Pytorch installed. Read our [Windows Guide](https://unsloth.ai/docs/get-started/install-and-update/windows-installation). + ### Docker Use our official [Unsloth Docker image](https://hub.docker.com/r/unsloth/unsloth) ```unsloth/unsloth``` container. Read our [Docker Guide](https://unsloth.ai/docs/get-started/install-and-update/docker). + ### Blackwell & DGX Spark For RTX 50x, B200, 6000 GPUs: `pip install unsloth`. Read our [Blackwell Guide](https://unsloth.ai/docs/basics/fine-tuning-llms-with-blackwell-rtx-50-series-and-unsloth) and [DGX Spark Guide](https://unsloth.ai/docs/basics/fine-tuning-llms-with-nvidia-dgx-spark-and-unsloth) for more details. ## 🦥 Unsloth News -- New RoPE & MLP **Triton Kernels** & **Padding Free + Packing**: 3x faster training & 30% less VRAM. [Blog](https://docs.unsloth.ai/new/3x-faster-training-packing) -- **Ministral 3** by Mistral: Run Ministral 3 or fine-tune with vision/RL sodoku notebooks. [Guide](https://docs.unsloth.ai/new/ministral-3) • [Notebooks](https://docs.unsloth.ai/new/ministral-3#fine-tuningb) -- **500K Context**: Training a 20B model with >500K context is now possible on an 80GB GPU. [Blog](https://docs.unsloth.ai/new/500k-context-length-fine-tuning) -- **FP8 Reinforcement Learning**: You can now do FP8 GRPO on consumer GPUs. [Blog](https://docs.unsloth.ai/new/fp8-reinforcement-learning) • [Notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_8B_FP8_GRPO.ipynb) -- **DeepSeek-OCR**: Fine-tune to improve language understanding by 89%. [Guide](https://docs.unsloth.ai/new/deepseek-ocr-run-and-fine-tune) • [Notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Deepseek_OCR_(3B).ipynb) -- **Docker**: Use Unsloth with no setup & environment issues with our new image. [Guide](https://docs.unsloth.ai/new/how-to-train-llms-with-unsloth-and-docker) • [Docker image](https://hub.docker.com/r/unsloth/unsloth) -- **gpt-oss RL**: Introducing the fastest possible inference for gpt-oss RL! [Read blog](https://docs.unsloth.ai/new/gpt-oss-reinforcement-learning) -- **Vision RL**: You can now train VLMs with GRPO or GSPO in Unsloth! [Read guide](https://docs.unsloth.ai/new/vision-reinforcement-learning-vlm-rl) -- **gpt-oss** by OpenAI: Read our [Unsloth Flex Attention](https://docs.unsloth.ai/new/long-context-gpt-oss-training) blog and [gpt-oss Guide](https://docs.unsloth.ai/basics/gpt-oss). 20B works on 14GB VRAM. 120B on 65GB. +- New RoPE & MLP **Triton Kernels** & **Padding Free + Packing**: 3x faster training & 30% less VRAM. [Blog](https://unsloth.ai/docs/new/3x-faster-training-packing) +- **New Mistral**: Run Ministral 3 or Devstral 2 and fine-tune with vision/RL sodoku notebooks. [Guide](https://unsloth.ai/docs/models/ministral-3) • [Notebooks](https://unsloth.ai/docs/models/ministral-3#fine-tuning-ministral-3) +- **500K Context**: Training a 20B model with >500K context is now possible on an 80GB GPU. [Blog](https://unsloth.ai/docs/new/500k-context-length-fine-tuning) +- **FP8 Reinforcement Learning**: You can now do FP8 GRPO on consumer GPUs. [Blog](https://unsloth.ai/docs/new/fp8-reinforcement-learning) • [Notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_8B_FP8_GRPO.ipynb) +- **DeepSeek-OCR**: Fine-tune to improve language understanding by 89%. [Guide](https://unsloth.ai/docs/models/deepseek-ocr-how-to-run-and-fine-tune) • [Notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Deepseek_OCR_(3B).ipynb) +- **Docker**: Use Unsloth with no setup & environment issues with our new image. [Guide](https://unsloth.ai/docs/new/how-to-fine-tune-llms-with-unsloth-and-docker) • [Docker image](https://hub.docker.com/r/unsloth/unsloth) +- **gpt-oss RL**: Introducing the fastest possible inference for gpt-oss RL! [Read blog](https://unsloth.ai/docs/models/gpt-oss-how-to-run-and-fine-tune/gpt-oss-reinforcement-learning) +- **Vision RL**: You can now train VLMs with GRPO or GSPO in Unsloth! [Read guide](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide/vision-reinforcement-learning-vlm-rl) +- **gpt-oss** by OpenAI: Read our [Unsloth Flex Attention](https://unsloth.ai/docs/models/gpt-oss-how-to-run-and-fine-tune/long-context-gpt-oss-training) blog and [gpt-oss Guide](https://unsloth.ai/docs/models/gpt-oss-how-to-run-and-fine-tune). 20B works on 14GB VRAM. 120B on 65GB.
Click for more news -- **Quantization-Aware Training**: We collabed with Pytorch, recovering ~70% accuracy. [Read blog](https://docs.unsloth.ai/new/quantization-aware-training-qat) -- **Memory-efficient RL**: We're introducing even better RL. Our new kernels & algos allows faster RL with 50% less VRAM & 10× more context. [Read blog](https://docs.unsloth.ai/new/memory-efficient-rl) -- **Gemma 3n** by Google: [Read Blog](https://docs.unsloth.ai/basics/gemma-3n-how-to-run-and-fine-tune). We [uploaded GGUFs, 4-bit models](https://huggingface.co/collections/unsloth/gemma-3n-685d3874830e49e1c93f9339). -- **[Text-to-Speech (TTS)](https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning)** is now supported, including `sesame/csm-1b` and STT `openai/whisper-large-v3`. -- **[Qwen3](https://docs.unsloth.ai/basics/qwen3-how-to-run-and-fine-tune)** is now supported. Qwen3-30B-A3B fits on 17.5GB VRAM. -- Introducing **[Dynamic 2.0](https://docs.unsloth.ai/basics/unsloth-dynamic-2.0-ggufs)** quants that set new benchmarks on 5-shot MMLU & Aider Polyglot. -- [**EVERYTHING** is now supported](https://unsloth.ai/blog/gemma3#everything) - all models (TTS, BERT, Mamba), FFT, etc. [MultiGPU](https://docs.unsloth.ai/basics/multi-gpu-training-with-unsloth) coming soon. Enable FFT with `full_finetuning = True`, 8-bit with `load_in_8bit = True`. +- **Quantization-Aware Training**: We collabed with Pytorch, recovering ~70% accuracy. [Read blog](https://unsloth.ai/docs/basics/quantization-aware-training-qat) +- **Memory-efficient RL**: We're introducing even better RL. Our new kernels & algos allows faster RL with 50% less VRAM & 10× more context. [Read blog](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide/memory-efficient-rl) +- **Gemma 3n** by Google: [Read Blog](https://unsloth.ai/docs/models/gemma-3-how-to-run-and-fine-tune/gemma-3n-how-to-run-and-fine-tune). We [uploaded GGUFs, 4-bit models](https://huggingface.co/collections/unsloth/gemma-3n-685d3874830e49e1c93f9339). +- **[Text-to-Speech (TTS)](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning)** is now supported, including `sesame/csm-1b` and STT `openai/whisper-large-v3`. +- **[Qwen3](https://unsloth.ai/docs/models/qwen3-how-to-run-and-fine-tune)** is now supported. Qwen3-30B-A3B fits on 17.5GB VRAM. +- Introducing **[Dynamic 2.0](https://unsloth.ai/docs/basics/unsloth-dynamic-2.0-ggufs)** quants that set new benchmarks on 5-shot MMLU & Aider Polyglot. +- [**EVERYTHING** is now supported](https://unsloth.ai/blog/gemma3#everything) - all models (TTS, BERT, Mamba), FFT, etc. [MultiGPU](https://unsloth.ai/docs/basics/multi-gpu-training-with-unsloth) coming soon. Enable FFT with `full_finetuning = True`, 8-bit with `load_in_8bit = True`. - 📣 [DeepSeek-R1](https://unsloth.ai/blog/deepseek-r1) - run or fine-tune them [with our guide](https://unsloth.ai/blog/deepseek-r1). All model uploads: [here](https://huggingface.co/collections/unsloth/deepseek-r1-all-versions-678e1c48f5d2fce87892ace5). - 📣 Introducing Long-context [Reasoning (GRPO)](https://unsloth.ai/blog/grpo) in Unsloth. Train your own reasoning model with just 5GB VRAM. Transform Llama, Phi, Mistral etc. into reasoning LLMs! - 📣 Introducing Unsloth [Dynamic 4-bit Quantization](https://unsloth.ai/blog/dynamic-4bit)! We dynamically opt not to quantize certain parameters and this greatly increases accuracy while only using <10% more VRAM than BnB 4-bit. See our collection on [Hugging Face here.](https://huggingface.co/collections/unsloth/unsloth-4-bit-dynamic-quants-67503bb873f89e15276c44e7) @@ -84,28 +86,29 @@ For RTX 50x, B200, 6000 GPUs: `pip install unsloth`. Read our [Blackwell Guide](
## 🔗 Links and Resources -| Type | Links | -| ------------------------------- | --------------------------------------- | -|   **r/unsloth Reddit** | [Join Reddit community](https://reddit.com/r/unsloth)| -| 📚 **Documentation & Wiki** | [Read Our Docs](https://docs.unsloth.ai) | -|   **Twitter (aka X)** | [Follow us on X](https://twitter.com/unslothai)| -| 💾 **Installation** | [Pip & Docker Install](https://docs.unsloth.ai/get-started/installing-+-updating)| -| 🔮 **Our Models** | [Unsloth Catalog](https://docs.unsloth.ai/get-started/all-our-models)| -| ✍️ **Blog** | [Read our Blogs](https://unsloth.ai/blog)| +| Type | Links | +| ----------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ | +|   **r/unsloth Reddit** | [Join Reddit community](https://reddit.com/r/unsloth) | +| 📚 **Documentation & Wiki** | [Read Our Docs](https://unsloth.ai/docs) | +|   **Twitter (aka X)** | [Follow us on X](https://twitter.com/unslothai) | +| 💾 **Installation** | [Pip & Docker Install](https://unsloth.ai/docs/get-started/install-and-update) | +| 🔮 **Our Models** | [Unsloth Catalog](https://unsloth.ai/docs/get-started/unsloth-model-catalog) | +| ✍️ **Blog** | [Read our Blogs](https://unsloth.ai/blog) | ## ⭐ Key Features -- Supports **full-finetuning**, pretraining, 4b-bit, 16-bit and **FP8** training -- Supports **all models** including [TTS](https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning), multimodal, [BERT](https://docs.unsloth.ai/get-started/unsloth-notebooks#other-important-notebooks) and more! Any model that works in transformers, works in Unsloth. -- The most efficient library for [Reinforcement Learning (RL)](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide), using 80% less VRAM. Supports GRPO, GSPO, DrGRPO, DAPO etc. -- **0% loss in accuracy** - no approximation methods - all exact. -- Export and [deploy your model](https://unsloth.ai/docs/basics/inference-and-deployment) to GGUF, llama.cpp, vLLM, SGLang and Hugging Face. -- Supports NVIDIA (since 2018), [AMD](https://docs.unsloth.ai/get-started/install-and-update/amd) and Intel GPUs. Minimum CUDA Capability 7.0 (V100, T4, Titan V, RTX 20, 30, 40x, A100, H100, L40 etc) -- Works on **Linux**, WSL and **Windows** -- All kernels written in [OpenAI's Triton](https://openai.com/index/triton/) language. Manual backprop engine. -- If you trained a model with 🦥Unsloth, you can use this cool sticker!   + +* Supports **full-finetuning**, pretraining, 4b-bit, 16-bit and **FP8** training +* Supports **all models** including [TTS](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning), multimodal, [BERT](https://unsloth.ai/docs/get-started/unsloth-notebooks#other-important-notebooks) and more! Any model that works in transformers, works in Unsloth. +* The most efficient library for [Reinforcement Learning (RL)](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide), using 80% less VRAM. Supports GRPO, GSPO, DrGRPO, DAPO etc. +* **0% loss in accuracy** - no approximation methods - all exact. +* Export and [deploy your model](https://unsloth.ai/docs/basics/inference-and-deployment) to GGUF, llama.cpp, vLLM, SGLang and Hugging Face. +* Supports NVIDIA (since 2018), [AMD](https://unsloth.ai/docs/get-started/install-and-update/amd) and Intel GPUs. Minimum CUDA Capability 7.0 (V100, T4, Titan V, RTX 20, 30, 40x, A100, H100, L40 etc) +* Works on **Linux**, WSL and **Windows** +* All kernels written in OpenAI's Triton language. Manual backprop engine. +* If you trained a model with 🦥Unsloth, you can use this cool sticker!   ## 💾 Install Unsloth -You can also see our docs for more detailed installation and updating instructions [here](https://docs.unsloth.ai/get-started/installing-+-updating). +You can also see our docs for more detailed installation and updating instructions [here](https://unsloth.ai/docs/get-started/install-and-update). Unsloth supports Python 3.13 or lower. @@ -125,7 +128,7 @@ See [here](#advanced-pip-installation) for advanced pip install instructions. You should install the latest driver for your GPU. Download drivers here: [NVIDIA GPU Driver](https://www.nvidia.com/Download/index.aspx). 3. **Install Visual Studio C++:** - You will need Visual Studio, with C++ installed. By default, C++ is not installed with [Visual Studio](https://visualstudio.microsoft.com/vs/community/), so make sure you select all of the C++ options. Also select options for Windows 10/11 SDK. For detailed instructions with options, see [here](https://docs.unsloth.ai/get-started/installing-+-updating). + You will need Visual Studio, with C++ installed. By default, C++ is not installed with [Visual Studio](https://visualstudio.microsoft.com/vs/community/), so make sure you select all of the C++ options. Also select options for Windows 10/11 SDK. For detailed instructions with options, see [here](https://unsloth.ai/docs/get-started/install-and-update/windows-installation#method-3-windows-directly). 5. **Install CUDA Toolkit:** Follow the instructions to install [CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit-archive). @@ -140,19 +143,7 @@ See [here](#advanced-pip-installation) for advanced pip install instructions. pip install unsloth ``` -#### Notes -To run Unsloth directly on Windows: -- Install Triton from this Windows fork and follow the instructions [here](https://github.com/woct0rdho/triton-windows) (be aware that the Windows fork requires PyTorch >= 2.4 and CUDA 12) -- In the `SFTConfig`, set `dataset_num_proc=1` to avoid a crashing issue: -```python -SFTConfig( - dataset_num_proc=1, - ... -) -``` - #### Advanced/Troubleshooting - For **advanced installation instructions** or if you see weird errors during installations: First try using an isolated environment via then `pip install unsloth` @@ -269,7 +260,7 @@ print(f'pip install --upgrade pip && pip install --no-deps git+https://github.co ``` ### Docker Installation You can use our pre-built Docker container with all dependencies to use Unsloth instantly with no setup required. -[Read our guide](https://docs.unsloth.ai/get-started/install-and-update/docker). +[Read our guide](https://unsloth.ai/docs/get-started/install-and-update/docker). This container requires installing [NVIDIA's Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). @@ -284,9 +275,9 @@ docker run -d -e JUPYTER_PASSWORD="mypassword" \ Access Jupyter Lab at `http://localhost:8888` and start fine-tuning! ## 📜 Documentation -- Go to our official [Documentation](https://unsloth.ai/docs) for [running models](https://docs.unsloth.ai/basics/running-and-saving-models), [saving to GGUF](https://docs.unsloth.ai/basics/running-and-saving-models/saving-to-gguf), [checkpointing](https://docs.unsloth.ai/basics/finetuning-from-last-checkpoint), [evaluation](https://docs.unsloth.ai/get-started/fine-tuning-llms-guide#evaluation) and more! -- Read our Guides for: [Fine-tuning](https://docs.unsloth.ai/get-started/fine-tuning-llms-guide), [Reinforcement Learning](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide), [Text-to-Speech (TTS)](https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning), [Vision](https://docs.unsloth.ai/basics/vision-fine-tuning) and [any model](https://docs.unsloth.ai/models/tutorials-how-to-fine-tune-and-run-llms). -- We support Huggingface's transformers, TRL, Trainer, Seq2SeqTrainer and Pytorch code. +* Go to our official [Documentation](https://unsloth.ai/docs) for [running models](https://unsloth.ai/docs/basics/inference-and-deployment), [saving to GGUF](https://unsloth.ai/docs/basics/inference-and-deployment/saving-to-gguf), [checkpointing](https://unsloth.ai/docs/basics/finetuning-from-last-checkpoint), [evaluation](https://unsloth.ai/docs/get-started/fine-tuning-llms-guide#evaluation) and more! +* Read our Guides for: [Fine-tuning](https://unsloth.ai/docs/get-started/fine-tuning-llms-guide), [Reinforcement Learning](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide), [Text-to-Speech (TTS)](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning), [Vision](https://unsloth.ai/docs/basics/vision-fine-tuning) and [any model](https://unsloth.ai/docs/models/tutorials-how-to-fine-tune-and-run-llms). +* We support Huggingface's transformers, TRL, Trainer, Seq2SeqTrainer and Pytorch code. Unsloth example code to fine-tune gpt-oss-20b: @@ -311,8 +302,9 @@ model, tokenizer = FastModel.from_pretrained( max_seq_length = 2048, # Choose any for long context! load_in_4bit = True, # 4-bit quantization. False = 16-bit LoRA. load_in_8bit = False, # 8-bit quantization - load_in_16bit = False, # [NEW!] 16-bit LoRA + load_in_16bit = False, # 16-bit LoRA full_finetuning = False, # Use for full fine-tuning. + trust_remote_code = False, # Enable to support new models # token = "hf_...", # use one if using gated models ) @@ -351,7 +343,7 @@ trainer = SFTTrainer( ) trainer.train() -# Go to https://docs.unsloth.ai for advanced tips like +# Go to https://unsloth.ai/docs for advanced tips like # (1) Saving to GGUF / merging to 16bit for vLLM or SGLang # (2) Continued training from a saved LoRA adapter # (3) Adding an evaluation loop / OOMs @@ -360,14 +352,15 @@ trainer.train() ## 💡 Reinforcement Learning -[RL](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide) including [GRPO](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide#training-with-grpo), [GSPO](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide/gspo-reinforcement-learning), **FP8** traning, DrGRPO, DAPO, PPO, Reward Modelling, Online DPO all work with Unsloth. -Read our [Reinforcement Learning Guide](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide) or our [advanced RL docs](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide/advanced-rl-documentation) for batching, generation & training parameters. +[RL](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide) including [GRPO](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide#training-with-grpo), [GSPO](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide/gspo-reinforcement-learning), [**FP8** training](https://unsloth.ai/docs/new/fp8-reinforcement-learning), DrGRPO, DAPO, PPO, Reward Modelling, Online DPO all work with Unsloth. + +Read our [Reinforcement Learning Guide](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide) or our [advanced RL docs](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide/advanced-rl-documentation) for batching, generation & training parameters. List of RL notebooks: - gpt-oss GSPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/gpt-oss-(20B)-GRPO.ipynb) -- Qwen2.5-VL GSPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen2_5_7B_VL_GRPO.ipynb) +- - ***FP8*** Qwen3-8B GRPO notebook (L4): [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_8B_FP8_GRPO.ipynb) +- Qwen2.3-VL GSPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_VL_(8B)-Vision-GRPO.ipynb) - Advanced Qwen3 GRPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_(4B)-GRPO.ipynb) -- ***FP8*** Qwen3-8B GRPO notebook (L4): [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_8B_FP8_GRPO.ipynb) - ORPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3_(8B)-ORPO.ipynb) - DPO Zephyr notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Zephyr_(7B)-DPO.ipynb) - KTO notebook: [Link](https://colab.research.google.com/drive/1MRgGtLWuZX4ypSfGguFgC-IblTvO2ivM?usp=sharing) @@ -427,6 +420,4 @@ You can cite the Unsloth repo as follows: - The [llama.cpp library](https://github.com/ggml-org/llama.cpp) that lets users save models with Unsloth - The Hugging Face team and their libraries: [transformers](https://github.com/huggingface/transformers) and [TRL](https://github.com/huggingface/trl) - The Pytorch and [Torch AO](https://github.com/unslothai/unsloth/pull/3391) team for their contributions -- [Erik](https://github.com/erikwijmans) for his help adding [Apple's ML Cross Entropy](https://github.com/apple/ml-cross-entropy) in Unsloth -- [Etherl](https://github.com/Etherll) for adding support for [TTS, diffusion and BERT models](https://github.com/unslothai/notebooks/pull/34) - And of course for every single person who has contributed or has used Unsloth! From 982ae7bbebc1bef9c24dae857c794ac82cd75981 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 31 Dec 2025 21:35:48 -0800 Subject: [PATCH 60/96] Fix correctness bugs in rl.py, rl_replacements.py, and vision.py (#3811) * Fix correctness bugs in rl.py, rl_replacements.py, and vision.py 1. rl_replacements.py (lines 864, 870): Fixed undefined `nanmin`/`nanmax` functions by using `.nan_to_num(nan=inf/-inf).min()/.max()` pattern. PyTorch doesn't have torch.nanmin/nanmax, so we replace NaN values before computing min/max. 2. vision.py (line 150): Fixed bug where code checked for "input" key but then accessed kwargs["input_ids"] instead of kwargs["input"]. 3. vision.py (line 159): Fixed bug where literal string "key" was used instead of the variable `key` when accessing kwargs. 4. rl.py (lines 903, 905): Fixed non-existent `MathError` exception by replacing with `ValueError`. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- unsloth/models/rl.py | 4 ++-- unsloth/models/rl_replacements.py | 10 ++++++++-- unsloth/models/vision.py | 4 ++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 03f2c44701..e1c43b8b85 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -900,9 +900,9 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): if "temperature" in call_args: check_temperature = ( "if temperature <= 0:\n" - " raise MathError('Unsloth: Please set a positive non-zero temperature since your results will be wrong.')\n" + " raise ValueError('Unsloth: Please set a positive non-zero temperature since your results will be wrong.')\n" "elif temperature >= 10:\n" - " raise MathError('Unsloth: Please set a positive non-zero temperature less than 10, since sampling will be quite erratic.')\n" + " raise ValueError('Unsloth: Please set a positive non-zero temperature less than 10, since sampling will be quite erratic.')\n" "\n" ) extra_args += check_temperature diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 3dfeea6871..5e079335ae 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -861,13 +861,19 @@ def grpo_trainer_compute_loss(function_name, function): else torch.tensor(0.0, device = self.model.device) ) self._metrics[mode]["sampling/importance_sampling_ratio/min"].append( - nanmin(self.accelerator.gather(min_importance_sampling_ratio)).item() + self.accelerator.gather(min_importance_sampling_ratio) + .nan_to_num(nan = float("inf")) + .min() + .item() ) self._metrics[mode]["sampling/importance_sampling_ratio/mean"].append( self.accelerator.gather(mean_importance_sampling_ratio).nanmean().item() ) self._metrics[mode]["sampling/importance_sampling_ratio/max"].append( - nanmax(self.accelerator.gather(max_importance_sampling_ratio)).item() + self.accelerator.gather(max_importance_sampling_ratio) + .nan_to_num(nan = float("-inf")) + .max() + .item() ) return loss diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 36cfbf0b17..c909f963b9 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -147,7 +147,7 @@ def unsloth_base_fast_generate( elif "input_ids" in kwargs: input_ids = kwargs["input_ids"] elif "input" in kwargs: - input_ids = kwargs["input_ids"] + input_ids = kwargs["input"] elif "input_features" in kwargs: input_ids = kwargs["input_features"] elif "input_embeds" in kwargs: @@ -156,7 +156,7 @@ def unsloth_base_fast_generate( input_ids = kwargs["inputs"] else: key = next(iter(kwargs.keys())) - if type(kwargs["key"]) is not torch.Tensor: + if type(kwargs[key]) is not torch.Tensor: raise TypeError("Unsloth: You need to pass in input_ids to .generate!") input_ids = kwargs[key] assert type(input_ids) is torch.Tensor From 963bc35a961d02fba7a0245938e2af306479d4d6 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 1 Jan 2026 02:36:33 -0800 Subject: [PATCH 61/96] Fix correctness bugs across multiple model files (#3813) 1. cohere.py:347-348 - Fixed wrong variable names in QK normalization. Used `Q`/`K` but variables were named `Qn`/`Kn`. This caused NameError when `use_qk_norm=True` (e.g., c4ai-command-r-plus models). 2. cohere.py:482 - Fixed wrong object reference in inference loop. Used `self.mlp` but should be `decoder_layer.mlp` since we're iterating through decoder layers. Caused AttributeError during inference. 3. falcon_h1.py:459,461 - Fixed wrong attribute names in inference path. Used `post_attention_layernorm` and `mlp` but Falcon H1 uses `pre_ff_layernorm` and `feed_forward`. Caused AttributeError during generation. 4. qwen3_moe.py:210 - Fixed wrong module path with incorrect capitalization. Used `transformers.models.Qwen3Moe` but should be `transformers.models.qwen3_moe`. Caused AttributeError when patching rotary embeddings. 5. qwen3_moe.py:239 - Fixed wrong model_patcher class. Used `FastQwen3Model` but should be `FastQwen3MoeModel` for MoE models. Caused incorrect patching for Qwen3 MoE models. 6. hf_hub.py:21-22 - Fixed floor division and missing return for billion values. Used `//` instead of `/` for millions, and had no return for values >= 1B. Caused incorrect formatting and None return for large numbers. 7. save.py:550 - Fixed self-assignment that did nothing. `sharded_ram_usage = sharded_ram_usage` should be `= max_shard_size`. Caused integer shard sizes to be ignored. 8. rl.py:562-567 - Fixed orphan string not included in length_check. The elif branch for max_seq_length validation was a standalone string expression, not concatenated to length_check. Caused silent skip of the max_seq_length > model_max_seq_length warning. 9. granite.py:49-52 - Fixed wrong model name and version in error message. Said "Gemma2" and "4.42.3" but should be "Granite" and "4.45.0". --- unsloth/models/cohere.py | 6 +++--- unsloth/models/falcon_h1.py | 4 ++-- unsloth/models/granite.py | 6 +++--- unsloth/models/qwen3_moe.py | 4 ++-- unsloth/models/rl.py | 6 +++++- unsloth/save.py | 2 +- unsloth/utils/hf_hub.py | 4 +++- 7 files changed, 19 insertions(+), 13 deletions(-) diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.py index a091a0173f..e9f56763d6 100644 --- a/unsloth/models/cohere.py +++ b/unsloth/models/cohere.py @@ -344,8 +344,8 @@ def CohereAttention_fast_forward_inference( Kn = Kn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2) Vn = Vn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2) if self.use_qk_norm: - Q = fast_layernorm_inference(self.q_norm, Q, self.q_norm_out_weight) - K = fast_layernorm_inference(self.k_norm, K, self.k_norm_out_weight) + Qn = fast_layernorm_inference(self.q_norm, Qn, self.q_norm_out_weight) + Kn = fast_layernorm_inference(self.k_norm, Kn, self.k_norm_out_weight) # cos, sin = self.rotary_emb(Vn, seq_len = kv_seq_len) # Qn, Kn = inplace_rope_embedding(Qn, Kn, cos, sin, position_ids) @@ -479,7 +479,7 @@ def CohereModel_fast_forward_inference( ) ) - hidden_states_mlp = fast_swiglu_inference(self.mlp, hidden_states) + hidden_states_mlp = fast_swiglu_inference(decoder_layer.mlp, hidden_states) residual += hidden_states_attention residual += hidden_states_mlp hidden_states = residual diff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.py index fc5ea458a6..428f49d727 100644 --- a/unsloth/models/falcon_h1.py +++ b/unsloth/models/falcon_h1.py @@ -456,9 +456,9 @@ def FalconH1DecoderLayer_fast_forward( # Fully Connected residual = hidden_states hidden_states = fast_rms_layernorm_inference( - self.post_attention_layernorm, hidden_states + self.pre_ff_layernorm, hidden_states ) - hidden_states = fast_swiglu_inference(self.mlp, hidden_states) + hidden_states = fast_swiglu_inference(self.feed_forward, hidden_states) hidden_states += residual else: residual = hidden_states diff --git a/unsloth/models/granite.py b/unsloth/models/granite.py index 2632ab6914..f85f1b641f 100644 --- a/unsloth/models/granite.py +++ b/unsloth/models/granite.py @@ -46,9 +46,9 @@ except: transformers_version = Version(transformers_version) if not transformers_version >= Version("4.45.0"): raise ImportError( - f"Unsloth: Your transformers version of {transformers_version} does not support Gemma2.\n" - f"The minimum required version is 4.42.3.\n" - f'Try `pip install --upgrade "transformers>=4.42.3"`\n' + f"Unsloth: Your transformers version of {transformers_version} does not support Granite.\n" + f"The minimum required version is 4.45.0.\n" + f'Try `pip install --upgrade "transformers>=4.45.0"`\n' f"to obtain the latest transformers build, then restart this session." ) diff --git a/unsloth/models/qwen3_moe.py b/unsloth/models/qwen3_moe.py index bec3fa7b0d..e1f8c71b6b 100644 --- a/unsloth/models/qwen3_moe.py +++ b/unsloth/models/qwen3_moe.py @@ -207,7 +207,7 @@ class FastQwen3MoeModel(FastQwen3Model): # https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py\ import transformers.models.qwen3_moe.modeling_qwen3_moe - transformers.models.Qwen3Moe.modeling_qwen3_moe.Qwen3MoeRotaryEmbedding = ( + transformers.models.qwen3_moe.modeling_qwen3_moe.Qwen3MoeRotaryEmbedding = ( LlamaRotaryEmbedding ) return @@ -236,7 +236,7 @@ class FastQwen3MoeModel(FastQwen3Model): device_map = device_map, rope_scaling = rope_scaling, fix_tokenizer = fix_tokenizer, - model_patcher = FastQwen3Model, + model_patcher = FastQwen3MoeModel, tokenizer_name = tokenizer_name, trust_remote_code = trust_remote_code, **kwargs, diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index e1c43b8b85..22189f459c 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -559,8 +559,12 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): " if args_max_seq_length is None and model_max_seq_length is not None:\n" " max_seq_length = model.max_seq_length\n" " if hasattr(args, 'max_seq_length'): args.max_seq_length = max_seq_length\n" + " elif args_max_seq_length is not None and model_max_seq_length is not None:\n" + " if args_max_seq_length > model_max_seq_length:\n" + " print('Unsloth: You set `max_seq_length` as ' + str(args_max_seq_length) + ' but '\n" + " 'the maximum the model supports is ' + str(model_max_seq_length) + '. We shall reduce it.')\n" + " args.max_seq_length = model_max_seq_length\n" ) - " elif args_max_seq_length is not None and model_max_seq_length is not None:\n" " if args_max_seq_length > model_max_seq_length:\n" " print('Unsloth: You set `max_seq_length` as ' + str(args_max_seq_length) + ' but \n" " the maximum the model supports is ' + str(model_max_seq_length) + '. We shall reduce it.')\n" " args.max_seq_length = model_max_seq_length\n" extra_args += length_check # At this point max_seq_length might be set, but trl is moving to max_length diff --git a/unsloth/save.py b/unsloth/save.py index 3a275cf0c3..ceb36854d2 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -547,7 +547,7 @@ def unsloth_save_model( elif mb_found: sharded_ram_usage = int(mb_found.group(1)) * 1024 * 1024 elif type(max_shard_size) is int: - sharded_ram_usage = sharded_ram_usage + sharded_ram_usage = max_shard_size # Switch to our fast saving modules if it's a slow PC! n_cpus = psutil.cpu_count(logical = False) diff --git a/unsloth/utils/hf_hub.py b/unsloth/utils/hf_hub.py index 75df00fbf0..e3960ba0ce 100644 --- a/unsloth/utils/hf_hub.py +++ b/unsloth/utils/hf_hub.py @@ -19,7 +19,9 @@ def formatted_int(value: int) -> str: elif value < MILLION: return f"{float(value) / 1000:,.1f}K" elif value < BILLION: - return f"{float(value) // 1000000:,.1f}M" + return f"{float(value) / 1000000:,.1f}M" + else: + return f"{float(value) / 1000000000:,.1f}B" def get_model_info( From f7e0f4b152b67479f3b3b889f198daa3b9b28691 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 1 Jan 2026 12:54:21 +0000 Subject: [PATCH 62/96] Add TODO comment for ensure_weight_tying in vision models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- unsloth/models/vision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 9f847f2837..b4ce718f46 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -930,7 +930,7 @@ class FastBaseModel: task_type = TaskType.CAUSAL_LM, temporary_location = "_unsloth_temporary_saved_buffers", qat_scheme = None, - ensure_weight_tying = False, + ensure_weight_tying = False, # [TODO] Add `ensure_weight_tying` for `modules_to_save` for vision models **kwargs, ): if os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1": From 1080d0c4dc15ec97e40eacf17e81ff04c8518c88 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Fri, 2 Jan 2026 07:19:08 +0000 Subject: [PATCH 63/96] Fix Gemma3 QAT training instability with int8-int4 scheme Gemma3 models have a large vocabulary (262144 tokens) which causes training loss to explode when using int8 embedding quantization. This fix auto-detects Gemma3 models and switches from int8-int4 (phone-deployment) to int4 weight-only QAT for stable training. --- unsloth/models/_utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index ccb547f58e..3851e18f92 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -2198,6 +2198,18 @@ def _prepare_model_for_qat( from torchao.quantization.granularity import PerGroup, PerAxis from torchao.quantization.qat import QATConfig + # Gemma3 models have issues with int8 embedding quantization due to their + # large vocabulary size (262144). Auto-switch to int4 weight-only instead. + if qat_scheme == "int8-int4": + model_types = get_transformers_model_type(model.config) + is_gemma3 = any("gemma3" in mt or "gemma_3" in mt for mt in model_types) + if is_gemma3: + print( + "Unsloth: Gemma3 has a large vocabulary causing int8 embedding issues. " + "Switching to int4 weight-only QAT for training stability." + ) + qat_scheme = "int4" + if not isinstance(qat_scheme, TorchAOConfig): torchao_config: Optional[TorchAOConfig] = None if qat_scheme == "fp8-int4": From ae219fe05225b768953d2e6cdaac97b8813d8746 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 2 Jan 2026 00:14:44 -0800 Subject: [PATCH 64/96] fix_huggingface_hub --- unsloth/__init__.py | 3 +++ unsloth/import_fixes.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/unsloth/__init__.py b/unsloth/__init__.py index d10a0f8030..c74b248a83 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -30,16 +30,19 @@ from .import_fixes import ( check_fbgemm_gpu_version, torchvision_compatibility_check, fix_diffusers_warnings, + fix_huggingface_hub, ) fix_message_factory_issue() check_fbgemm_gpu_version() torchvision_compatibility_check() fix_diffusers_warnings() +fix_huggingface_hub() del fix_message_factory_issue del check_fbgemm_gpu_version del torchvision_compatibility_check del fix_diffusers_warnings +del fix_huggingface_hub # This check is critical because Unsloth optimizes these libraries by modifying # their code at import time. If they're imported first, the original (slower, diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index efc7a7f4cd..f388f4ea8d 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -539,3 +539,10 @@ def fix_executorch(): def fix_diffusers_warnings(): # Silence Flax classes are deprecated and will be removed in Diffusers v1.0.0. os.environ["DIFFUSERS_VERBOSITY"] = "error" + + +def fix_huggingface_hub(): + # huggingface_hub.is_offline_mode got removed, so add it back + import huggingface_hub + if not hasattr(huggingface_hub, "is_offline_mode"): + huggingface_hub.is_offline_mode = lambda: huggingface_hub.constants.HF_HUB_OFFLINE From 13e1255b6c8a35c6f1a96c14e0153ddb14289e60 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 2 Jan 2026 02:48:28 -0800 Subject: [PATCH 65/96] Update loader.py --- unsloth/models/loader.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 91016a13ba..247c72f43f 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -204,6 +204,17 @@ class FastLanguageModel(FastLlamaModel): "Unsloth: Please install vLLM before enabling `fast_inference`!\n" "You can do this in a terminal via `pip install vllm`" ) + if DEVICE_TYPE_TORCH == "cuda": + for i in range(DEVICE_COUNT): + # [TODO] DGX Spark vLLM breaks + if "NVIDIA GB10" in str(torch.cuda.get_device_name(i)).upper(): + print( + "Unsloth: DGX Spark detected - `fast_inference=True` is currently broken as of January 2026.\n" + "Defaulting to native Unsloth inference." + ) + fast_inference = False + break + # [TODO] For now fast_inference only works with fast_inference ie vLLM if load_in_fp8 != False: if not fast_inference: @@ -744,6 +755,17 @@ class FastModel(FastBaseModel): "Unsloth: Please install vLLM before enabling `fast_inference`!\n" "You can do this in a terminal via `pip install vllm`" ) + if DEVICE_TYPE_TORCH == "cuda": + for i in range(DEVICE_COUNT): + # [TODO] DGX Spark vLLM breaks + if "NVIDIA GB10" in str(torch.cuda.get_device_name(i)).upper(): + print( + "Unsloth: DGX Spark detected - `fast_inference=True` is currently broken as of January 2026.\n" + "Defaulting to native Unsloth inference." + ) + fast_inference = False + break + # [TODO] For now fast_inference only works with fast_inference ie vLLM if load_in_fp8 != False: if not fast_inference: From a24695dcc2e8bd34eca2cdc00e93a20bc2704c65 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 2 Jan 2026 03:41:51 -0800 Subject: [PATCH 66/96] Update import_fixes.py --- unsloth/import_fixes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index f388f4ea8d..da0fbc613b 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -97,6 +97,8 @@ if os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") != "1": sys.stderr = HidePrintMessage(sys.stderr) # https://github.com/pytorch/FBGEMM/blob/d99cd96490ec4aabac2ee95b1e76ea4dcfcfa628/fbgemm_gpu/experimental/gemm/triton_gemm/utils.py#L43-L52 sys.stderr.add_filter("TMA benchmarks will be running") + # Skipping import of cpp extensions due to incompatible torch version 2.9.0+cu128 for torchao version 0.15.0 + logging.getLogger("torchao").setLevel(logging.ERROR) # Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype' From 01e8f78f139a728e7a3e2fb8817d393ccde21e45 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 2 Jan 2026 05:05:47 -0800 Subject: [PATCH 67/96] Update import_fixes.py --- unsloth/import_fixes.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index da0fbc613b..f0dde256c1 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -20,6 +20,7 @@ from packaging.version import Version as TrueVersion import re import logging import textwrap +import warnings # We cannot do from unsloth_zoo.log import logger since FBGEMM might cause seg faults. UNSLOTH_ENABLE_LOGGING = os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") in ( @@ -99,6 +100,8 @@ if os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") != "1": sys.stderr.add_filter("TMA benchmarks will be running") # Skipping import of cpp extensions due to incompatible torch version 2.9.0+cu128 for torchao version 0.15.0 logging.getLogger("torchao").setLevel(logging.ERROR) + # SyntaxWarning: invalid escape sequence '\.' + warnings.filterwarnings("ignore", message = "invalid escape sequence", category = SyntaxWarning) # Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype' From c7d5f1569c4509a485258773f274f1599d0953ff Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Fri, 2 Jan 2026 13:58:08 +0000 Subject: [PATCH 68/96] Add helpful error messages for fast_generate when fast_inference=False When users load a model with fast_inference=False but then try to use vLLM-style arguments with fast_generate, they previously got confusing errors. This adds a wrapper that detects common mistakes and provides helpful guidance: - Using sampling_params: explains to use HF generate args instead - Using lora_request: explains LoRA weights are already merged - Passing text strings: shows how to tokenize input first Changes: - Add make_fast_generate_wrapper to _utils.py - Apply wrapper in llama.py when fast_inference=False - Apply wrapper in vision.py when fast_inference=False --- unsloth/models/_utils.py | 56 ++++++++++++++++++++++++++++++++++++++++ unsloth/models/llama.py | 2 +- unsloth/models/vision.py | 2 +- 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 3851e18f92..c1f626ec66 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -73,6 +73,7 @@ __all__ = [ "verify_fp8_support_if_applicable", "_get_inference_mode_context_manager", "hf_login", + "make_fast_generate_wrapper", ] import torch @@ -2378,3 +2379,58 @@ def hf_login(token: Optional[str] = None) -> Optional[str]: except Exception as e: logger.info(f"Failed to login to huggingface using token with error: {e}") return token + + +def make_fast_generate_wrapper(original_generate): + """ + Creates a wrapper around model.generate that checks for incorrect + vLLM-style usage when fast_inference=False. + """ + @functools.wraps(original_generate) + def _fast_generate_wrapper(*args, **kwargs): + # Check for vLLM-specific arguments + if "sampling_params" in kwargs: + raise ValueError( + "Unsloth: `sampling_params` is only supported when `fast_inference=True` (vLLM). " + "Since `fast_inference=False`, use HuggingFace generate arguments instead:\n" + " model.fast_generate(**tokens.to('cuda'), max_new_tokens=64, temperature=1.0, top_p=0.95)" + ) + + if "lora_request" in kwargs: + raise ValueError( + "Unsloth: `lora_request` is only supported when `fast_inference=True` (vLLM). " + "Since `fast_inference=False`, LoRA weights are already merged into the model." + ) + + # Check if first positional argument is a string or list of strings + if len(args) > 0: + first_arg = args[0] + is_string_input = False + + if isinstance(first_arg, str): + is_string_input = True + elif isinstance(first_arg, (list, tuple)) and len(first_arg) > 0: + if isinstance(first_arg[0], str): + is_string_input = True + + if is_string_input: + raise ValueError( + "Unsloth: Passing text strings to `fast_generate` is only supported " + "when `fast_inference=True` (vLLM). Since `fast_inference=False`, you must " + "tokenize the input first:\n\n" + " messages = tokenizer.apply_chat_template(\n" + " [{\"role\": \"user\", \"content\": \"Your prompt here\"}],\n" + " tokenize=True, add_generation_prompt=True,\n" + " return_tensors=\"pt\", return_dict=True\n" + " )\n" + " output = model.fast_generate(\n" + " **messages.to('cuda'),\n" + " max_new_tokens=64,\n" + " temperature=1.0,\n" + " )" + ) + + # Call original generate + return original_generate(*args, **kwargs) + + return _fast_generate_wrapper diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 29d41f4bb1..92d51b73ad 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -2326,7 +2326,7 @@ class FastLlamaModel: attn_implementation = "eager", **kwargs, ) - model.fast_generate = model.generate + model.fast_generate = make_fast_generate_wrapper(model.generate) model.fast_generate_batches = None else: from unsloth_zoo.vllm_utils import ( diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 1924373f67..6c5356e0b9 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -673,7 +673,7 @@ class FastBaseModel: **kwargs, ) if hasattr(model, "generate"): - model.fast_generate = model.generate + model.fast_generate = make_fast_generate_wrapper(model.generate) model.fast_generate_batches = error_out_no_vllm if offload_embedding: if bool( From f23735af0a673994ed005ab8d3f96a9aa8a6aefd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 2 Jan 2026 13:58:49 +0000 Subject: [PATCH 69/96] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/import_fixes.py | 9 +++++++-- unsloth/models/_utils.py | 5 +++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index f0dde256c1..bb6996a3e3 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -101,7 +101,9 @@ if os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") != "1": # Skipping import of cpp extensions due to incompatible torch version 2.9.0+cu128 for torchao version 0.15.0 logging.getLogger("torchao").setLevel(logging.ERROR) # SyntaxWarning: invalid escape sequence '\.' - warnings.filterwarnings("ignore", message = "invalid escape sequence", category = SyntaxWarning) + warnings.filterwarnings( + "ignore", message = "invalid escape sequence", category = SyntaxWarning + ) # Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype' @@ -549,5 +551,8 @@ def fix_diffusers_warnings(): def fix_huggingface_hub(): # huggingface_hub.is_offline_mode got removed, so add it back import huggingface_hub + if not hasattr(huggingface_hub, "is_offline_mode"): - huggingface_hub.is_offline_mode = lambda: huggingface_hub.constants.HF_HUB_OFFLINE + huggingface_hub.is_offline_mode = ( + lambda: huggingface_hub.constants.HF_HUB_OFFLINE + ) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index c1f626ec66..1cead3afaf 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -2386,6 +2386,7 @@ def make_fast_generate_wrapper(original_generate): Creates a wrapper around model.generate that checks for incorrect vLLM-style usage when fast_inference=False. """ + @functools.wraps(original_generate) def _fast_generate_wrapper(*args, **kwargs): # Check for vLLM-specific arguments @@ -2419,9 +2420,9 @@ def make_fast_generate_wrapper(original_generate): "when `fast_inference=True` (vLLM). Since `fast_inference=False`, you must " "tokenize the input first:\n\n" " messages = tokenizer.apply_chat_template(\n" - " [{\"role\": \"user\", \"content\": \"Your prompt here\"}],\n" + ' [{"role": "user", "content": "Your prompt here"}],\n' " tokenize=True, add_generation_prompt=True,\n" - " return_tensors=\"pt\", return_dict=True\n" + ' return_tensors="pt", return_dict=True\n' " )\n" " output = model.fast_generate(\n" " **messages.to('cuda'),\n" From d688d3f564195b397bcb0bb7bea69061945cedfd Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 2 Jan 2026 06:07:16 -0800 Subject: [PATCH 70/96] Bug fixes --- pyproject.toml | 4 ++-- unsloth/models/_utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index decc0e9f5f..20e3fd847f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ huggingfacenotorch = [ ] huggingface = [ "unsloth[huggingfacenotorch]", - "unsloth_zoo>=2025.12.7", + "unsloth_zoo>=2025.12.8", "torchvision", "unsloth[triton]", ] @@ -523,7 +523,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3 ; ('linux' in sys_platform)", ] colab-new = [ - "unsloth_zoo>=2025.12.7", + "unsloth_zoo>=2025.12.8", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,<=4.57.3", diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 1cead3afaf..545ba4794a 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.12.9" +__version__ = "2025.12.10" __all__ = [ "SUPPORTS_BFLOAT16", From 9a3908c55266f8f41a7d26dd02d701c75d8c8cc5 Mon Sep 17 00:00:00 2001 From: Fizza-Mukhtar Date: Fri, 2 Jan 2026 08:42:59 -0800 Subject: [PATCH 71/96] Make llama.cpp CURL support optional during CMake builds --- unsloth/save.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/unsloth/save.py b/unsloth/save.py index ceb36854d2..29d9cdcaff 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -130,6 +130,10 @@ ALLOWED_QUANTS = { "q3_k_xs": "3-bit extra small quantization", } +def has_curl(): + return shutil.which("curl") is not None + +CURL_FLAG = "-DLLAMA_CURL=ON" if has_curl() else "-DLLAMA_CURL=OFF" def print_quantization_methods(): for key, value in ALLOWED_QUANTS.items(): @@ -879,8 +883,9 @@ def install_llama_cpp_make_non_blocking(): # Uses new CMAKE n_jobs = max(int(psutil.cpu_count()), 1) # Use less CPUs since 1.5x faster check = os.system( - "cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF -DLLAMA_CURL=ON" + f"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}" ) + if check != 0: raise RuntimeError( f"*** Unsloth: Failed compiling llama.cpp using os.system(...) with error {check}. Please report this ASAP!" @@ -991,11 +996,12 @@ def install_llama_cpp_old(version = -10): if try_execute(commands) == "CMAKE": # Instead use CMAKE commands = [ - "cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF -DLLAMA_CURL=ON", + "cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}", f"cmake --build llama.cpp/build --config Release -j{psutil.cpu_count()*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}", "cp llama.cpp/build/bin/llama-* llama.cpp", "rm -rf llama.cpp/build", ] + try_execute(commands) # Check if successful @@ -1037,7 +1043,7 @@ def install_llama_cpp_blocking(use_cuda = False): if try_execute(commands) == "CMAKE": # Instead use CMAKE commands = [ - "cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF -DLLAMA_CURL=ON", + "cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}", f"cmake --build llama.cpp/build --config Release -j{psutil.cpu_count()*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}", "cp llama.cpp/build/bin/llama-* llama.cpp", "rm -rf llama.cpp/build", From 8fa3228590a38379851a770a822adce859c7ba26 Mon Sep 17 00:00:00 2001 From: Fizza-Mukhtar Date: Fri, 2 Jan 2026 08:55:58 -0800 Subject: [PATCH 72/96] Make llama.cpp CURL support optional during CMake builds --- unsloth/save.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unsloth/save.py b/unsloth/save.py index 29d9cdcaff..359010dbe2 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -996,7 +996,7 @@ def install_llama_cpp_old(version = -10): if try_execute(commands) == "CMAKE": # Instead use CMAKE commands = [ - "cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}", + f"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}", f"cmake --build llama.cpp/build --config Release -j{psutil.cpu_count()*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}", "cp llama.cpp/build/bin/llama-* llama.cpp", "rm -rf llama.cpp/build", @@ -1043,7 +1043,7 @@ def install_llama_cpp_blocking(use_cuda = False): if try_execute(commands) == "CMAKE": # Instead use CMAKE commands = [ - "cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}", + f"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}", f"cmake --build llama.cpp/build --config Release -j{psutil.cpu_count()*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}", "cp llama.cpp/build/bin/llama-* llama.cpp", "rm -rf llama.cpp/build", From c50b7499ff2f6f0485c6afb51d6c6aea209ae28e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 2 Jan 2026 16:58:04 +0000 Subject: [PATCH 73/96] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/save.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/unsloth/save.py b/unsloth/save.py index 359010dbe2..714df7682a 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -130,11 +130,14 @@ ALLOWED_QUANTS = { "q3_k_xs": "3-bit extra small quantization", } + def has_curl(): return shutil.which("curl") is not None + CURL_FLAG = "-DLLAMA_CURL=ON" if has_curl() else "-DLLAMA_CURL=OFF" + def print_quantization_methods(): for key, value in ALLOWED_QUANTS.items(): print(f'"{key}" ==> {value}') From 1ea6585b0ce2faed080c79ab8699b72b683de50c Mon Sep 17 00:00:00 2001 From: Kaitao Yang Date: Sat, 3 Jan 2026 22:38:37 -0800 Subject: [PATCH 74/96] remove redundant code of has_block --- unsloth/utils/attention_dispatch.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py index ccd49dada8..0e5f3c1951 100644 --- a/unsloth/utils/attention_dispatch.py +++ b/unsloth/utils/attention_dispatch.py @@ -219,16 +219,10 @@ def run_attention( ) if config.n_groups != 1 and not requires_grad: - if has_block: - out = out.view(bsz, q_len, config.n_kv_heads, config.n_groups, head_dim) - else: - out = out.view(bsz, q_len, config.n_kv_heads, config.n_groups, head_dim) + out = out.view(bsz, q_len, config.n_kv_heads, config.n_groups, head_dim) out = out.reshape(bsz, q_len, n_heads, head_dim) else: - if has_block: - out = out.view(bsz, q_len, n_heads, head_dim) - else: - out = out.view(bsz, q_len, n_heads, head_dim) + out = out.view(bsz, q_len, n_heads, head_dim) return out else: local_mask = context.attention_mask From 3d15865bbc569ce1e3847caa9191390e53d55ac9 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Sun, 4 Jan 2026 12:21:39 +0000 Subject: [PATCH 75/96] rl.py fixes: buffer reset, safer attribute access, typo fix 1. Auto-reset gradient checkpointing buffers after trainer.train() - Import and call reset_unsloth_gradient_checkpointing_buffers() in prepare_for_training_mode wrapper to free memory after training while keeping buffers ready for subsequent runs 2. Replace eval/exec with safer getattr/setattr - eval(f"trl.trainer.{trainer}") -> getattr(trl.trainer, trainer) - exec(f"...{unwrap} = ...") -> setattr(current_trainer, unwrap, ...) - exec(f"Trainer.prediction_step=...") -> direct assignment 3. Fix psutil.cpu_count() potentially returning None - Change psutil.cpu_count()+4 to (psutil.cpu_count() or 1)+4 - Prevents TypeError on systems where cpu_count() returns None 4. Fix typo: oriignal_is_vlm_text -> original_is_vlm_text --- unsloth/models/rl.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 22189f459c..9ea57e32d3 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -199,15 +199,15 @@ def PatchRL(FastLanguageModel): unwrap = "unwrap_model_for_generation" for trainer in trainers: try: - current_trainer = eval(f"trl.trainer.{trainer}") + current_trainer = getattr(trl.trainer, trainer) except: continue if hasattr(current_trainer, unwrap): try: - exec(f"trl.trainer.{trainer}.{unwrap} = unsloth_{unwrap}") + setattr(current_trainer, unwrap, unsloth_unwrap_model_for_generation) except: continue - exec(f"Trainer.prediction_step=unsloth_prediction_step") + Trainer.prediction_step = unsloth_prediction_step selective_log_softmax = RL_REPLACEMENTS["selective_log_softmax"] @@ -234,6 +234,7 @@ from transformers.training_args import ParallelMode # Also patches W&B since multiple runs must use wandb.finish() import functools from types import MethodType +from unsloth_zoo.gradient_checkpointing import reset_unsloth_gradient_checkpointing_buffers def prepare_for_training_mode(f): @functools.wraps(f) def wrapper(self, *args, **kwargs): @@ -244,6 +245,11 @@ def prepare_for_training_mode(f): # Return inference mode if hasattr(self, 'model') and hasattr(self.model, "for_inference"): self.model.for_inference() + # Reset gradient checkpointing buffers to free memory while staying ready for next run + try: + reset_unsloth_gradient_checkpointing_buffers() + except: + pass # Patch W&B to enable logging on future runs, otherwise it'll overwrite the first run try: import wandb @@ -817,7 +823,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): num_proc_check = ( "if dataset_num_proc is None:\n" " import psutil\n" - " dataset_num_proc = min(max(psutil.cpu_count()+4, 2), 64)\n" + " dataset_num_proc = min(max((psutil.cpu_count() or 1)+4, 2), 64)\n" " memory_gb_left = psutil.virtual_memory().available / (1024**3)\n" " if memory_gb_left <= 4: dataset_num_proc = 1 # Too risky, so set to 1\n" " elif memory_gb_left <= 6: dataset_num_proc = min(2, dataset_num_proc)\n" @@ -994,10 +1000,10 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): # Temporary patch _is_vlm to False # as of 0.22 it only exists in sfttrainer - oriignal_is_vlm_text = "self._is_vlm = True" + original_is_vlm_text = "self._is_vlm = True" new_is_vlm_text = "self._is_vlm = False" RLTrainer_source = RLTrainer_source.replace( - oriignal_is_vlm_text, new_is_vlm_text + original_is_vlm_text, new_is_vlm_text ) # Remove multiple doc strings From 08d619fca1a61774db4f2225b96723c9ffa1573c Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Sun, 4 Jan 2026 12:57:10 +0000 Subject: [PATCH 76/96] Handle older unsloth-zoo without reset_unsloth_gradient_checkpointing_buffers --- unsloth/models/rl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 9ea57e32d3..88aeeda8a1 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -234,7 +234,10 @@ from transformers.training_args import ParallelMode # Also patches W&B since multiple runs must use wandb.finish() import functools from types import MethodType -from unsloth_zoo.gradient_checkpointing import reset_unsloth_gradient_checkpointing_buffers +try: + from unsloth_zoo.gradient_checkpointing import reset_unsloth_gradient_checkpointing_buffers +except: + def reset_unsloth_gradient_checkpointing_buffers(): pass def prepare_for_training_mode(f): @functools.wraps(f) def wrapper(self, *args, **kwargs): From d31ec48a94482fa1265ebc670866acf700d8226a Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Sun, 4 Jan 2026 12:58:45 +0000 Subject: [PATCH 77/96] Fix psutil.cpu_count() potentially returning None in save.py --- unsloth/save.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/unsloth/save.py b/unsloth/save.py index 714df7682a..071e032c53 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -879,12 +879,12 @@ def install_llama_cpp_make_non_blocking(): IS_CMAKE = False if check == 0: # Uses old MAKE - n_jobs = max(int(psutil.cpu_count() * 1.5), 1) + n_jobs = max(int((psutil.cpu_count() or 1) * 1.5), 1) full_command = ["make", "all", "-j" + str(n_jobs), "-C", "llama.cpp"] IS_CMAKE = False else: # Uses new CMAKE - n_jobs = max(int(psutil.cpu_count()), 1) # Use less CPUs since 1.5x faster + n_jobs = max(int(psutil.cpu_count() or 1), 1) # Use less CPUs since 1.5x faster check = os.system( f"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}" ) @@ -994,13 +994,13 @@ def install_llama_cpp_old(version = -10): # Try using MAKE commands = [ "make clean -C llama.cpp", - f"make all -j{psutil.cpu_count()*2} -C llama.cpp", + f"make all -j{(psutil.cpu_count() or 1)*2} -C llama.cpp", ] if try_execute(commands) == "CMAKE": # Instead use CMAKE commands = [ f"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}", - f"cmake --build llama.cpp/build --config Release -j{psutil.cpu_count()*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}", + f"cmake --build llama.cpp/build --config Release -j{(psutil.cpu_count() or 1)*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}", "cp llama.cpp/build/bin/llama-* llama.cpp", "rm -rf llama.cpp/build", ] @@ -1040,14 +1040,14 @@ def install_llama_cpp_blocking(use_cuda = False): "make clean -C llama.cpp", # https://github.com/ggerganov/llama.cpp/issues/7062 # Weirdly GPU conversion for GGUF breaks?? - # f"{use_cuda} make all -j{psutil.cpu_count()*2} -C llama.cpp", - f"make all -j{psutil.cpu_count()*2} -C llama.cpp", + # f"{use_cuda} make all -j{(psutil.cpu_count() or 1)*2} -C llama.cpp", + f"make all -j{(psutil.cpu_count() or 1)*2} -C llama.cpp", ] if try_execute(commands) == "CMAKE": # Instead use CMAKE commands = [ f"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}", - f"cmake --build llama.cpp/build --config Release -j{psutil.cpu_count()*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}", + f"cmake --build llama.cpp/build --config Release -j{(psutil.cpu_count() or 1)*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}", "cp llama.cpp/build/bin/llama-* llama.cpp", "rm -rf llama.cpp/build", ] From 402e7d6285a3610350953c5f2d954fc5c8ddd3d2 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Sun, 4 Jan 2026 13:03:06 +0000 Subject: [PATCH 78/96] Respect user quantization_config --- unsloth/models/loader.py | 152 ++++++++++++++++++++++++++++++--------- unsloth/models/vision.py | 7 +- 2 files changed, 122 insertions(+), 37 deletions(-) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 247c72f43f..645c23d50b 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -151,8 +151,41 @@ class FastLanguageModel(FastLlamaModel): *args, **kwargs, ): + # Respect user-provided quantization_config (e.g. BitsAndBytesConfig) + quantization_config = kwargs.get("quantization_config", None) + if quantization_config is not None: + if getattr(quantization_config, "load_in_4bit", False): + load_in_4bit = True + load_in_8bit = False + if getattr(quantization_config, "load_in_8bit", False): + load_in_8bit = True + load_in_4bit = False + + load_in_4bit_kwargs = load_in_4bit + load_in_8bit_kwargs = load_in_8bit + if quantization_config is not None: + load_in_4bit_kwargs = False + load_in_8bit_kwargs = False + # Login to allow private models token = hf_login(token) + # Align dtype with bnb_4bit_compute_dtype if provided and dtype is unset. + if dtype is None and quantization_config is not None: + bnb_compute_dtype = None + if isinstance(quantization_config, dict): + if quantization_config.get("load_in_4bit", False): + bnb_compute_dtype = quantization_config.get( + "bnb_4bit_compute_dtype", None + ) + else: + if getattr(quantization_config, "load_in_4bit", False): + bnb_compute_dtype = getattr( + quantization_config, "bnb_4bit_compute_dtype", None + ) + if isinstance(bnb_compute_dtype, str): + bnb_compute_dtype = getattr(torch, bnb_compute_dtype, None) + if isinstance(bnb_compute_dtype, torch.dtype): + dtype = bnb_compute_dtype if load_in_8bit or full_finetuning or qat_scheme is not None: return FastModel.from_pretrained( model_name = model_name, @@ -546,7 +579,7 @@ class FastLanguageModel(FastLlamaModel): model_name = model_name, max_seq_length = max_seq_length, dtype = _get_dtype(dtype), - load_in_4bit = load_in_4bit, + load_in_4bit = load_in_4bit_kwargs, token = token, device_map = device_map, rope_scaling = rope_scaling, @@ -583,22 +616,30 @@ class FastLanguageModel(FastLlamaModel): ) if load_in_4bit: - # Fix up bitsandbytes config - compute_dtype = dtype_from_config(model.config) - quantization_config = { - # Sometimes compute_dtype is not a string!! - "bnb_4bit_compute_dtype": compute_dtype, - "bnb_4bit_quant_type": "nf4", - "bnb_4bit_use_double_quant": True, - "llm_int8_enable_fp32_cpu_offload": False, - "llm_int8_has_fp16_weight": False, - "llm_int8_skip_modules": None, - "llm_int8_threshold": 6.0, - "load_in_4bit": True, - "load_in_8bit": False, - "quant_method": "bitsandbytes", - } - model.config.update({"quantization_config": quantization_config}) + # Fix up bitsandbytes config, but respect user-provided quantization_config + if quantization_config is None: + compute_dtype = dtype_from_config(model.config) + quantization_config = { + # Sometimes compute_dtype is not a string!! + "bnb_4bit_compute_dtype": compute_dtype, + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": True, + "llm_int8_enable_fp32_cpu_offload": False, + "llm_int8_has_fp16_weight": False, + "llm_int8_skip_modules": None, + "llm_int8_threshold": 6.0, + "load_in_4bit": True, + "load_in_8bit": False, + "quant_method": "bitsandbytes", + } + model.config.update({"quantization_config": quantization_config}) + else: + if hasattr(quantization_config, "to_dict"): + model.config.update( + {"quantization_config": quantization_config.to_dict()} + ) + elif isinstance(quantization_config, dict): + model.config.update({"quantization_config": quantization_config}) if load_in_fp8 != False: _tag_model_with_fp8_torchao_config(model, fp8_mode) @@ -690,12 +731,45 @@ class FastModel(FastBaseModel): *args, **kwargs, ): + # Respect user-provided quantization_config (e.g. BitsAndBytesConfig) + quantization_config = kwargs.get("quantization_config", None) + if quantization_config is not None: + if getattr(quantization_config, "load_in_4bit", False): + load_in_4bit = True + load_in_8bit = False + if getattr(quantization_config, "load_in_8bit", False): + load_in_8bit = True + load_in_4bit = False + + load_in_4bit_kwargs = load_in_4bit + load_in_8bit_kwargs = load_in_8bit + if quantization_config is not None: + load_in_4bit_kwargs = False + load_in_8bit_kwargs = False + # Login to allow private models token = hf_login(token) if whisper_language is not None: assert type(whisper_language) is str if whisper_task is not None: assert type(whisper_task) is str + # Align dtype with bnb_4bit_compute_dtype if provided and dtype is unset. + if dtype is None and quantization_config is not None: + bnb_compute_dtype = None + if isinstance(quantization_config, dict): + if quantization_config.get("load_in_4bit", False): + bnb_compute_dtype = quantization_config.get( + "bnb_4bit_compute_dtype", None + ) + else: + if getattr(quantization_config, "load_in_4bit", False): + bnb_compute_dtype = getattr( + quantization_config, "bnb_4bit_compute_dtype", None + ) + if isinstance(bnb_compute_dtype, str): + bnb_compute_dtype = getattr(torch, bnb_compute_dtype, None) + if isinstance(bnb_compute_dtype, torch.dtype): + dtype = bnb_compute_dtype SUPPORTS_BFLOAT16 = is_bfloat16_supported() if dtype is None: dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16 @@ -1173,8 +1247,8 @@ class FastModel(FastBaseModel): model_name = model_name, max_seq_length = max_seq_length, dtype = _get_dtype(dtype), - load_in_4bit = load_in_4bit, - load_in_8bit = load_in_8bit, + load_in_4bit = load_in_4bit_kwargs, + load_in_8bit = load_in_8bit_kwargs, load_in_16bit = load_in_16bit, full_finetuning = full_finetuning, token = token, @@ -1220,22 +1294,30 @@ class FastModel(FastBaseModel): ) if load_in_4bit: - # Fix up bitsandbytes config - compute_dtype = dtype_from_config(model.config) - quantization_config = { - # Sometimes compute_dtype is not a string!! - "bnb_4bit_compute_dtype": compute_dtype, - "bnb_4bit_quant_type": "nf4", - "bnb_4bit_use_double_quant": True, - "llm_int8_enable_fp32_cpu_offload": False, - "llm_int8_has_fp16_weight": False, - "llm_int8_skip_modules": None, - "llm_int8_threshold": 6.0, - "load_in_4bit": True, - "load_in_8bit": False, - "quant_method": "bitsandbytes", - } - model.config.update({"quantization_config": quantization_config}) + # Fix up bitsandbytes config, but respect user-provided quantization_config + if quantization_config is None: + compute_dtype = dtype_from_config(model.config) + quantization_config = { + # Sometimes compute_dtype is not a string!! + "bnb_4bit_compute_dtype": compute_dtype, + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": True, + "llm_int8_enable_fp32_cpu_offload": False, + "llm_int8_has_fp16_weight": False, + "llm_int8_skip_modules": None, + "llm_int8_threshold": 6.0, + "load_in_4bit": True, + "load_in_8bit": False, + "quant_method": "bitsandbytes", + } + model.config.update({"quantization_config": quantization_config}) + else: + if hasattr(quantization_config, "to_dict"): + model.config.update( + {"quantization_config": quantization_config.to_dict()} + ) + elif isinstance(quantization_config, dict): + model.config.update({"quantization_config": quantization_config}) if load_in_fp8 != False: _tag_model_with_fp8_torchao_config(model, fp8_mode) diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 6c5356e0b9..6de942d7d2 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -529,6 +529,7 @@ class FastBaseModel: del kwargs["attn_implementation"] bnb_config = None + user_quantization_config = kwargs.get("quantization_config", None) if full_finetuning and (load_in_4bit or load_in_8bit): print( "Unsloth: You selected full finetuning support, but 4bit / 8bit is enabled - disabling LoRA / QLoRA." @@ -596,7 +597,8 @@ class FastBaseModel: ): pass else: - kwargs["quantization_config"] = bnb_config + if user_quantization_config is None: + kwargs["quantization_config"] = bnb_config else: if auto_config is None: auto_config = AutoConfig.from_pretrained( @@ -641,7 +643,8 @@ class FastBaseModel: ) except: pass - kwargs["quantization_config"] = quantization_config + if user_quantization_config is None: + kwargs["quantization_config"] = quantization_config # Check if using forced float32 - we load it in bfloat16, then cast to float16! torch_dtype = dtype From bfa225b00c5faba087bae3f75d8aa55803cacb52 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Sun, 4 Jan 2026 13:14:03 +0000 Subject: [PATCH 79/96] Handle dict quantization_config flags --- unsloth/models/loader.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 645c23d50b..4488ad9a07 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -154,10 +154,16 @@ class FastLanguageModel(FastLlamaModel): # Respect user-provided quantization_config (e.g. BitsAndBytesConfig) quantization_config = kwargs.get("quantization_config", None) if quantization_config is not None: - if getattr(quantization_config, "load_in_4bit", False): + if isinstance(quantization_config, dict): + q_load_in_4bit = quantization_config.get("load_in_4bit", False) + q_load_in_8bit = quantization_config.get("load_in_8bit", False) + else: + q_load_in_4bit = getattr(quantization_config, "load_in_4bit", False) + q_load_in_8bit = getattr(quantization_config, "load_in_8bit", False) + if q_load_in_4bit: load_in_4bit = True load_in_8bit = False - if getattr(quantization_config, "load_in_8bit", False): + if q_load_in_8bit: load_in_8bit = True load_in_4bit = False @@ -734,10 +740,16 @@ class FastModel(FastBaseModel): # Respect user-provided quantization_config (e.g. BitsAndBytesConfig) quantization_config = kwargs.get("quantization_config", None) if quantization_config is not None: - if getattr(quantization_config, "load_in_4bit", False): + if isinstance(quantization_config, dict): + q_load_in_4bit = quantization_config.get("load_in_4bit", False) + q_load_in_8bit = quantization_config.get("load_in_8bit", False) + else: + q_load_in_4bit = getattr(quantization_config, "load_in_4bit", False) + q_load_in_8bit = getattr(quantization_config, "load_in_8bit", False) + if q_load_in_4bit: load_in_4bit = True load_in_8bit = False - if getattr(quantization_config, "load_in_8bit", False): + if q_load_in_8bit: load_in_8bit = True load_in_4bit = False From e22ca346cf687cc79393f0a122ce4c15ba965e63 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Sun, 4 Jan 2026 13:18:15 +0000 Subject: [PATCH 80/96] Keep 4bit flag for fast_inference --- unsloth/models/loader.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 4488ad9a07..eb3b21e206 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -167,12 +167,6 @@ class FastLanguageModel(FastLlamaModel): load_in_8bit = True load_in_4bit = False - load_in_4bit_kwargs = load_in_4bit - load_in_8bit_kwargs = load_in_8bit - if quantization_config is not None: - load_in_4bit_kwargs = False - load_in_8bit_kwargs = False - # Login to allow private models token = hf_login(token) # Align dtype with bnb_4bit_compute_dtype if provided and dtype is unset. @@ -581,6 +575,12 @@ class FastLanguageModel(FastLlamaModel): if fast_inference: fast_inference, model_name = fast_inference_setup(model_name, model_config) + load_in_4bit_kwargs = load_in_4bit + load_in_8bit_kwargs = load_in_8bit + if quantization_config is not None and not fast_inference: + load_in_4bit_kwargs = False + load_in_8bit_kwargs = False + model, tokenizer = dispatch_model.from_pretrained( model_name = model_name, max_seq_length = max_seq_length, @@ -753,12 +753,6 @@ class FastModel(FastBaseModel): load_in_8bit = True load_in_4bit = False - load_in_4bit_kwargs = load_in_4bit - load_in_8bit_kwargs = load_in_8bit - if quantization_config is not None: - load_in_4bit_kwargs = False - load_in_8bit_kwargs = False - # Login to allow private models token = hf_login(token) if whisper_language is not None: @@ -1255,6 +1249,12 @@ class FastModel(FastBaseModel): if auto_model is None: auto_model = AutoModelForVision2Seq if is_vlm else AutoModelForCausalLM + load_in_4bit_kwargs = load_in_4bit + load_in_8bit_kwargs = load_in_8bit + if quantization_config is not None and not fast_inference: + load_in_4bit_kwargs = False + load_in_8bit_kwargs = False + model, tokenizer = FastBaseModel.from_pretrained( model_name = model_name, max_seq_length = max_seq_length, From e63c2744ec0762e8688de50a57a5391216f06a53 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 4 Jan 2026 06:12:44 -0800 Subject: [PATCH 81/96] Versioning --- pyproject.toml | 4 ++-- unsloth/__init__.py | 2 +- unsloth/models/_utils.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 20e3fd847f..e7b84f3c8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ huggingfacenotorch = [ ] huggingface = [ "unsloth[huggingfacenotorch]", - "unsloth_zoo>=2025.12.8", + "unsloth_zoo>=2026.1.1", "torchvision", "unsloth[triton]", ] @@ -523,7 +523,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3 ; ('linux' in sys_platform)", ] colab-new = [ - "unsloth_zoo>=2025.12.8", + "unsloth_zoo>=2026.1.1", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,<=4.57.3", diff --git a/unsloth/__init__.py b/unsloth/__init__.py index c74b248a83..d9633e8ec1 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -79,7 +79,7 @@ from importlib.metadata import PackageNotFoundError # Check for unsloth_zoo try: unsloth_zoo_version = importlib_version("unsloth_zoo") - if Version(unsloth_zoo_version) < Version("2025.12.4"): + if Version(unsloth_zoo_version) < Version("2026.1.1"): print( "Unsloth: Please update Unsloth and Unsloth-Zoo to the latest version!\n" "Do this via `pip install --upgrade --force-reinstall --no-cache-dir --no-deps unsloth unsloth_zoo`" diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 545ba4794a..5952d4af0c 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.12.10" +__version__ = "2026.1.1" __all__ = [ "SUPPORTS_BFLOAT16", From b5addbc936933ad3ca682a0cc2f3eeececfba321 Mon Sep 17 00:00:00 2001 From: Kaitao Yang Date: Sun, 4 Jan 2026 09:21:44 -0800 Subject: [PATCH 82/96] remove unused variable BlockDiagonalCausalMask --- unsloth/utils/attention_dispatch.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py index 0e5f3c1951..a7620549be 100644 --- a/unsloth/utils/attention_dispatch.py +++ b/unsloth/utils/attention_dispatch.py @@ -32,9 +32,6 @@ from ..utils.packing import ( if HAS_FLASH_ATTENTION: from flash_attn import flash_attn_func, flash_attn_varlen_func HAS_XFORMERS = xformers is not None -BlockDiagonalCausalMask = None -if HAS_XFORMERS: - BlockDiagonalCausalMask = xformers.attn_bias.BlockDiagonalCausalMask SDPA_HAS_GQA = "enable_gqa" in (scaled_dot_product_attention.__doc__ or "") FLASH_VARLEN = "flash_varlen" From 6c6d0dfef1bce443b2030dd46f04e9fcbb981dff Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 05:02:53 +0000 Subject: [PATCH 83/96] Fix vLLM PDL bug on Blackwell GPUs (B200/B100) vLLM's LoRA Triton kernels use tl.extra.cuda.gdc_wait() for PDL optimization on SM90+ GPUs. This fails on SM100 (Blackwell) during CUDA graph capture because Triton's pipeliner cannot handle gdc_wait in complex kernels. This fix: - Detects SM100 GPUs and applies the workaround automatically - Sets TRITON_DISABLE_PDL=1 environment variable - Monkey-patches supports_pdl to return False in lora_expand_op and lora_shrink_op - Checks GitHub issue #30872 status (with 3s timeout) to auto-disable the workaround once the upstream fix is merged - Includes quick internet connectivity check (0.5s) to avoid delays when offline Fixes the error: 'tt.elementwise_inline_asm' op pipeliner doesn't know how to predicate this op LLVM ERROR: Fatal pipeliner error See: https://github.com/vllm-project/vllm/issues/30872 --- unsloth/__init__.py | 3 + unsloth/import_fixes.py | 121 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) diff --git a/unsloth/__init__.py b/unsloth/__init__.py index d9633e8ec1..86fb00fe0e 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -126,6 +126,7 @@ from .import_fixes import ( fix_xformers_performance_issue, fix_vllm_aimv2_issue, fix_vllm_guided_decoding_params, + fix_vllm_pdl_blackwell, ignore_logger_messages, patch_ipykernel_hf_xet, patch_trackio, @@ -138,6 +139,7 @@ from .import_fixes import ( fix_xformers_performance_issue() fix_vllm_aimv2_issue() fix_vllm_guided_decoding_params() +fix_vllm_pdl_blackwell() ignore_logger_messages() patch_ipykernel_hf_xet() patch_trackio() @@ -149,6 +151,7 @@ fix_executorch() del fix_xformers_performance_issue del fix_vllm_aimv2_issue del fix_vllm_guided_decoding_params +del fix_vllm_pdl_blackwell del ignore_logger_messages del patch_ipykernel_hf_xet del patch_trackio diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index bb6996a3e3..91ba35e21e 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -556,3 +556,124 @@ def fix_huggingface_hub(): huggingface_hub.is_offline_mode = ( lambda: huggingface_hub.constants.HF_HUB_OFFLINE ) + + +def fix_vllm_pdl_blackwell(): + """ + Fix vLLM PDL (Programmatic Dependent Launch) bug on Blackwell GPUs (SM100). + + The issue: vLLM's LoRA Triton kernels use tl.extra.cuda.gdc_wait() for PDL + optimization on SM90+ GPUs. This fails on SM100 (B200/B100) during CUDA graph + capture because Triton's pipeliner can't handle gdc_wait in complex kernels. + + See: https://github.com/vllm-project/vllm/issues/30872 + """ + if importlib.util.find_spec("vllm") is None: + return + + # Check if we have a CUDA GPU + try: + import torch + if not torch.cuda.is_available(): + return + major, minor = torch.cuda.get_device_capability() + except Exception: + return + + # Only SM100 (Blackwell) is affected - SM90 (Hopper) works fine + if major != 10: + return + + gpu_name = torch.cuda.get_device_name() + + # Check if vLLM has the PDL-related modules before doing internet check + try: + has_expand_op = importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_expand_op") is not None + except (ModuleNotFoundError, ValueError): + has_expand_op = False + try: + has_shrink_op = importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_shrink_op") is not None + except (ModuleNotFoundError, ValueError): + has_shrink_op = False + if not has_expand_op and not has_shrink_op: + # Old vLLM version without PDL support - just set env var to be safe + os.environ["TRITON_DISABLE_PDL"] = "1" + logger.info( + f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM{major}{minor} ({gpu_name}) - " + f"vLLM PDL modules not found" + ) + return + + # Check if GitHub issue is closed (fix merged upstream) + issue_closed = False + try: + import socket + import urllib.request + import json as json_module + + # Quick internet connectivity check (0.5s timeout) + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(0.5) + try: + sock.connect(("api.github.com", 443)) + has_internet = True + except (socket.timeout, OSError): + has_internet = False + finally: + sock.close() + + if has_internet: + api_url = "https://api.github.com/repos/vllm-project/vllm/issues/30872" + req = urllib.request.Request( + api_url, + headers={ + "User-Agent": "Unsloth-PDL-Fix", + "Accept": "application/vnd.github.v3+json", + } + ) + with urllib.request.urlopen(req, timeout=3) as response: + data = json_module.loads(response.read().decode()) + issue_closed = data.get("state") == "closed" + except Exception: + # If we can't check, assume issue is still open (apply fix to be safe) + pass + + if issue_closed: + logger.info( + f"Unsloth: SM{major}{minor} ({gpu_name}) detected but PDL issue #30872 " + f"is closed - skipping PDL fix" + ) + return + + # Apply the PDL fix + os.environ["TRITON_DISABLE_PDL"] = "1" + + def fake_supports_pdl(device=None): + return False + + patched = [] + + try: + import vllm.lora.ops.triton_ops.lora_expand_op as expand_op + expand_op.supports_pdl = fake_supports_pdl + patched.append("lora_expand_op") + except (ImportError, ModuleNotFoundError, AttributeError): + pass + + try: + import vllm.lora.ops.triton_ops.lora_shrink_op as shrink_op + shrink_op.supports_pdl = fake_supports_pdl + patched.append("lora_shrink_op") + except (ImportError, ModuleNotFoundError, AttributeError): + pass + + if patched: + logger.info( + f"Unsloth: Applied PDL fix for SM{major}{minor} ({gpu_name}) - " + f"patched: {', '.join(patched)}" + ) + else: + # Just set the env var - vLLM might be an older version without supports_pdl + logger.info( + f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM{major}{minor} ({gpu_name})" + ) From efe949c941a752e3f1872b84d2fc9b1b54e9358c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 05:03:28 +0000 Subject: [PATCH 84/96] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/import_fixes.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 91ba35e21e..e8c5e16df4 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -574,6 +574,7 @@ def fix_vllm_pdl_blackwell(): # Check if we have a CUDA GPU try: import torch + if not torch.cuda.is_available(): return major, minor = torch.cuda.get_device_capability() @@ -588,11 +589,17 @@ def fix_vllm_pdl_blackwell(): # Check if vLLM has the PDL-related modules before doing internet check try: - has_expand_op = importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_expand_op") is not None + has_expand_op = ( + importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_expand_op") + is not None + ) except (ModuleNotFoundError, ValueError): has_expand_op = False try: - has_shrink_op = importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_shrink_op") is not None + has_shrink_op = ( + importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_shrink_op") + is not None + ) except (ModuleNotFoundError, ValueError): has_shrink_op = False if not has_expand_op and not has_shrink_op: @@ -626,12 +633,12 @@ def fix_vllm_pdl_blackwell(): api_url = "https://api.github.com/repos/vllm-project/vllm/issues/30872" req = urllib.request.Request( api_url, - headers={ + headers = { "User-Agent": "Unsloth-PDL-Fix", "Accept": "application/vnd.github.v3+json", - } + }, ) - with urllib.request.urlopen(req, timeout=3) as response: + with urllib.request.urlopen(req, timeout = 3) as response: data = json_module.loads(response.read().decode()) issue_closed = data.get("state") == "closed" except Exception: @@ -648,13 +655,14 @@ def fix_vllm_pdl_blackwell(): # Apply the PDL fix os.environ["TRITON_DISABLE_PDL"] = "1" - def fake_supports_pdl(device=None): + def fake_supports_pdl(device = None): return False patched = [] try: import vllm.lora.ops.triton_ops.lora_expand_op as expand_op + expand_op.supports_pdl = fake_supports_pdl patched.append("lora_expand_op") except (ImportError, ModuleNotFoundError, AttributeError): @@ -662,6 +670,7 @@ def fix_vllm_pdl_blackwell(): try: import vllm.lora.ops.triton_ops.lora_shrink_op as shrink_op + shrink_op.supports_pdl = fake_supports_pdl patched.append("lora_shrink_op") except (ImportError, ModuleNotFoundError, AttributeError): From 36c9a841eb959f279b0170041f86e43c7421b514 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 05:03:56 +0000 Subject: [PATCH 85/96] Sync chat_template from tokenizer to vLLM When using base models with custom chat templates applied after loading, vLLM's internal tokenizer may not have the chat_template set. This causes issues during RL training with vLLM inference. This fix syncs the chat_template from the processing_class (the tokenizer you loaded and configured) to vLLM's internal tokenizer during trainer initialization, but only if vLLM's tokenizer does not already have one set. --- unsloth/models/rl.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 88aeeda8a1..20dafaaaa4 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -694,6 +694,20 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): ) RLTrainer_post += training_check + # Sync chat_template from processing_class to vLLM's tokenizer + # This fixes base models that have custom chat templates applied after loading + if "model" in call_args: + vllm_chat_template_sync = ( + "if hasattr(self, 'llm') and self.llm is not None and hasattr(self.llm, 'get_tokenizer'):\n" + " _vllm_tok = self.llm.get_tokenizer()\n" + " _pc = getattr(self, 'processing_class', None)\n" + " if _pc is not None and getattr(_pc, 'chat_template', None) is not None:\n" + " if _vllm_tok.chat_template is None:\n" + " _vllm_tok.chat_template = _pc.chat_template\n" + "pass\n" + ) + RLTrainer_post += vllm_chat_template_sync + # Edit optional metrics other_metrics_processor = "" if trainer_file in RL_METRICS_CHANGES: From fbdb3b524e93ea99d8845696e9c40ce64bef349d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 05:10:24 +0000 Subject: [PATCH 86/96] Add tokenizer fallback for chat_template sync --- unsloth/models/rl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 20dafaaaa4..b75ae383db 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -700,7 +700,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): vllm_chat_template_sync = ( "if hasattr(self, 'llm') and self.llm is not None and hasattr(self.llm, 'get_tokenizer'):\n" " _vllm_tok = self.llm.get_tokenizer()\n" - " _pc = getattr(self, 'processing_class', None)\n" + " _pc = getattr(self, 'processing_class', None) or getattr(self, 'tokenizer', None)\n" " if _pc is not None and getattr(_pc, 'chat_template', None) is not None:\n" " if _vllm_tok.chat_template is None:\n" " _vllm_tok.chat_template = _pc.chat_template\n" From 227c31f0caf3cb30a06a77c0a1b010cc7e09007d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 05:24:52 +0000 Subject: [PATCH 87/96] Address review feedback: refactor and scan all GPUs - Add _spec_exists helper function to reduce duplication - Scan all GPUs for SM100 instead of just device 0 - Use loop for module patching to improve maintainability --- unsloth/import_fixes.py | 85 ++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 44 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index e8c5e16df4..7d368d027a 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -571,42 +571,44 @@ def fix_vllm_pdl_blackwell(): if importlib.util.find_spec("vllm") is None: return - # Check if we have a CUDA GPU + # Check if any CUDA GPU is SM100 (Blackwell) try: import torch if not torch.cuda.is_available(): return - major, minor = torch.cuda.get_device_capability() + + # Scan all GPUs for SM100 - fix applies globally via env var and monkey-patch + has_sm100 = False + sm100_gpu_name = None + for i in range(torch.cuda.device_count()): + major, minor = torch.cuda.get_device_capability(i) + if major == 10: + has_sm100 = True + sm100_gpu_name = torch.cuda.get_device_name(i) + break + + if not has_sm100: + return except Exception: return - # Only SM100 (Blackwell) is affected - SM90 (Hopper) works fine - if major != 10: - return - - gpu_name = torch.cuda.get_device_name() + # Helper to check if module spec exists + def _spec_exists(name): + try: + return importlib.util.find_spec(name) is not None + except (ModuleNotFoundError, ValueError): + return False # Check if vLLM has the PDL-related modules before doing internet check - try: - has_expand_op = ( - importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_expand_op") - is not None - ) - except (ModuleNotFoundError, ValueError): - has_expand_op = False - try: - has_shrink_op = ( - importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_shrink_op") - is not None - ) - except (ModuleNotFoundError, ValueError): - has_shrink_op = False + has_expand_op = _spec_exists("vllm.lora.ops.triton_ops.lora_expand_op") + has_shrink_op = _spec_exists("vllm.lora.ops.triton_ops.lora_shrink_op") + if not has_expand_op and not has_shrink_op: # Old vLLM version without PDL support - just set env var to be safe os.environ["TRITON_DISABLE_PDL"] = "1" logger.info( - f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM{major}{minor} ({gpu_name}) - " + f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM100 ({sm100_gpu_name}) - " f"vLLM PDL modules not found" ) return @@ -633,12 +635,12 @@ def fix_vllm_pdl_blackwell(): api_url = "https://api.github.com/repos/vllm-project/vllm/issues/30872" req = urllib.request.Request( api_url, - headers = { + headers={ "User-Agent": "Unsloth-PDL-Fix", "Accept": "application/vnd.github.v3+json", }, ) - with urllib.request.urlopen(req, timeout = 3) as response: + with urllib.request.urlopen(req, timeout=3) as response: data = json_module.loads(response.read().decode()) issue_closed = data.get("state") == "closed" except Exception: @@ -647,7 +649,7 @@ def fix_vllm_pdl_blackwell(): if issue_closed: logger.info( - f"Unsloth: SM{major}{minor} ({gpu_name}) detected but PDL issue #30872 " + f"Unsloth: SM100 ({sm100_gpu_name}) detected but PDL issue #30872 " f"is closed - skipping PDL fix" ) return @@ -655,34 +657,29 @@ def fix_vllm_pdl_blackwell(): # Apply the PDL fix os.environ["TRITON_DISABLE_PDL"] = "1" - def fake_supports_pdl(device = None): + def fake_supports_pdl(device=None): return False patched = [] - - try: - import vllm.lora.ops.triton_ops.lora_expand_op as expand_op - - expand_op.supports_pdl = fake_supports_pdl - patched.append("lora_expand_op") - except (ImportError, ModuleNotFoundError, AttributeError): - pass - - try: - import vllm.lora.ops.triton_ops.lora_shrink_op as shrink_op - - shrink_op.supports_pdl = fake_supports_pdl - patched.append("lora_shrink_op") - except (ImportError, ModuleNotFoundError, AttributeError): - pass + modules_to_patch = { + "lora_expand_op": "vllm.lora.ops.triton_ops.lora_expand_op", + "lora_shrink_op": "vllm.lora.ops.triton_ops.lora_shrink_op", + } + for name, path in modules_to_patch.items(): + try: + module = importlib.import_module(path) + module.supports_pdl = fake_supports_pdl + patched.append(name) + except (ImportError, ModuleNotFoundError, AttributeError): + pass if patched: logger.info( - f"Unsloth: Applied PDL fix for SM{major}{minor} ({gpu_name}) - " + f"Unsloth: Applied PDL fix for SM100 ({sm100_gpu_name}) - " f"patched: {', '.join(patched)}" ) else: # Just set the env var - vLLM might be an older version without supports_pdl logger.info( - f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM{major}{minor} ({gpu_name})" + f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM100 ({sm100_gpu_name})" ) From eac1f6b0101ce12ae3d630160f9e3d8593a70779 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 05:24:59 +0000 Subject: [PATCH 88/96] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/import_fixes.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 7d368d027a..77693d4cf3 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -635,12 +635,12 @@ def fix_vllm_pdl_blackwell(): api_url = "https://api.github.com/repos/vllm-project/vllm/issues/30872" req = urllib.request.Request( api_url, - headers={ + headers = { "User-Agent": "Unsloth-PDL-Fix", "Accept": "application/vnd.github.v3+json", }, ) - with urllib.request.urlopen(req, timeout=3) as response: + with urllib.request.urlopen(req, timeout = 3) as response: data = json_module.loads(response.read().decode()) issue_closed = data.get("state") == "closed" except Exception: @@ -657,7 +657,7 @@ def fix_vllm_pdl_blackwell(): # Apply the PDL fix os.environ["TRITON_DISABLE_PDL"] = "1" - def fake_supports_pdl(device=None): + def fake_supports_pdl(device = None): return False patched = [] @@ -680,6 +680,4 @@ def fix_vllm_pdl_blackwell(): ) else: # Just set the env var - vLLM might be an older version without supports_pdl - logger.info( - f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM100 ({sm100_gpu_name})" - ) + logger.info(f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM100 ({sm100_gpu_name})") From ba548ff8c22b055c5acaa02eb0d67c2e238413ce Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 05:25:53 +0000 Subject: [PATCH 89/96] Combine nested if statements for clarity --- unsloth/models/rl.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index b75ae383db..e1ecd6df2f 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -701,9 +701,8 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): "if hasattr(self, 'llm') and self.llm is not None and hasattr(self.llm, 'get_tokenizer'):\n" " _vllm_tok = self.llm.get_tokenizer()\n" " _pc = getattr(self, 'processing_class', None) or getattr(self, 'tokenizer', None)\n" - " if _pc is not None and getattr(_pc, 'chat_template', None) is not None:\n" - " if _vllm_tok.chat_template is None:\n" - " _vllm_tok.chat_template = _pc.chat_template\n" + " if _pc is not None and getattr(_pc, 'chat_template', None) is not None and _vllm_tok.chat_template is None:\n" + " _vllm_tok.chat_template = _pc.chat_template\n" "pass\n" ) RLTrainer_post += vllm_chat_template_sync From 35219633ab161f062a826f84b037ac18f6390e7e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 06:53:42 +0000 Subject: [PATCH 90/96] Fix PDL patch: target utils.py source module and clear lru_cache - Patch vllm.lora.ops.triton_ops.utils directly where supports_pdl is defined - Clear lru_cache before patching to prevent stale cached results - Add fused_moe_lora_op to consumer modules list - Use *args, **kwargs in fake function for compatibility --- unsloth/import_fixes.py | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 77693d4cf3..469674b29e 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -601,10 +601,11 @@ def fix_vllm_pdl_blackwell(): return False # Check if vLLM has the PDL-related modules before doing internet check + has_utils = _spec_exists("vllm.lora.ops.triton_ops.utils") has_expand_op = _spec_exists("vllm.lora.ops.triton_ops.lora_expand_op") has_shrink_op = _spec_exists("vllm.lora.ops.triton_ops.lora_shrink_op") - if not has_expand_op and not has_shrink_op: + if not has_utils and not has_expand_op and not has_shrink_op: # Old vLLM version without PDL support - just set env var to be safe os.environ["TRITON_DISABLE_PDL"] = "1" logger.info( @@ -657,19 +658,39 @@ def fix_vllm_pdl_blackwell(): # Apply the PDL fix os.environ["TRITON_DISABLE_PDL"] = "1" - def fake_supports_pdl(device = None): + def fake_supports_pdl(*args, **kwargs): return False patched = [] - modules_to_patch = { + + # First, patch the source module (utils.py) where supports_pdl is defined. + # This is critical because supports_pdl uses @lru_cache - we must clear the + # cache to prevent stale cached results from the original function. + try: + utils_module = importlib.import_module("vllm.lora.ops.triton_ops.utils") + if hasattr(utils_module, "supports_pdl"): + original_fn = utils_module.supports_pdl + if hasattr(original_fn, "cache_clear"): + original_fn.cache_clear() + utils_module.supports_pdl = fake_supports_pdl + patched.append("utils") + except (ImportError, ModuleNotFoundError, AttributeError): + pass + + # Also patch the consumer modules that import supports_pdl from utils. + # This ensures the patched function is used even if the module was already + # imported before this fix runs. + consumer_modules = { "lora_expand_op": "vllm.lora.ops.triton_ops.lora_expand_op", "lora_shrink_op": "vllm.lora.ops.triton_ops.lora_shrink_op", + "fused_moe_lora_op": "vllm.lora.ops.triton_ops.fused_moe_lora_op", } - for name, path in modules_to_patch.items(): + for name, path in consumer_modules.items(): try: module = importlib.import_module(path) - module.supports_pdl = fake_supports_pdl - patched.append(name) + if hasattr(module, "supports_pdl"): + module.supports_pdl = fake_supports_pdl + patched.append(name) except (ImportError, ModuleNotFoundError, AttributeError): pass From aff2dc9061faba13bae73035ac47b780a21c60fb Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 10:02:11 +0000 Subject: [PATCH 91/96] Add None check for vLLM tokenizer - Check _vllm_tok is not None before accessing attributes - Use getattr for safer chat_template access --- unsloth/models/rl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index e1ecd6df2f..fd0c69bb0e 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -701,7 +701,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): "if hasattr(self, 'llm') and self.llm is not None and hasattr(self.llm, 'get_tokenizer'):\n" " _vllm_tok = self.llm.get_tokenizer()\n" " _pc = getattr(self, 'processing_class', None) or getattr(self, 'tokenizer', None)\n" - " if _pc is not None and getattr(_pc, 'chat_template', None) is not None and _vllm_tok.chat_template is None:\n" + " if _vllm_tok is not None and _pc is not None and getattr(_pc, 'chat_template', None) is not None and getattr(_vllm_tok, 'chat_template', None) is None:\n" " _vllm_tok.chat_template = _pc.chat_template\n" "pass\n" ) From 6bf555a34c17820f3931f2e9ebfe8c9fb4fee229 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 12:32:16 +0000 Subject: [PATCH 92/96] Remove unnecessary PDL module existence check Old vLLM versions without PDL modules don't need the fix. The patching code already handles missing modules gracefully. --- unsloth/import_fixes.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 469674b29e..ef647ec65a 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -593,27 +593,6 @@ def fix_vllm_pdl_blackwell(): except Exception: return - # Helper to check if module spec exists - def _spec_exists(name): - try: - return importlib.util.find_spec(name) is not None - except (ModuleNotFoundError, ValueError): - return False - - # Check if vLLM has the PDL-related modules before doing internet check - has_utils = _spec_exists("vllm.lora.ops.triton_ops.utils") - has_expand_op = _spec_exists("vllm.lora.ops.triton_ops.lora_expand_op") - has_shrink_op = _spec_exists("vllm.lora.ops.triton_ops.lora_shrink_op") - - if not has_utils and not has_expand_op and not has_shrink_op: - # Old vLLM version without PDL support - just set env var to be safe - os.environ["TRITON_DISABLE_PDL"] = "1" - logger.info( - f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM100 ({sm100_gpu_name}) - " - f"vLLM PDL modules not found" - ) - return - # Check if GitHub issue is closed (fix merged upstream) issue_closed = False try: From 9b6d536e0ee0ccc8eb2ddb9bb733044b182fd13e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 12:34:32 +0000 Subject: [PATCH 93/96] Keep PDL module check but remove unnecessary env var setting The check skips the GitHub API call for old vLLM versions. No need to set TRITON_DISABLE_PDL for versions without PDL support. --- unsloth/import_fixes.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index ef647ec65a..e8c4a2f665 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -593,6 +593,22 @@ def fix_vllm_pdl_blackwell(): except Exception: return + # Helper to check if module spec exists + def _spec_exists(name): + try: + return importlib.util.find_spec(name) is not None + except (ModuleNotFoundError, ValueError): + return False + + # Check if vLLM has the PDL-related modules before doing internet check + has_utils = _spec_exists("vllm.lora.ops.triton_ops.utils") + has_expand_op = _spec_exists("vllm.lora.ops.triton_ops.lora_expand_op") + has_shrink_op = _spec_exists("vllm.lora.ops.triton_ops.lora_shrink_op") + + if not has_utils and not has_expand_op and not has_shrink_op: + # Old vLLM version without PDL support - nothing to patch + return + # Check if GitHub issue is closed (fix merged upstream) issue_closed = False try: From 9ced3523aa73b9161ec87b2f9c2a62c3d0378b7a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 13:15:17 +0000 Subject: [PATCH 94/96] Replace GitHub API check with vLLM version check for PDL fix The GitHub issue check had issues: 1. Network latency on import 2. Issue being closed does not mean the fix is in the installed vLLM version Now skip the PDL workaround if vLLM version > 0.13.2, which is when the upstream fix is expected to be included. --- unsloth/import_fixes.py | 43 +++++++---------------------------------- 1 file changed, 7 insertions(+), 36 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index e8c4a2f665..86a504b7b2 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -609,47 +609,18 @@ def fix_vllm_pdl_blackwell(): # Old vLLM version without PDL support - nothing to patch return - # Check if GitHub issue is closed (fix merged upstream) - issue_closed = False + # Check if vLLM version includes the fix (expected in versions > 0.13.2) try: - import socket - import urllib.request - import json as json_module - - # Quick internet connectivity check (0.5s timeout) - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(0.5) - try: - sock.connect(("api.github.com", 443)) - has_internet = True - except (socket.timeout, OSError): - has_internet = False - finally: - sock.close() - - if has_internet: - api_url = "https://api.github.com/repos/vllm-project/vllm/issues/30872" - req = urllib.request.Request( - api_url, - headers = { - "User-Agent": "Unsloth-PDL-Fix", - "Accept": "application/vnd.github.v3+json", - }, + vllm_version = Version(importlib_version("vllm")) + if vllm_version > Version("0.13.2"): + logger.info( + f"Unsloth: SM100 ({sm100_gpu_name}) detected but vLLM {vllm_version} " + f"should include PDL fix - skipping workaround" ) - with urllib.request.urlopen(req, timeout = 3) as response: - data = json_module.loads(response.read().decode()) - issue_closed = data.get("state") == "closed" + return except Exception: - # If we can't check, assume issue is still open (apply fix to be safe) pass - if issue_closed: - logger.info( - f"Unsloth: SM100 ({sm100_gpu_name}) detected but PDL issue #30872 " - f"is closed - skipping PDL fix" - ) - return - # Apply the PDL fix os.environ["TRITON_DISABLE_PDL"] = "1" From cb42ce8dae17efa1f52c09104703da180e3eadd3 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 13:19:37 +0000 Subject: [PATCH 95/96] Address review feedback: add constant and debug logging --- unsloth/import_fixes.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 86a504b7b2..958173213d 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -609,17 +609,18 @@ def fix_vllm_pdl_blackwell(): # Old vLLM version without PDL support - nothing to patch return - # Check if vLLM version includes the fix (expected in versions > 0.13.2) + # Check if vLLM version includes the fix + VLLM_PDL_FIX_VERSION = "0.13.2" try: vllm_version = Version(importlib_version("vllm")) - if vllm_version > Version("0.13.2"): + if vllm_version > Version(VLLM_PDL_FIX_VERSION): logger.info( f"Unsloth: SM100 ({sm100_gpu_name}) detected but vLLM {vllm_version} " f"should include PDL fix - skipping workaround" ) return - except Exception: - pass + except Exception as e: + logger.debug(f"Unsloth: vLLM version check failed ({e}), applying PDL workaround.") # Apply the PDL fix os.environ["TRITON_DISABLE_PDL"] = "1" From c612bfe3a3472df436e951aee2930185d334b3c4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 13:19:44 +0000 Subject: [PATCH 96/96] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/import_fixes.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 958173213d..1e05e462e9 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -620,7 +620,9 @@ def fix_vllm_pdl_blackwell(): ) return except Exception as e: - logger.debug(f"Unsloth: vLLM version check failed ({e}), applying PDL workaround.") + logger.debug( + f"Unsloth: vLLM version check failed ({e}), applying PDL workaround." + ) # Apply the PDL fix os.environ["TRITON_DISABLE_PDL"] = "1"