From 0b42a72e44e42e3854b51d2571fa179caf5ed7fc Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 20 Aug 2025 07:39:43 -0700 Subject: [PATCH] Bug fixes (#3195) * Fix mamba * Update loader.py * Update vision.py * Update loader.py * Filter vLLM standby logs (#3131) * filter vLLM standby logs * safeguard standby logger patch * Update unsloth/models/_utils.py * Update unsloth/models/_utils.py * Update unsloth/models/_utils.py --------- Co-authored-by: Daniel Han * Update loader.py * Add scaler * Update llama.py * Update _utils.py * Versioning * GPT OSS fix * GPT OSS fix * Update loader.py * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Update vision.py * Update llama.py * Update llama.py * Update llama.py * Versioning * Update mapper.py * Update vision.py * Update vision.py * Update vision.py * Upcast norms * Update loader.py * Update vision.py * Upcast layernorms * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update save.py * Update rl.py * Update pyproject.toml * Update rl.py * Update rl_replacements.py * Update rl.py * Update rl.py * Update rl.py * Update _utils.py * Update __init__.py * Torch 2.8 * Update rl_replacements.py * Update loader.py * UNSLOTH_ENABLE_CCE * Fix * Update loader.py * Update loader.py * Update __init__.py * Update __init__.py * Update __init__.py * Update __init__.py * Import fixes * Update loader.py * Fix aimv2 issue * Update loader.py * Update import_fixes.py * Update import_fixes.py * Update loader.py * Update loader.py * Update loader.py * Upgrade * Update loader.py * Update loader.py * Update loader.py * Update loader.py --------- Co-authored-by: Datta Nimmaturi --- pyproject.toml | 4 +- unsloth/__init__.py | 71 ++++++----------------- unsloth/import_fixes.py | 119 +++++++++++++++++++++++++++++++++++++++ unsloth/models/_utils.py | 2 +- unsloth/models/loader.py | 11 ++-- 5 files changed, 145 insertions(+), 62 deletions(-) create mode 100644 unsloth/import_fixes.py diff --git a/pyproject.toml b/pyproject.toml index c4c3ebe6f5..83b75b0a00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ triton = [ ] huggingface = [ - "unsloth_zoo>=2025.8.7", + "unsloth_zoo>=2025.8.8", "packaging", "tyro", "transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1", @@ -453,7 +453,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3", ] colab-new = [ - "unsloth_zoo>=2025.8.7", + "unsloth_zoo>=2025.8.8", "packaging", "tyro", "transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1", diff --git a/unsloth/__init__.py b/unsloth/__init__.py index a43dc4f70f..a6ea8f4c9f 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -17,6 +17,10 @@ from packaging.version import Version import os, re, subprocess, inspect import numpy as np +# Fix some issues before importing other packages +from .import_fixes import fix_message_factory_issue +fix_message_factory_issue(); del fix_message_factory_issue; + # Check if modules that need patching are already imported critical_modules = ['trl', 'transformers', 'peft'] already_imported = [mod for mod in critical_modules if mod in sys.modules] @@ -53,6 +57,7 @@ os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python" # Log Unsloth is being used os.environ["UNSLOTH_IS_PRESENT"] = "1" +# Try importing PyTorch and check version try: import torch except ModuleNotFoundError: @@ -93,7 +98,7 @@ if DEVICE_TYPE == "cuda" and os.environ.get("UNSLOTH_VLLM_STANDBY", "0")=="0": # We support Pytorch 2 # Fixes https://github.com/unslothai/unsloth/issues/38 -torch_version = str(torch.__version__).split(".") +torch_version = str(re.match(r"[0-9\.]{3,}", str(torch.__version__)).group(0)).split(".") major_torch, minor_torch = torch_version[0], torch_version[1] major_torch, minor_torch = int(major_torch), int(minor_torch) if (major_torch < 2): @@ -104,35 +109,21 @@ elif (major_torch == 2) and (minor_torch < 2): del os.environ["PYTORCH_CUDA_ALLOC_CONF"] pass -# Fix Xformers performance issues since 0.0.25 +# CCE fails on Torch 2.8 and above +# OutOfResources: out of resource: shared memory, Required: 98304, Hardware limit: 65536. Reducing block sizes or `num_stages` +if (major_torch >= 2 and minor_torch >= 8) or (major_torch > 2): + os.environ["UNSLOTH_ENABLE_CCE"] = "0" +pass + +# Fix other issues import importlib.util from pathlib import Path from importlib.metadata import version as importlib_version from packaging.version import Version -try: - xformers_version = importlib_version("xformers") - if Version(xformers_version) < Version("0.0.29"): - xformers_location = importlib.util.find_spec("xformers").origin - xformers_location = os.path.split(xformers_location)[0] - cutlass = Path(xformers_location) / "ops" / "fmha" / "cutlass.py" - - if cutlass.exists(): - with open(cutlass, "r+", encoding = "utf-8") as f: - text = f.read() - # See https://github.com/facebookresearch/xformers/issues/1176#issuecomment-2545829591 - if "num_splits_key=-1," in text: - text = text.replace("num_splits_key=-1,", "num_splits_key=None,") - f.seek(0) - f.write(text) - f.truncate() - print("Unsloth: Patching Xformers to fix some performance issues.") - pass - pass - pass - pass -except: - pass -pass +from .import_fixes import fix_xformers_performance_issue +fix_xformers_performance_issue(); del fix_xformers_performance_issue; +from .import_fixes import fix_vllm_aimv2_issue +fix_vllm_aimv2_issue(); del fix_vllm_aimv2_issue; # Torch 2.4 has including_emulation if DEVICE_TYPE == "cuda": @@ -154,7 +145,6 @@ elif DEVICE_TYPE == "xpu": SUPPORTS_BFLOAT16 = torch.xpu.is_bf16_supported() pass - # For Gradio HF Spaces? # if "SPACE_AUTHOR_NAME" not in os.environ and "SPACE_REPO_NAME" not in os.environ: import triton @@ -222,7 +212,7 @@ elif DEVICE_TYPE == "xpu": # Check for unsloth_zoo try: unsloth_zoo_version = importlib_version("unsloth_zoo") - if Version(unsloth_zoo_version) < Version("2025.8.1"): + if Version(unsloth_zoo_version) < Version("2025.8.8"): print( "Unsloth: Please update Unsloth and Unsloth-Zoo to the latest version!\n"\ "Do this via `pip install --upgrade --force-reinstall --no-cache-dir --no-deps unsloth unsloth_zoo`" @@ -240,31 +230,6 @@ except: raise ImportError("Unsloth: Please install unsloth_zoo via `pip install unsloth_zoo`") pass -try: - # Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype' - # MUST do this at the start primarily due to tensorflow causing issues - import google.protobuf.message_factory - class MessageFactory: - def CreatePrototype(self, *args, **kwargs): return - def GetMessages(self, *args, **kwargs): return - def GetPrototype(self, *args, **kwargs): return - if not hasattr(google.protobuf.message_factory, "MessageFactory"): - google.protobuf.message_factory.MessageFactory = MessageFactory - elif hasattr(google.protobuf.message_factory, "MessageFactory") and \ - not hasattr(google.protobuf.message_factory.MessageFactory, "GetPrototype") and \ - not hasattr(google.protobuf.message_factory, "GetMessageClass"): - google.protobuf.message_factory.MessageFactory = MessageFactory - elif hasattr(google.protobuf.message_factory, "MessageFactory") and \ - not hasattr(google.protobuf.message_factory.MessageFactory, "GetPrototype") and \ - hasattr(google.protobuf.message_factory, "GetMessageClass"): - GetMessageClass = google.protobuf.message_factory.GetMessageClass - def GetPrototype(self, descriptor): - return GetMessageClass(descriptor) - google.protobuf.message_factory.MessageFactory.GetPrototype = GetPrototype - pass -except: - pass - from .models import * from .models import __version__ from .save import * diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py new file mode 100644 index 0000000000..a07f9970f8 --- /dev/null +++ b/unsloth/import_fixes.py @@ -0,0 +1,119 @@ +# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import importlib.util +from pathlib import Path +from importlib.metadata import version as importlib_version +from packaging.version import Version +UNSLOTH_ENABLE_LOGGING = os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") == "1" + +# Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype' +# MUST do this at the start primarily due to tensorflow causing issues +def fix_message_factory_issue(): + try: + import google.protobuf.message_factory + class MessageFactory: + def CreatePrototype(self, *args, **kwargs): return + def GetMessages(self, *args, **kwargs): return + def GetPrototype(self, *args, **kwargs): return + if not hasattr(google.protobuf.message_factory, "MessageFactory"): + if UNSLOTH_ENABLE_LOGGING: + print("Unsloth: Patching protobuf.MessageFactory as it doesn't exist") + google.protobuf.message_factory.MessageFactory = MessageFactory + elif hasattr(google.protobuf.message_factory, "MessageFactory") and \ + not hasattr(google.protobuf.message_factory.MessageFactory, "GetPrototype") and \ + not hasattr(google.protobuf.message_factory, "GetMessageClass"): + google.protobuf.message_factory.MessageFactory = MessageFactory + if UNSLOTH_ENABLE_LOGGING: + print("Unsloth: Patching protobuf.MessageFactory as it doesn't exist") + elif hasattr(google.protobuf.message_factory, "MessageFactory") and \ + not hasattr(google.protobuf.message_factory.MessageFactory, "GetPrototype") and \ + hasattr(google.protobuf.message_factory, "GetMessageClass"): + GetMessageClass = google.protobuf.message_factory.GetMessageClass + def GetPrototype(self, descriptor): + return GetMessageClass(descriptor) + google.protobuf.message_factory.MessageFactory.GetPrototype = GetPrototype + if UNSLOTH_ENABLE_LOGGING: + print("Unsloth: Patching protobuf.MessageFactory.GetPrototype") + pass + except: + pass +pass + +# Fix Xformers performance issues since 0.0.25 +def fix_xformers_performance_issue(): + if importlib.util.find_spec("xformers") is None: return + xformers_version = importlib_version("xformers") + if Version(xformers_version) < Version("0.0.29"): + xformers_location = importlib.util.find_spec("xformers").origin + xformers_location = os.path.split(xformers_location)[0] + cutlass = Path(xformers_location) / "ops" / "fmha" / "cutlass.py" + try: + if cutlass.exists(): + with open(cutlass, "r+", encoding = "utf-8") as f: + text = f.read() + # See https://github.com/facebookresearch/xformers/issues/1176#issuecomment-2545829591 + if "num_splits_key=-1," in text: + text = text.replace( + "num_splits_key=-1,", + "num_splits_key=None,", + ) + f.seek(0) + f.write(text) + f.truncate() + if UNSLOTH_ENABLE_LOGGING: + print("Unsloth: Patching Xformers to fix some performance issues.") + except Exception as e: + if UNSLOTH_ENABLE_LOGGING: + print(f"Unsloth: Failed patching Xformers with error = {str(e)}") +pass + +# ValueError: 'aimv2' is already used by a Transformers config, pick another name. +def fix_vllm_aimv2_issue(): + if importlib.util.find_spec("vllm") is None: return + vllm_version = importlib_version("vllm") + if Version(vllm_version) < Version("0.10.1"): + vllm_version = importlib.util.find_spec("vllm").origin + vllm_version = os.path.split(vllm_version)[0] + ovis_config = Path(vllm_version) / "transformers_utils" / "configs" / "ovis.py" + try: + if ovis_config.exists(): + with open(ovis_config, "r+", encoding = "utf-8") as f: + text = f.read() + # See https://github.com/vllm-project/vllm-ascend/issues/2046 + if 'AutoConfig.register("aimv2", AIMv2Config)' in text: + text = text.replace( + 'AutoConfig.register("aimv2", AIMv2Config)', + '', + ) + text = text.replace( + '''backbone_config.pop('model_type') + backbone_config = AutoConfig.for_model(model_type, + **backbone_config)''', + '''if model_type != "aimv2": + backbone_config.pop('model_type') + backbone_config = AutoConfig.for_model(model_type, **backbone_config) + else: + backbone_config = AIMv2Config(**backbone_config)''' + ) + f.seek(0) + f.write(text) + f.truncate() + if UNSLOTH_ENABLE_LOGGING: + print("Unsloth: Patching vLLM to fix `'aimv2' is already used by a Transformers config, pick another name.`") + except Exception as e: + if UNSLOTH_ENABLE_LOGGING: + print(f"Unsloth: Failed patching vLLM with error = {str(e)}") +pass diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 85f1a9a960..fde776a5e6 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.8.8" +__version__ = "2025.8.9" __all__ = [ "SUPPORTS_BFLOAT16", diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index fae6ae0770..3aed8654f8 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -618,9 +618,6 @@ class FastModel(FastBaseModel): "os.environ['TRITON_F32_DEFAULT'] = 'ieee'" elif "gpt-oss" in lowered_model_name: os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" - # CCE fails on Tesla T4 - # OutOfResources: out of resource: shared memory, Required: 98304, Hardware limit: 65536. Reducing block sizes or `num_stages` - os.environ["UNSLOTH_ENABLE_CCE"] = "0" if not load_in_4bit: # Only upcast MoE biases for MXFP4, not BnB # Set norms to float32 since anyways they get upcasted to float32 @@ -639,11 +636,13 @@ class FastModel(FastBaseModel): # Set down projection compute dtype to be float32 for float16 machines # Set norms to float32 since anyways they get upcasted to float32 os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = \ - "all;None;None;"\ - "if 'down_projs' in name and hasattr(module, 'weight') and "\ - "torch.amax(dequantize_module_weight(module)) >= 1024:"\ + "torch.float16;torch.bfloat16;torch.float16;"\ + "if ('down_projs' in name) and hasattr(module, 'weight') and "\ + "torch.amax(dequantize_module_weight(module)) >= 0:"\ "module._pre_set_compute_dtype = torch.float32\n"\ ""\ + "if ('mlp.router' in name) and hasattr(module, 'weight'):"\ + "module._pre_set_compute_dtype = torch.float32\n"\ ";" # Set norms to float32 since anyways they get upcasted to float32 os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1"