Bug fixes (#3195)
* Fix mamba * Update loader.py * Update vision.py * Update loader.py * Filter vLLM standby logs (#3131) * filter vLLM standby logs * safeguard standby logger patch * Update unsloth/models/_utils.py * Update unsloth/models/_utils.py * Update unsloth/models/_utils.py --------- Co-authored-by: Daniel Han <danielhanchen@gmail.com> * Update loader.py * Add scaler * Update llama.py * Update _utils.py * Versioning * GPT OSS fix * GPT OSS fix * Update loader.py * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Update vision.py * Update llama.py * Update llama.py * Update llama.py * Versioning * Update mapper.py * Update vision.py * Update vision.py * Update vision.py * Upcast norms * Update loader.py * Update vision.py * Upcast layernorms * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update save.py * Update rl.py * Update pyproject.toml * Update rl.py * Update rl_replacements.py * Update rl.py * Update rl.py * Update rl.py * Update _utils.py * Update __init__.py * Torch 2.8 * Update rl_replacements.py * Update loader.py * UNSLOTH_ENABLE_CCE * Fix * Update loader.py * Update loader.py * Update __init__.py * Update __init__.py * Update __init__.py * Update __init__.py * Import fixes * Update loader.py * Fix aimv2 issue * Update loader.py * Update import_fixes.py * Update import_fixes.py * Update loader.py * Update loader.py * Update loader.py * Upgrade * Update loader.py * Update loader.py * Update loader.py * Update loader.py --------- Co-authored-by: Datta Nimmaturi <venkatadattasainimmaturi@gmail.com>
This commit is contained in:
Daniel Han
GitHub
parent
dfb936743d
commit
2525052e4f
5 changed files with 145 additions and 62 deletions
|
|
@ -37,7 +37,7 @@ triton = [
|
|||
]
|
||||
|
||||
huggingface = [
|
||||
"unsloth_zoo>=2025.8.7",
|
||||
"unsloth_zoo>=2025.8.8",
|
||||
"packaging",
|
||||
"tyro",
|
||||
"transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1",
|
||||
|
|
@ -453,7 +453,7 @@ colab-ampere-torch220 = [
|
|||
"flash-attn>=2.6.3",
|
||||
]
|
||||
colab-new = [
|
||||
"unsloth_zoo>=2025.8.7",
|
||||
"unsloth_zoo>=2025.8.8",
|
||||
"packaging",
|
||||
"tyro",
|
||||
"transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1",
|
||||
|
|
|
|||
|
|
@ -17,6 +17,10 @@ from packaging.version import Version
|
|||
import os, re, subprocess, inspect
|
||||
import numpy as np
|
||||
|
||||
# Fix some issues before importing other packages
|
||||
from .import_fixes import fix_message_factory_issue
|
||||
fix_message_factory_issue(); del fix_message_factory_issue;
|
||||
|
||||
# Check if modules that need patching are already imported
|
||||
critical_modules = ['trl', 'transformers', 'peft']
|
||||
already_imported = [mod for mod in critical_modules if mod in sys.modules]
|
||||
|
|
@ -53,6 +57,7 @@ os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
|
|||
# Log Unsloth is being used
|
||||
os.environ["UNSLOTH_IS_PRESENT"] = "1"
|
||||
|
||||
# Try importing PyTorch and check version
|
||||
try:
|
||||
import torch
|
||||
except ModuleNotFoundError:
|
||||
|
|
@ -93,7 +98,7 @@ if DEVICE_TYPE == "cuda" and os.environ.get("UNSLOTH_VLLM_STANDBY", "0")=="0":
|
|||
|
||||
# We support Pytorch 2
|
||||
# Fixes https://github.com/unslothai/unsloth/issues/38
|
||||
torch_version = str(torch.__version__).split(".")
|
||||
torch_version = str(re.match(r"[0-9\.]{3,}", str(torch.__version__)).group(0)).split(".")
|
||||
major_torch, minor_torch = torch_version[0], torch_version[1]
|
||||
major_torch, minor_torch = int(major_torch), int(minor_torch)
|
||||
if (major_torch < 2):
|
||||
|
|
@ -104,35 +109,21 @@ elif (major_torch == 2) and (minor_torch < 2):
|
|||
del os.environ["PYTORCH_CUDA_ALLOC_CONF"]
|
||||
pass
|
||||
|
||||
# Fix Xformers performance issues since 0.0.25
|
||||
# CCE fails on Torch 2.8 and above
|
||||
# OutOfResources: out of resource: shared memory, Required: 98304, Hardware limit: 65536. Reducing block sizes or `num_stages`
|
||||
if (major_torch >= 2 and minor_torch >= 8) or (major_torch > 2):
|
||||
os.environ["UNSLOTH_ENABLE_CCE"] = "0"
|
||||
pass
|
||||
|
||||
# Fix other issues
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from importlib.metadata import version as importlib_version
|
||||
from packaging.version import Version
|
||||
try:
|
||||
xformers_version = importlib_version("xformers")
|
||||
if Version(xformers_version) < Version("0.0.29"):
|
||||
xformers_location = importlib.util.find_spec("xformers").origin
|
||||
xformers_location = os.path.split(xformers_location)[0]
|
||||
cutlass = Path(xformers_location) / "ops" / "fmha" / "cutlass.py"
|
||||
|
||||
if cutlass.exists():
|
||||
with open(cutlass, "r+", encoding = "utf-8") as f:
|
||||
text = f.read()
|
||||
# See https://github.com/facebookresearch/xformers/issues/1176#issuecomment-2545829591
|
||||
if "num_splits_key=-1," in text:
|
||||
text = text.replace("num_splits_key=-1,", "num_splits_key=None,")
|
||||
f.seek(0)
|
||||
f.write(text)
|
||||
f.truncate()
|
||||
print("Unsloth: Patching Xformers to fix some performance issues.")
|
||||
pass
|
||||
pass
|
||||
pass
|
||||
pass
|
||||
except:
|
||||
pass
|
||||
pass
|
||||
from .import_fixes import fix_xformers_performance_issue
|
||||
fix_xformers_performance_issue(); del fix_xformers_performance_issue;
|
||||
from .import_fixes import fix_vllm_aimv2_issue
|
||||
fix_vllm_aimv2_issue(); del fix_vllm_aimv2_issue;
|
||||
|
||||
# Torch 2.4 has including_emulation
|
||||
if DEVICE_TYPE == "cuda":
|
||||
|
|
@ -154,7 +145,6 @@ elif DEVICE_TYPE == "xpu":
|
|||
SUPPORTS_BFLOAT16 = torch.xpu.is_bf16_supported()
|
||||
pass
|
||||
|
||||
|
||||
# For Gradio HF Spaces?
|
||||
# if "SPACE_AUTHOR_NAME" not in os.environ and "SPACE_REPO_NAME" not in os.environ:
|
||||
import triton
|
||||
|
|
@ -222,7 +212,7 @@ elif DEVICE_TYPE == "xpu":
|
|||
# Check for unsloth_zoo
|
||||
try:
|
||||
unsloth_zoo_version = importlib_version("unsloth_zoo")
|
||||
if Version(unsloth_zoo_version) < Version("2025.8.1"):
|
||||
if Version(unsloth_zoo_version) < Version("2025.8.8"):
|
||||
print(
|
||||
"Unsloth: Please update Unsloth and Unsloth-Zoo to the latest version!\n"\
|
||||
"Do this via `pip install --upgrade --force-reinstall --no-cache-dir --no-deps unsloth unsloth_zoo`"
|
||||
|
|
@ -240,31 +230,6 @@ except:
|
|||
raise ImportError("Unsloth: Please install unsloth_zoo via `pip install unsloth_zoo`")
|
||||
pass
|
||||
|
||||
try:
|
||||
# Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype'
|
||||
# MUST do this at the start primarily due to tensorflow causing issues
|
||||
import google.protobuf.message_factory
|
||||
class MessageFactory:
|
||||
def CreatePrototype(self, *args, **kwargs): return
|
||||
def GetMessages(self, *args, **kwargs): return
|
||||
def GetPrototype(self, *args, **kwargs): return
|
||||
if not hasattr(google.protobuf.message_factory, "MessageFactory"):
|
||||
google.protobuf.message_factory.MessageFactory = MessageFactory
|
||||
elif hasattr(google.protobuf.message_factory, "MessageFactory") and \
|
||||
not hasattr(google.protobuf.message_factory.MessageFactory, "GetPrototype") and \
|
||||
not hasattr(google.protobuf.message_factory, "GetMessageClass"):
|
||||
google.protobuf.message_factory.MessageFactory = MessageFactory
|
||||
elif hasattr(google.protobuf.message_factory, "MessageFactory") and \
|
||||
not hasattr(google.protobuf.message_factory.MessageFactory, "GetPrototype") and \
|
||||
hasattr(google.protobuf.message_factory, "GetMessageClass"):
|
||||
GetMessageClass = google.protobuf.message_factory.GetMessageClass
|
||||
def GetPrototype(self, descriptor):
|
||||
return GetMessageClass(descriptor)
|
||||
google.protobuf.message_factory.MessageFactory.GetPrototype = GetPrototype
|
||||
pass
|
||||
except:
|
||||
pass
|
||||
|
||||
from .models import *
|
||||
from .models import __version__
|
||||
from .save import *
|
||||
|
|
|
|||
119
unsloth/import_fixes.py
Normal file
119
unsloth/import_fixes.py
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from importlib.metadata import version as importlib_version
|
||||
from packaging.version import Version
|
||||
UNSLOTH_ENABLE_LOGGING = os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") == "1"
|
||||
|
||||
# Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype'
|
||||
# MUST do this at the start primarily due to tensorflow causing issues
|
||||
def fix_message_factory_issue():
|
||||
try:
|
||||
import google.protobuf.message_factory
|
||||
class MessageFactory:
|
||||
def CreatePrototype(self, *args, **kwargs): return
|
||||
def GetMessages(self, *args, **kwargs): return
|
||||
def GetPrototype(self, *args, **kwargs): return
|
||||
if not hasattr(google.protobuf.message_factory, "MessageFactory"):
|
||||
if UNSLOTH_ENABLE_LOGGING:
|
||||
print("Unsloth: Patching protobuf.MessageFactory as it doesn't exist")
|
||||
google.protobuf.message_factory.MessageFactory = MessageFactory
|
||||
elif hasattr(google.protobuf.message_factory, "MessageFactory") and \
|
||||
not hasattr(google.protobuf.message_factory.MessageFactory, "GetPrototype") and \
|
||||
not hasattr(google.protobuf.message_factory, "GetMessageClass"):
|
||||
google.protobuf.message_factory.MessageFactory = MessageFactory
|
||||
if UNSLOTH_ENABLE_LOGGING:
|
||||
print("Unsloth: Patching protobuf.MessageFactory as it doesn't exist")
|
||||
elif hasattr(google.protobuf.message_factory, "MessageFactory") and \
|
||||
not hasattr(google.protobuf.message_factory.MessageFactory, "GetPrototype") and \
|
||||
hasattr(google.protobuf.message_factory, "GetMessageClass"):
|
||||
GetMessageClass = google.protobuf.message_factory.GetMessageClass
|
||||
def GetPrototype(self, descriptor):
|
||||
return GetMessageClass(descriptor)
|
||||
google.protobuf.message_factory.MessageFactory.GetPrototype = GetPrototype
|
||||
if UNSLOTH_ENABLE_LOGGING:
|
||||
print("Unsloth: Patching protobuf.MessageFactory.GetPrototype")
|
||||
pass
|
||||
except:
|
||||
pass
|
||||
pass
|
||||
|
||||
# Fix Xformers performance issues since 0.0.25
|
||||
def fix_xformers_performance_issue():
|
||||
if importlib.util.find_spec("xformers") is None: return
|
||||
xformers_version = importlib_version("xformers")
|
||||
if Version(xformers_version) < Version("0.0.29"):
|
||||
xformers_location = importlib.util.find_spec("xformers").origin
|
||||
xformers_location = os.path.split(xformers_location)[0]
|
||||
cutlass = Path(xformers_location) / "ops" / "fmha" / "cutlass.py"
|
||||
try:
|
||||
if cutlass.exists():
|
||||
with open(cutlass, "r+", encoding = "utf-8") as f:
|
||||
text = f.read()
|
||||
# See https://github.com/facebookresearch/xformers/issues/1176#issuecomment-2545829591
|
||||
if "num_splits_key=-1," in text:
|
||||
text = text.replace(
|
||||
"num_splits_key=-1,",
|
||||
"num_splits_key=None,",
|
||||
)
|
||||
f.seek(0)
|
||||
f.write(text)
|
||||
f.truncate()
|
||||
if UNSLOTH_ENABLE_LOGGING:
|
||||
print("Unsloth: Patching Xformers to fix some performance issues.")
|
||||
except Exception as e:
|
||||
if UNSLOTH_ENABLE_LOGGING:
|
||||
print(f"Unsloth: Failed patching Xformers with error = {str(e)}")
|
||||
pass
|
||||
|
||||
# ValueError: 'aimv2' is already used by a Transformers config, pick another name.
|
||||
def fix_vllm_aimv2_issue():
|
||||
if importlib.util.find_spec("vllm") is None: return
|
||||
vllm_version = importlib_version("vllm")
|
||||
if Version(vllm_version) < Version("0.10.1"):
|
||||
vllm_version = importlib.util.find_spec("vllm").origin
|
||||
vllm_version = os.path.split(vllm_version)[0]
|
||||
ovis_config = Path(vllm_version) / "transformers_utils" / "configs" / "ovis.py"
|
||||
try:
|
||||
if ovis_config.exists():
|
||||
with open(ovis_config, "r+", encoding = "utf-8") as f:
|
||||
text = f.read()
|
||||
# See https://github.com/vllm-project/vllm-ascend/issues/2046
|
||||
if 'AutoConfig.register("aimv2", AIMv2Config)' in text:
|
||||
text = text.replace(
|
||||
'AutoConfig.register("aimv2", AIMv2Config)',
|
||||
'',
|
||||
)
|
||||
text = text.replace(
|
||||
'''backbone_config.pop('model_type')
|
||||
backbone_config = AutoConfig.for_model(model_type,
|
||||
**backbone_config)''',
|
||||
'''if model_type != "aimv2":
|
||||
backbone_config.pop('model_type')
|
||||
backbone_config = AutoConfig.for_model(model_type, **backbone_config)
|
||||
else:
|
||||
backbone_config = AIMv2Config(**backbone_config)'''
|
||||
)
|
||||
f.seek(0)
|
||||
f.write(text)
|
||||
f.truncate()
|
||||
if UNSLOTH_ENABLE_LOGGING:
|
||||
print("Unsloth: Patching vLLM to fix `'aimv2' is already used by a Transformers config, pick another name.`")
|
||||
except Exception as e:
|
||||
if UNSLOTH_ENABLE_LOGGING:
|
||||
print(f"Unsloth: Failed patching vLLM with error = {str(e)}")
|
||||
pass
|
||||
|
|
@ -12,7 +12,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
__version__ = "2025.8.8"
|
||||
__version__ = "2025.8.9"
|
||||
|
||||
__all__ = [
|
||||
"SUPPORTS_BFLOAT16",
|
||||
|
|
|
|||
|
|
@ -618,9 +618,6 @@ class FastModel(FastBaseModel):
|
|||
"os.environ['TRITON_F32_DEFAULT'] = 'ieee'"
|
||||
elif "gpt-oss" in lowered_model_name:
|
||||
os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1"
|
||||
# CCE fails on Tesla T4
|
||||
# OutOfResources: out of resource: shared memory, Required: 98304, Hardware limit: 65536. Reducing block sizes or `num_stages`
|
||||
os.environ["UNSLOTH_ENABLE_CCE"] = "0"
|
||||
if not load_in_4bit:
|
||||
# Only upcast MoE biases for MXFP4, not BnB
|
||||
# Set norms to float32 since anyways they get upcasted to float32
|
||||
|
|
@ -639,11 +636,13 @@ class FastModel(FastBaseModel):
|
|||
# Set down projection compute dtype to be float32 for float16 machines
|
||||
# Set norms to float32 since anyways they get upcasted to float32
|
||||
os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = \
|
||||
"all;None;None;"\
|
||||
"if 'down_projs' in name and hasattr(module, 'weight') and "\
|
||||
"torch.amax(dequantize_module_weight(module)) >= 1024:"\
|
||||
"torch.float16;torch.bfloat16;torch.float16;"\
|
||||
"if ('down_projs' in name) and hasattr(module, 'weight') and "\
|
||||
"torch.amax(dequantize_module_weight(module)) >= 0:"\
|
||||
"module._pre_set_compute_dtype = torch.float32\n"\
|
||||
""\
|
||||
"if ('mlp.router' in name) and hasattr(module, 'weight'):"\
|
||||
"module._pre_set_compute_dtype = torch.float32\n"\
|
||||
";"
|
||||
# Set norms to float32 since anyways they get upcasted to float32
|
||||
os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue