diff --git a/pyproject.toml b/pyproject.toml index 8cdd90ea99..f7a8cf91c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,9 @@ windows=[ "bitsandbytes>=0.45.5 ; platform_system == 'Windows'", "xformers>=0.0.22.post7 ; platform_system == 'Windows'", ] +base = [ + "unsloth[huggingface]", +] cu118only = [ "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and platform_system == 'Linux'", "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and platform_system == 'Linux'", diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 2e88e213e6..a036970b40 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -76,6 +76,7 @@ platform_system = platform_system() import numpy as np import contextlib import re +import functools import warnings, subprocess, re, inspect, psutil, os, math from unsloth_zoo.utils import Version from unsloth import DEVICE_TYPE, DEVICE_COUNT @@ -422,6 +423,7 @@ HAS_FLASH_ATTENTION_SOFTCAPPING = False if DEVICE_TYPE == "cuda": major_version, minor_version = torch.cuda.get_device_capability() + torch.cuda.get_device_capability = functools.cache(torch.cuda.get_device_capability) if major_version >= 8: SUPPORTS_BFLOAT16 = True @@ -586,7 +588,6 @@ UNSLOTH_COMPILE_DEBUG = os.environ.get("UNSLOTH_COMPILE_DEBUG", UNSLOTH_COMPILE_MAXIMUM = os.environ.get("UNSLOTH_COMPILE_MAXIMUM", "0") == "1" UNSLOTH_COMPILE_IGNORE_ERRORS = os.environ.get("UNSLOTH_COMPILE_IGNORE_ERRORS", "1") == "1" # Just remove max_autotune_gemm warning -import functools from torch._inductor.runtime.hints import DeviceProperties @functools.lru_cache(None) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index b3218d2498..efbf4ed37c 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -592,6 +592,14 @@ class FastModel(FastBaseModel): "os.environ['TRITON_F32_DEFAULT'] = 'ieee';" elif "gpt-oss" in lowered_model_name: os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" + if not load_in_4bit: + # Only upcast MoE biases for MXFP4, not BnB + os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = \ + "all;None;None;"\ + "x = 'gate_up_proj_bias'\n"\ + "if hasattr(module, x): setattr(module, x, torch.nn.Parameter(getattr(module, x).to(torch.float32)) if isinstance(getattr(module, x), torch.nn.Parameter) else getattr(module, x).to(torch.float32))\n"\ + "x = 'down_proj_bias'\n"\ + "if hasattr(module, x): setattr(module, x, torch.nn.Parameter(getattr(module, x).to(torch.float32)) if isinstance(getattr(module, x), torch.nn.Parameter) else getattr(module, x).to(torch.float32))\n;" else: for check_model_name in DISABLE_COMPILE_MODEL_NAMES: if check_model_name in lowered_model_name: diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index bba8f982da..8f5ffbb509 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -924,12 +924,12 @@ __INT_TO_FLOAT_MAPPER = \ "unsloth/gpt-oss-20b-unsloth-bnb-4bit" : ( "unsloth/gpt-oss-20b", "openai/gpt-oss-20b", - "unsloth/gpt-oss-20b-bnb-4bit", + "unsloth/gpt-oss-20b-unsloth-bnb-4bit", ), "unsloth/gpt-oss-120b-unsloth-bnb-4bit" : ( "unsloth/gpt-oss-120b", "openai/gpt-oss-120b", - "unsloth/gpt-oss-120b-bnb-4bit", + "unsloth/gpt-oss-120b-unsloth-bnb-4bit", ), } diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 623f263127..38606c4ff9 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -365,8 +365,10 @@ class FastBaseModel: allow_float16_runs = (checker == "float16" and dtype == torch.float16) if allow_all_runs or allow_float16_runs: - dtype = eval(_dtype) - bnb_compute_dtype = eval(_bnb_compute_dtype) + if eval(_dtype) is not None: + dtype = eval(_dtype) + if eval(_bnb_compute_dtype) is not None: + bnb_compute_dtype = eval(_bnb_compute_dtype) correct_dtype = bnb_compute_dtype custom_datatype = _custom_datatype # Execute code as well @@ -421,6 +423,10 @@ class FastBaseModel: os.environ["UNSLOTH_ENABLE_FULL_FINETUNING"] = "0" pass + # Fix AttributeError: 'BitsAndBytesConfig' object has no attribute 'get_loading_attributes' + if bnb_config is not None and not hasattr(bnb_config, "get_loading_attributes"): + bnb_config.get_loading_attributes = lambda *args, **kwargs: {} + # Cannot be None, since HF now checks for the config if load_in_4bit: kwargs["quantization_config"] = bnb_config @@ -429,23 +435,16 @@ class FastBaseModel: if do_forced_float32: torch_dtype = torch.bfloat16 raise_handler = RaiseUninitialized() - # MXFP4 -> BF16 GPT-OSS check - if "gpt-oss" in os.environ.get("UNSLOTH_MODEL_NAME", "") and \ - "quantization_config" not in kwargs: - - from unsloth_zoo.temporary_patches.gpt_oss import load_gpt_oss_MXFP4 - model = load_gpt_oss_MXFP4(model_name, torch_dtype) - else: - model = auto_model.from_pretrained( - model_name, - device_map = device_map, - torch_dtype = torch_dtype, - # quantization_config = bnb_config, - token = token, - trust_remote_code = trust_remote_code, - # attn_implementation = attn_implementation, - **kwargs, - ) + model = auto_model.from_pretrained( + model_name, + device_map = device_map, + torch_dtype = torch_dtype, + # quantization_config = bnb_config, + token = token, + trust_remote_code = trust_remote_code, + # attn_implementation = attn_implementation, + **kwargs, + ) raise_handler.remove() # Return old flag os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = old_hf_transfer