From 92c024df79ff1e0e338569b5e6ad007eef7cfa88 Mon Sep 17 00:00:00 2001 From: DoubleMathew Date: Wed, 6 Aug 2025 14:13:35 -0500 Subject: [PATCH 01/13] gpt-oss manually call temporary patch (#3104) Co-authored-by: Mathew Mathew --- unsloth/models/loader.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index b3218d2498..77f953a5a3 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -592,6 +592,19 @@ class FastModel(FastBaseModel): "os.environ['TRITON_F32_DEFAULT'] = 'ieee';" elif "gpt-oss" in lowered_model_name: os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" + # the temporary patches for init need UNSLOTH_MODEL_NAME to be set + # which doesn't happen at import so manually call here + # before creating the compiled cache + try: + from unsloth_zoo.temporary_patches.gpt_oss import ( + patch_GptOssExperts_MXFP4, + patch_GptOssExperts_bitsandbytes, + ) + + patch_GptOssExperts_MXFP4() + patch_GptOssExperts_bitsandbytes() + except: + pass else: for check_model_name in DISABLE_COMPILE_MODEL_NAMES: if check_model_name in lowered_model_name: From 382042f3b0a171dfdc31cd8aa6f46310cfa19917 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 7 Aug 2025 02:57:19 -0700 Subject: [PATCH 02/13] GPT OSS fixes --- unsloth/models/_utils.py | 3 ++- unsloth/models/loader.py | 20 ++++++-------------- unsloth/models/vision.py | 6 ++++-- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 2e88e213e6..a036970b40 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -76,6 +76,7 @@ platform_system = platform_system() import numpy as np import contextlib import re +import functools import warnings, subprocess, re, inspect, psutil, os, math from unsloth_zoo.utils import Version from unsloth import DEVICE_TYPE, DEVICE_COUNT @@ -422,6 +423,7 @@ HAS_FLASH_ATTENTION_SOFTCAPPING = False if DEVICE_TYPE == "cuda": major_version, minor_version = torch.cuda.get_device_capability() + torch.cuda.get_device_capability = functools.cache(torch.cuda.get_device_capability) if major_version >= 8: SUPPORTS_BFLOAT16 = True @@ -586,7 +588,6 @@ UNSLOTH_COMPILE_DEBUG = os.environ.get("UNSLOTH_COMPILE_DEBUG", UNSLOTH_COMPILE_MAXIMUM = os.environ.get("UNSLOTH_COMPILE_MAXIMUM", "0") == "1" UNSLOTH_COMPILE_IGNORE_ERRORS = os.environ.get("UNSLOTH_COMPILE_IGNORE_ERRORS", "1") == "1" # Just remove max_autotune_gemm warning -import functools from torch._inductor.runtime.hints import DeviceProperties @functools.lru_cache(None) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 77f953a5a3..15a506174f 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -591,20 +591,12 @@ class FastModel(FastBaseModel): "if name.endswith(('q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj', 'head')): module.to(torch.float16); "\ "os.environ['TRITON_F32_DEFAULT'] = 'ieee';" elif "gpt-oss" in lowered_model_name: - os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" - # the temporary patches for init need UNSLOTH_MODEL_NAME to be set - # which doesn't happen at import so manually call here - # before creating the compiled cache - try: - from unsloth_zoo.temporary_patches.gpt_oss import ( - patch_GptOssExperts_MXFP4, - patch_GptOssExperts_bitsandbytes, - ) - - patch_GptOssExperts_MXFP4() - patch_GptOssExperts_bitsandbytes() - except: - pass + os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = \ + "all;None;None;"\ + "x = 'gate_up_proj_bias'\n"\ + "if hasattr(module, x): setattr(module, x, torch.nn.Parameter(getattr(module, x).to(torch.float32)) if isinstance(getattr(module, x), torch.nn.Parameter) else getattr(module, x).to(torch.float32))\n"\ + "x = 'down_proj_bias'\n"\ + "if hasattr(module, x): setattr(module, x, torch.nn.Parameter(getattr(module, x).to(torch.float32)) if isinstance(getattr(module, x), torch.nn.Parameter) else getattr(module, x).to(torch.float32))\n;" else: for check_model_name in DISABLE_COMPILE_MODEL_NAMES: if check_model_name in lowered_model_name: diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 623f263127..b9cc01e530 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -365,8 +365,10 @@ class FastBaseModel: allow_float16_runs = (checker == "float16" and dtype == torch.float16) if allow_all_runs or allow_float16_runs: - dtype = eval(_dtype) - bnb_compute_dtype = eval(_bnb_compute_dtype) + if eval(_dtype) is not None: + dtype = eval(_dtype) + if eval(_bnb_compute_dtype) is not None: + bnb_compute_dtype = eval(_bnb_compute_dtype) correct_dtype = bnb_compute_dtype custom_datatype = _custom_datatype # Execute code as well From 0171432daff975da75088c539705d0d5133424ba Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 7 Aug 2025 03:29:44 -0700 Subject: [PATCH 03/13] Update pyproject.toml --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 8cdd90ea99..f7a8cf91c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,9 @@ windows=[ "bitsandbytes>=0.45.5 ; platform_system == 'Windows'", "xformers>=0.0.22.post7 ; platform_system == 'Windows'", ] +base = [ + "unsloth[huggingface]", +] cu118only = [ "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and platform_system == 'Linux'", "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and platform_system == 'Linux'", From b7dcf7e5ed33ec34a4b241f0fd64eec4965aa671 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 7 Aug 2025 03:43:25 -0700 Subject: [PATCH 04/13] Update pyproject.toml --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f7a8cf91c8..f75a75d9cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,6 @@ huggingface = [ "accelerate>=0.34.1", "trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,!=0.15.0,!=0.19.0", "peft>=0.7.1,!=0.11.0", - "protobuf", "huggingface_hub>=0.34.0", "hf_transfer", "unsloth[triton]", From a1746fc03e47f907fdbb813918f27a81b20a72fa Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 7 Aug 2025 03:47:08 -0700 Subject: [PATCH 05/13] Update pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index f75a75d9cd..f7a8cf91c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ huggingface = [ "accelerate>=0.34.1", "trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,!=0.15.0,!=0.19.0", "peft>=0.7.1,!=0.11.0", + "protobuf", "huggingface_hub>=0.34.0", "hf_transfer", "unsloth[triton]", From d605b629ecc09e193048c69acd58b88435e6fc4a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 7 Aug 2025 04:59:19 -0700 Subject: [PATCH 06/13] Update mapper.py --- unsloth/models/mapper.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index bba8f982da..d15be81aec 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -921,15 +921,15 @@ __INT_TO_FLOAT_MAPPER = \ "unsloth/Qwen3-Coder-30B-A3B-Instruct", "Qwen/Qwen3-Coder-30B-A3B-Instruct", ), - "unsloth/gpt-oss-20b-unsloth-bnb-4bit" : ( + "unsloth/gpt-oss-20b" : ( "unsloth/gpt-oss-20b", "openai/gpt-oss-20b", - "unsloth/gpt-oss-20b-bnb-4bit", + "unsloth/gpt-oss-20b", ), - "unsloth/gpt-oss-120b-unsloth-bnb-4bit" : ( + "unsloth/gpt-oss-120b" : ( "unsloth/gpt-oss-120b", "openai/gpt-oss-120b", - "unsloth/gpt-oss-120b-bnb-4bit", + "unsloth/gpt-oss-120b", ), } From 4d89527df6dc67beb98661d4cb6a25af22ea737f Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 7 Aug 2025 05:03:21 -0700 Subject: [PATCH 07/13] Update vision.py --- unsloth/models/vision.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index b9cc01e530..501ae227e0 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -423,6 +423,10 @@ class FastBaseModel: os.environ["UNSLOTH_ENABLE_FULL_FINETUNING"] = "0" pass + # Fix AttributeError: 'BitsAndBytesConfig' object has no attribute 'get_loading_attributes' + if not hasattr(bnb_config, "get_loading_attributes"): + bnb_config.get_loading_attributes = lambda *args, **kwargs: {} + # Cannot be None, since HF now checks for the config if load_in_4bit: kwargs["quantization_config"] = bnb_config From 5a31edef5f78ecff19c7eef429266ee5d6649ba5 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 7 Aug 2025 05:18:53 -0700 Subject: [PATCH 08/13] Update loader.py --- unsloth/models/loader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 15a506174f..94a0c526db 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -591,6 +591,7 @@ class FastModel(FastBaseModel): "if name.endswith(('q_proj', 'k_proj', 'v_proj', 'o_proj', 'gate_proj', 'up_proj', 'down_proj', 'head')): module.to(torch.float16); "\ "os.environ['TRITON_F32_DEFAULT'] = 'ieee';" elif "gpt-oss" in lowered_model_name: + os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = \ "all;None;None;"\ "x = 'gate_up_proj_bias'\n"\ From accb7b1ead70539ac925b72db1c22282090ebf31 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 7 Aug 2025 07:10:36 -0700 Subject: [PATCH 09/13] Update vision.py --- unsloth/models/vision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 501ae227e0..68b22135fc 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -424,7 +424,7 @@ class FastBaseModel: pass # Fix AttributeError: 'BitsAndBytesConfig' object has no attribute 'get_loading_attributes' - if not hasattr(bnb_config, "get_loading_attributes"): + if bnb_config is not None and not hasattr(bnb_config, "get_loading_attributes"): bnb_config.get_loading_attributes = lambda *args, **kwargs: {} # Cannot be None, since HF now checks for the config From 1a8eabebab1fbface124d581c6b4f4aed574b5ec Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 7 Aug 2025 07:13:40 -0700 Subject: [PATCH 10/13] Update vision.py --- unsloth/models/vision.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 68b22135fc..38606c4ff9 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -435,23 +435,16 @@ class FastBaseModel: if do_forced_float32: torch_dtype = torch.bfloat16 raise_handler = RaiseUninitialized() - # MXFP4 -> BF16 GPT-OSS check - if "gpt-oss" in os.environ.get("UNSLOTH_MODEL_NAME", "") and \ - "quantization_config" not in kwargs: - - from unsloth_zoo.temporary_patches.gpt_oss import load_gpt_oss_MXFP4 - model = load_gpt_oss_MXFP4(model_name, torch_dtype) - else: - model = auto_model.from_pretrained( - model_name, - device_map = device_map, - torch_dtype = torch_dtype, - # quantization_config = bnb_config, - token = token, - trust_remote_code = trust_remote_code, - # attn_implementation = attn_implementation, - **kwargs, - ) + model = auto_model.from_pretrained( + model_name, + device_map = device_map, + torch_dtype = torch_dtype, + # quantization_config = bnb_config, + token = token, + trust_remote_code = trust_remote_code, + # attn_implementation = attn_implementation, + **kwargs, + ) raise_handler.remove() # Return old flag os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = old_hf_transfer From 9889015485af75b5705e83b50903a48983ffece0 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 7 Aug 2025 08:15:59 -0700 Subject: [PATCH 11/13] Update mapper.py --- unsloth/models/mapper.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index d15be81aec..8f5ffbb509 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -921,15 +921,15 @@ __INT_TO_FLOAT_MAPPER = \ "unsloth/Qwen3-Coder-30B-A3B-Instruct", "Qwen/Qwen3-Coder-30B-A3B-Instruct", ), - "unsloth/gpt-oss-20b" : ( + "unsloth/gpt-oss-20b-unsloth-bnb-4bit" : ( "unsloth/gpt-oss-20b", "openai/gpt-oss-20b", - "unsloth/gpt-oss-20b", + "unsloth/gpt-oss-20b-unsloth-bnb-4bit", ), - "unsloth/gpt-oss-120b" : ( + "unsloth/gpt-oss-120b-unsloth-bnb-4bit" : ( "unsloth/gpt-oss-120b", "openai/gpt-oss-120b", - "unsloth/gpt-oss-120b", + "unsloth/gpt-oss-120b-unsloth-bnb-4bit", ), } From aa96d65534285dfa86f00bea64370c80e893f79b Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 7 Aug 2025 09:38:24 -0700 Subject: [PATCH 12/13] Update loader.py --- unsloth/models/loader.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 94a0c526db..bc98c0dc19 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -592,12 +592,14 @@ class FastModel(FastBaseModel): "os.environ['TRITON_F32_DEFAULT'] = 'ieee';" elif "gpt-oss" in lowered_model_name: os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" - os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = \ - "all;None;None;"\ - "x = 'gate_up_proj_bias'\n"\ - "if hasattr(module, x): setattr(module, x, torch.nn.Parameter(getattr(module, x).to(torch.float32)) if isinstance(getattr(module, x), torch.nn.Parameter) else getattr(module, x).to(torch.float32))\n"\ - "x = 'down_proj_bias'\n"\ - "if hasattr(module, x): setattr(module, x, torch.nn.Parameter(getattr(module, x).to(torch.float32)) if isinstance(getattr(module, x), torch.nn.Parameter) else getattr(module, x).to(torch.float32))\n;" + if not model_name: + # Only upcast MoE biases for MXFP4, not BnB + os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = \ + "all;None;None;"\ + "x = 'gate_up_proj_bias'\n"\ + "if hasattr(module, x): setattr(module, x, torch.nn.Parameter(getattr(module, x).to(torch.float32)) if isinstance(getattr(module, x), torch.nn.Parameter) else getattr(module, x).to(torch.float32))\n"\ + "x = 'down_proj_bias'\n"\ + "if hasattr(module, x): setattr(module, x, torch.nn.Parameter(getattr(module, x).to(torch.float32)) if isinstance(getattr(module, x), torch.nn.Parameter) else getattr(module, x).to(torch.float32))\n;" else: for check_model_name in DISABLE_COMPILE_MODEL_NAMES: if check_model_name in lowered_model_name: From 180917f14889aea1e939272c706dc1ba6562042c Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 7 Aug 2025 09:38:49 -0700 Subject: [PATCH 13/13] Update loader.py --- unsloth/models/loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index bc98c0dc19..efbf4ed37c 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -592,7 +592,7 @@ class FastModel(FastBaseModel): "os.environ['TRITON_F32_DEFAULT'] = 'ieee';" elif "gpt-oss" in lowered_model_name: os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" - if not model_name: + if not load_in_4bit: # Only upcast MoE biases for MXFP4, not BnB os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = \ "all;None;None;"\