From dde18ddd28a14c842cbb27a4e5c0be026d6d7e63 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 16 Mar 2026 14:32:34 +0000 Subject: [PATCH] Fix FP8 MoE loader: guard import, pass auth to FastModel 1. Wrap moe_utils_fp8 import in try/except so unsloth does not crash when paired with an older unsloth_zoo that lacks the module. Falls back to a no-op stub. 2. Pass token and trust_remote_code to get_model_name() and _has_prequantized_fp8_config() in the FastModel path, matching the FastLanguageModel path. Without this, private/gated repos fail silently and fall through to incorrect offline quantization. --- unsloth/models/loader.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 29bf95308d..ea7cb60295 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -66,9 +66,13 @@ from ..device_type import ( from unsloth_zoo.utils import Version, _get_dtype from unsloth_zoo.hf_utils import dtype_from_config from unsloth_zoo.tiled_mlp import patch_tiled_mlp -from unsloth_zoo.temporary_patches.moe_utils_fp8 import ( - maybe_patch_stacked_moe_expert_fp8_scales, -) +try: + from unsloth_zoo.temporary_patches.moe_utils_fp8 import ( + maybe_patch_stacked_moe_expert_fp8_scales, + ) +except ImportError: + def maybe_patch_stacked_moe_expert_fp8_scales(model, model_name=None, token=None, revision=None): + return False transformers_version = Version(transformers_version) SUPPORTS_FOURBIT = transformers_version >= Version("4.37") @@ -1034,13 +1038,19 @@ class FastModel(FastBaseModel): load_in_4bit = load_in_4bit, load_in_fp8 = load_in_fp8, fast_inference = fast_inference, + token = token, + trust_remote_code = trust_remote_code, ) if ( load_in_fp8 != False and not fast_inference and new_model_name == old_model_name ): - if _has_prequantized_fp8_config(model_name): + if _has_prequantized_fp8_config( + model_name, + token = token, + trust_remote_code = trust_remote_code, + ): load_in_fp8 = False else: new_model_name = None