Fix FP8 MoE loader: guard import, pass auth to FastModel
1. Wrap moe_utils_fp8 import in try/except so unsloth does not crash when paired with an older unsloth_zoo that lacks the module. Falls back to a no-op stub. 2. Pass token and trust_remote_code to get_model_name() and _has_prequantized_fp8_config() in the FastModel path, matching the FastLanguageModel path. Without this, private/gated repos fail silently and fall through to incorrect offline quantization.
This commit is contained in:
parent
daa13cf3ad
commit
dde18ddd28
1 changed files with 14 additions and 4 deletions
|
|
@ -66,9 +66,13 @@ from ..device_type import (
|
|||
from unsloth_zoo.utils import Version, _get_dtype
|
||||
from unsloth_zoo.hf_utils import dtype_from_config
|
||||
from unsloth_zoo.tiled_mlp import patch_tiled_mlp
|
||||
from unsloth_zoo.temporary_patches.moe_utils_fp8 import (
|
||||
maybe_patch_stacked_moe_expert_fp8_scales,
|
||||
)
|
||||
try:
|
||||
from unsloth_zoo.temporary_patches.moe_utils_fp8 import (
|
||||
maybe_patch_stacked_moe_expert_fp8_scales,
|
||||
)
|
||||
except ImportError:
|
||||
def maybe_patch_stacked_moe_expert_fp8_scales(model, model_name=None, token=None, revision=None):
|
||||
return False
|
||||
|
||||
transformers_version = Version(transformers_version)
|
||||
SUPPORTS_FOURBIT = transformers_version >= Version("4.37")
|
||||
|
|
@ -1034,13 +1038,19 @@ class FastModel(FastBaseModel):
|
|||
load_in_4bit = load_in_4bit,
|
||||
load_in_fp8 = load_in_fp8,
|
||||
fast_inference = fast_inference,
|
||||
token = token,
|
||||
trust_remote_code = trust_remote_code,
|
||||
)
|
||||
if (
|
||||
load_in_fp8 != False
|
||||
and not fast_inference
|
||||
and new_model_name == old_model_name
|
||||
):
|
||||
if _has_prequantized_fp8_config(model_name):
|
||||
if _has_prequantized_fp8_config(
|
||||
model_name,
|
||||
token = token,
|
||||
trust_remote_code = trust_remote_code,
|
||||
):
|
||||
load_in_fp8 = False
|
||||
else:
|
||||
new_model_name = None
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue