Fix FP8 model loading: redirect to BF16 sibling for BNB/16-bit (#4095)
* Fix FP8 model loading for BNB/16-bit: redirect to BF16 sibling Models like Ministral-3-3B-Instruct-2512 ship with FP8 weights and an FP8 quantization_config in their config.json. Loading these with BNB 4-bit/8-bit fails because BNB cannot quantize FP8 tensors. Loading with 16-bit also fails because the FP8 quantization config has activation_scheme=static which is unsupported by transformers' FineGrainedFP8Config. When an FP8 model is detected and the user is not explicitly requesting FP8 loading, check if a BF16 sibling repo exists (model_name + "-BF16") and redirect to it. This happens early in the loading flow before any quantization config processing. Also pass the modified model_config to auto_model.from_pretrained to avoid transformers re-reading the original config from the model repo. Tested with Ministral-3-3B in 4-bit and 16-bit modes. Both now load and train correctly. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Simplify FP8 condition and narrow exception handling Simplify the load_in_fp8 check (works for bool and string values). Narrow inner except to KeyError and add comment for outer except. * Warn user when FP8 model has no BF16 sibling for redirect Previously the except block silently fell through with `pass`, so users would get a confusing BNB dtype error later. Now prints a clear message explaining the FP8 situation and suggesting load_in_fp8=True or uploading a BF16 version. * Fix FP8 redirect state corruption and add fbgemm_fp8 support - Fix state corruption: model_name was reassigned before AutoConfig.from_pretrained, so if config fetch failed, model_name pointed to BF16 repo while auto_config still had FP8. Now only updates state after both checks succeed. - Save original model_name so warning message is correct even on failure. - Handle fbgemm_fp8 quant method in addition to fp8. * Extract FP8 redirect to shared _redirect_fp8_to_bf16() in _utils.py Addresses reviewer feedback: - Move FP8 redirect logic to a shared function callable from both vision.py (FastBaseModel) and llama.py (FastLlamaModel) - Raise RuntimeError instead of warning when BF16 sibling not found - Add FP8 redirect to llama.py for text-only model loading path * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add Ministral 3B/8B/14B mapper entries Adds all 9 Ministral model variants to the mapper: - Instruct (3B, 8B, 14B) with FP8 variant mappings - Base (3B, 8B, 14B) - Reasoning (3B, 8B, 14B) This routes mistralai/Ministral-* to unsloth/Ministral-* repos (BF16 weights), which also avoids the FP8 config issue for the standard loading path through loader.py. * Add FP8 mapper entries for Mistral-Small-3.2 and Magistral-Small-2509 --------- Co-authored-by: Ubuntu <ubuntu@ip-172-31-16-253.us-east-2.compute.internal> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Daniel Han
GitHub
parent
37a4c826a0
commit
434b38f6e1
4 changed files with 176 additions and 10 deletions
|
|
@ -74,6 +74,7 @@ __all__ = [
|
|||
"dequantize_module_weight",
|
||||
"patch_hf_quantizer",
|
||||
"verify_fp8_support_if_applicable",
|
||||
"_redirect_fp8_to_bf16",
|
||||
"_get_inference_mode_context_manager",
|
||||
"hf_login",
|
||||
"is_moe_model",
|
||||
|
|
@ -2570,6 +2571,59 @@ def patch_hf_quantizer():
|
|||
patch_hf_quantizer()
|
||||
|
||||
|
||||
def _redirect_fp8_to_bf16(
|
||||
model_name, auto_config, load_in_fp8, token, trust_remote_code
|
||||
):
|
||||
"""
|
||||
Detect FP8 quantization in model config and redirect to BF16 sibling.
|
||||
|
||||
Models shipping FP8 as default (e.g. mistralai/Ministral-3-*B-Instruct)
|
||||
cannot be loaded with BNB 4-bit/8-bit or 16-bit mode. This detects
|
||||
quant_method in ("fp8", "fbgemm_fp8") and redirects to {model_name}-BF16.
|
||||
|
||||
Redirect is SKIPPED when load_in_fp8 is truthy (True or 'block'),
|
||||
meaning the user explicitly wants FP8 loading.
|
||||
|
||||
Returns (model_name, auto_config) -- possibly updated.
|
||||
"""
|
||||
if not hasattr(auto_config, "quantization_config"):
|
||||
return model_name, auto_config
|
||||
|
||||
_qc = auto_config.quantization_config
|
||||
_qm = (
|
||||
_qc.get("quant_method", "")
|
||||
if isinstance(_qc, dict)
|
||||
else getattr(_qc, "quant_method", "")
|
||||
)
|
||||
if _qm not in ("fp8", "fbgemm_fp8") or load_in_fp8:
|
||||
return model_name, auto_config
|
||||
|
||||
_bf16_name = model_name.rstrip("/") + "-BF16"
|
||||
_original_name = model_name
|
||||
try:
|
||||
from huggingface_hub import model_info as _hf_model_info
|
||||
from transformers import AutoConfig
|
||||
|
||||
_hf_model_info(_bf16_name, token = token)
|
||||
_bf16_config = AutoConfig.from_pretrained(
|
||||
_bf16_name,
|
||||
token = token,
|
||||
trust_remote_code = trust_remote_code,
|
||||
)
|
||||
print(
|
||||
f"Unsloth: {_original_name} uses FP8 weights. "
|
||||
f"Redirecting to {_bf16_name}."
|
||||
)
|
||||
return _bf16_name, _bf16_config
|
||||
except Exception:
|
||||
raise RuntimeError(
|
||||
f"Unsloth: {_original_name} uses FP8 weights but no BF16 version "
|
||||
f"was found at {_bf16_name}.\n"
|
||||
f"Loading FP8 weights with BitsAndBytes or in 16-bit will fail.\n"
|
||||
f"Set load_in_fp8=True to use FP8 mode, or upload a BF16 version."
|
||||
)
|
||||
|
||||
|
||||
def verify_fp8_support_if_applicable(model_config):
|
||||
quant_method = get_quant_type(model_config)
|
||||
if quant_method in ["fbgemm_fp8", "fp8"] and DEVICE_TYPE != "cuda":
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from ._utils import move_to_device
|
|||
from ._utils import (
|
||||
_get_inference_mode_context_manager,
|
||||
_prepare_model_for_qat,
|
||||
_redirect_fp8_to_bf16,
|
||||
)
|
||||
from .loader_utils import _get_fp8_mode_and_check_settings
|
||||
from ..utils.packing import (
|
||||
|
|
@ -2228,6 +2229,15 @@ class FastLlamaModel:
|
|||
token = token,
|
||||
attn_implementation = "sdpa",
|
||||
)
|
||||
# Handle FP8 models: redirect to BF16 sibling when the model ships with
|
||||
# FP8 weights. Redirect is skipped when load_in_fp8 is truthy (True or 'block').
|
||||
model_name, model_config = _redirect_fp8_to_bf16(
|
||||
model_name,
|
||||
model_config,
|
||||
load_in_fp8,
|
||||
token,
|
||||
trust_remote_code,
|
||||
)
|
||||
model_config.model_name = model_name
|
||||
model_max_seq_length = model_config.max_position_embeddings
|
||||
|
||||
|
|
|
|||
|
|
@ -965,11 +965,18 @@ __INT_TO_FLOAT_MAPPER = \
|
|||
"mistralai/Magistral-Small-2506",
|
||||
"unsloth/Magistral-Small-2506-bnb-4bit",
|
||||
),
|
||||
"unsloth/Mistral-Small-3.2-24B-Instruct-2506-unsloth-bnb-4bit" : (
|
||||
"unsloth/Mistral-Small-3.2-24B-Instruct-2506",
|
||||
"mistralai/Mistral-Small-3.2-24B-Instruct-2506",
|
||||
"unsloth/Mistral-Small-3.2-24B-Instruct-2506-bnb-4bit",
|
||||
),
|
||||
"unsloth/Mistral-Small-3.2-24B-Instruct-2506-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"mistralai/Mistral-Small-3.2-24B-Instruct-2506",
|
||||
"unsloth/Mistral-Small-3.2-24B-Instruct-2506-FP8",
|
||||
"unsloth/Mistral-Small-3.2-24B-Instruct-2506-FP8",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Mistral-Small-3.2-24B-Instruct-2506",
|
||||
"mistralai/Mistral-Small-3.2-24B-Instruct-2506",
|
||||
"unsloth/Mistral-Small-3.2-24B-Instruct-2506-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/gemma-3n-E4B-it-unsloth-bnb-4bit" : (
|
||||
"unsloth/gemma-3n-E4B-it",
|
||||
"google/gemma-3n-E4B-it",
|
||||
|
|
@ -1056,11 +1063,18 @@ __INT_TO_FLOAT_MAPPER = \
|
|||
"mistralai/Magistral-Small-2507",
|
||||
"unsloth/Magistral-Small-2507-bnb-4bit",
|
||||
),
|
||||
"unsloth/Magistral-Small-2509-unsloth-bnb-4bit" : (
|
||||
"unsloth/Magistral-Small-2509",
|
||||
"mistralai/Magistral-Small-2509",
|
||||
"unsloth/Magistral-Small-2509-bnb-4bit",
|
||||
),
|
||||
"unsloth/Magistral-Small-2509-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"mistralai/Magistral-Small-2509",
|
||||
"unsloth/Magistral-Small-2509-FP8-Dynamic",
|
||||
"unsloth/Magistral-Small-2509-FP8-Dynamic",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Magistral-Small-2509",
|
||||
"mistralai/Magistral-Small-2509",
|
||||
"unsloth/Magistral-Small-2509-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Apertus-70B-Instruct-2509-unsloth-bnb-4bit" : (
|
||||
"unsloth/Apertus-70B-Instruct-2509",
|
||||
"swiss-ai/Apertus-70B-2509",
|
||||
|
|
@ -1256,6 +1270,73 @@ __INT_TO_FLOAT_MAPPER = \
|
|||
"google/functiongemma-270m-it",
|
||||
"unsloth/functiongemma-270m-it-unsloth-bnb-4bit",
|
||||
),
|
||||
# Ministral 3 models
|
||||
"unsloth/Ministral-3-3B-Instruct-2512-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"mistralai/Ministral-3-3B-Instruct-2512",
|
||||
"unsloth/Ministral-3-3B-Instruct-2512-FP8",
|
||||
"unsloth/Ministral-3-3B-Instruct-2512-FP8",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Ministral-3-3B-Instruct-2512",
|
||||
"mistralai/Ministral-3-3B-Instruct-2512",
|
||||
"unsloth/Ministral-3-3B-Instruct-2512-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Ministral-3-3B-Base-2512-unsloth-bnb-4bit" : (
|
||||
"unsloth/Ministral-3-3B-Base-2512",
|
||||
"mistralai/Ministral-3-3B-Base-2512",
|
||||
"unsloth/Ministral-3-3B-Base-2512-bnb-4bit",
|
||||
),
|
||||
"unsloth/Ministral-3-3B-Reasoning-2512-unsloth-bnb-4bit" : (
|
||||
"unsloth/Ministral-3-3B-Reasoning-2512",
|
||||
"mistralai/Ministral-3-3B-Reasoning-2512",
|
||||
"unsloth/Ministral-3-3B-Reasoning-2512-bnb-4bit",
|
||||
),
|
||||
"unsloth/Ministral-3-8B-Instruct-2512-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"mistralai/Ministral-3-8B-Instruct-2512",
|
||||
"unsloth/Ministral-3-8B-Instruct-2512-FP8",
|
||||
"unsloth/Ministral-3-8B-Instruct-2512-FP8",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Ministral-3-8B-Instruct-2512",
|
||||
"mistralai/Ministral-3-8B-Instruct-2512",
|
||||
"unsloth/Ministral-3-8B-Instruct-2512-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Ministral-3-8B-Base-2512-unsloth-bnb-4bit" : (
|
||||
"unsloth/Ministral-3-8B-Base-2512",
|
||||
"mistralai/Ministral-3-8B-Base-2512",
|
||||
"unsloth/Ministral-3-8B-Base-2512-bnb-4bit",
|
||||
),
|
||||
"unsloth/Ministral-3-8B-Reasoning-2512-unsloth-bnb-4bit" : (
|
||||
"unsloth/Ministral-3-8B-Reasoning-2512",
|
||||
"mistralai/Ministral-3-8B-Reasoning-2512",
|
||||
"unsloth/Ministral-3-8B-Reasoning-2512-bnb-4bit",
|
||||
),
|
||||
"unsloth/Ministral-3-14B-Instruct-2512-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"mistralai/Ministral-3-14B-Instruct-2512",
|
||||
"unsloth/Ministral-3-14B-Instruct-2512-FP8",
|
||||
"unsloth/Ministral-3-14B-Instruct-2512-FP8",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Ministral-3-14B-Instruct-2512",
|
||||
"mistralai/Ministral-3-14B-Instruct-2512",
|
||||
"unsloth/Ministral-3-14B-Instruct-2512-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Ministral-3-14B-Base-2512-unsloth-bnb-4bit" : (
|
||||
"unsloth/Ministral-3-14B-Base-2512",
|
||||
"mistralai/Ministral-3-14B-Base-2512",
|
||||
"unsloth/Ministral-3-14B-Base-2512-bnb-4bit",
|
||||
),
|
||||
"unsloth/Ministral-3-14B-Reasoning-2512-unsloth-bnb-4bit" : (
|
||||
"unsloth/Ministral-3-14B-Reasoning-2512",
|
||||
"mistralai/Ministral-3-14B-Reasoning-2512",
|
||||
"unsloth/Ministral-3-14B-Reasoning-2512-bnb-4bit",
|
||||
),
|
||||
}
|
||||
|
||||
INT_TO_FLOAT_MAPPER = {}
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ from ..kernels import (
|
|||
post_patch_loss_function,
|
||||
)
|
||||
from ._utils import __version__, importlib_version, _prepare_model_for_qat
|
||||
from ._utils import _redirect_fp8_to_bf16
|
||||
from ._utils import *
|
||||
from .loader_utils import _get_fp8_mode_and_check_settings
|
||||
from ..save import patch_saving_functions
|
||||
|
|
@ -611,6 +612,24 @@ class FastBaseModel:
|
|||
model_class = None
|
||||
flex_attn_impl = prefer_flex_attn_if_supported(model_class, auto_config)
|
||||
|
||||
# Handle FP8 models: redirect to BF16 sibling when the model ships with
|
||||
# FP8 weights (e.g. Ministral-3-3B-Instruct-2512). FP8 weights cannot be
|
||||
# directly loaded by BNB, and the FP8 quantization config can cause issues
|
||||
# even for 16-bit loading.
|
||||
# Redirect is skipped when load_in_fp8 is truthy (True or 'block').
|
||||
model_name, auto_config = _redirect_fp8_to_bf16(
|
||||
model_name,
|
||||
auto_config,
|
||||
load_in_fp8,
|
||||
token,
|
||||
trust_remote_code,
|
||||
)
|
||||
# Re-resolve model_class after potential config change
|
||||
try:
|
||||
model_class = auto_model._model_mapping[auto_config.__class__]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
default_attn_impl = "flex_attention" if flex_attn_impl else "sdpa"
|
||||
if not ("attn_implementation" in kwargs):
|
||||
kwargs["attn_implementation"] = default_attn_impl
|
||||
|
|
@ -759,6 +778,7 @@ class FastBaseModel:
|
|||
if hasattr(auto_config, "attn_implementation"):
|
||||
setattr(auto_config, "attn_implementation", config_attn_impl)
|
||||
model_config = auto_config
|
||||
|
||||
verify_fp8_support_if_applicable(model_config)
|
||||
|
||||
raise_handler = RaiseUninitialized()
|
||||
|
|
@ -767,6 +787,7 @@ class FastBaseModel:
|
|||
load_in_fp8 = kwargs.pop("load_in_fp8", None)
|
||||
model = auto_model.from_pretrained(
|
||||
model_name,
|
||||
config = model_config,
|
||||
device_map = device_map,
|
||||
# torch_dtype = torch_dtype, # Transformers removed torch_dtype
|
||||
# quantization_config = bnb_config,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue