From 434b38f6e1d3b97f23d465bbdbefb53c1c835720 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 24 Feb 2026 05:56:07 -0800 Subject: [PATCH] Fix FP8 model loading: redirect to BF16 sibling for BNB/16-bit (#4095) * Fix FP8 model loading for BNB/16-bit: redirect to BF16 sibling Models like Ministral-3-3B-Instruct-2512 ship with FP8 weights and an FP8 quantization_config in their config.json. Loading these with BNB 4-bit/8-bit fails because BNB cannot quantize FP8 tensors. Loading with 16-bit also fails because the FP8 quantization config has activation_scheme=static which is unsupported by transformers' FineGrainedFP8Config. When an FP8 model is detected and the user is not explicitly requesting FP8 loading, check if a BF16 sibling repo exists (model_name + "-BF16") and redirect to it. This happens early in the loading flow before any quantization config processing. Also pass the modified model_config to auto_model.from_pretrained to avoid transformers re-reading the original config from the model repo. Tested with Ministral-3-3B in 4-bit and 16-bit modes. Both now load and train correctly. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Simplify FP8 condition and narrow exception handling Simplify the load_in_fp8 check (works for bool and string values). Narrow inner except to KeyError and add comment for outer except. * Warn user when FP8 model has no BF16 sibling for redirect Previously the except block silently fell through with `pass`, so users would get a confusing BNB dtype error later. Now prints a clear message explaining the FP8 situation and suggesting load_in_fp8=True or uploading a BF16 version. * Fix FP8 redirect state corruption and add fbgemm_fp8 support - Fix state corruption: model_name was reassigned before AutoConfig.from_pretrained, so if config fetch failed, model_name pointed to BF16 repo while auto_config still had FP8. Now only updates state after both checks succeed. - Save original model_name so warning message is correct even on failure. - Handle fbgemm_fp8 quant method in addition to fp8. * Extract FP8 redirect to shared _redirect_fp8_to_bf16() in _utils.py Addresses reviewer feedback: - Move FP8 redirect logic to a shared function callable from both vision.py (FastBaseModel) and llama.py (FastLlamaModel) - Raise RuntimeError instead of warning when BF16 sibling not found - Add FP8 redirect to llama.py for text-only model loading path * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add Ministral 3B/8B/14B mapper entries Adds all 9 Ministral model variants to the mapper: - Instruct (3B, 8B, 14B) with FP8 variant mappings - Base (3B, 8B, 14B) - Reasoning (3B, 8B, 14B) This routes mistralai/Ministral-* to unsloth/Ministral-* repos (BF16 weights), which also avoids the FP8 config issue for the standard loading path through loader.py. * Add FP8 mapper entries for Mistral-Small-3.2 and Magistral-Small-2509 --------- Co-authored-by: Ubuntu Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- unsloth/models/_utils.py | 54 +++++++++++++++++++++ unsloth/models/llama.py | 10 ++++ unsloth/models/mapper.py | 101 +++++++++++++++++++++++++++++++++++---- unsloth/models/vision.py | 21 ++++++++ 4 files changed, 176 insertions(+), 10 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 451f60091f..e7cc5ff1ef 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -74,6 +74,7 @@ __all__ = [ "dequantize_module_weight", "patch_hf_quantizer", "verify_fp8_support_if_applicable", + "_redirect_fp8_to_bf16", "_get_inference_mode_context_manager", "hf_login", "is_moe_model", @@ -2570,6 +2571,59 @@ def patch_hf_quantizer(): patch_hf_quantizer() +def _redirect_fp8_to_bf16( + model_name, auto_config, load_in_fp8, token, trust_remote_code +): + """ + Detect FP8 quantization in model config and redirect to BF16 sibling. + + Models shipping FP8 as default (e.g. mistralai/Ministral-3-*B-Instruct) + cannot be loaded with BNB 4-bit/8-bit or 16-bit mode. This detects + quant_method in ("fp8", "fbgemm_fp8") and redirects to {model_name}-BF16. + + Redirect is SKIPPED when load_in_fp8 is truthy (True or 'block'), + meaning the user explicitly wants FP8 loading. + + Returns (model_name, auto_config) -- possibly updated. + """ + if not hasattr(auto_config, "quantization_config"): + return model_name, auto_config + + _qc = auto_config.quantization_config + _qm = ( + _qc.get("quant_method", "") + if isinstance(_qc, dict) + else getattr(_qc, "quant_method", "") + ) + if _qm not in ("fp8", "fbgemm_fp8") or load_in_fp8: + return model_name, auto_config + + _bf16_name = model_name.rstrip("/") + "-BF16" + _original_name = model_name + try: + from huggingface_hub import model_info as _hf_model_info + from transformers import AutoConfig + + _hf_model_info(_bf16_name, token = token) + _bf16_config = AutoConfig.from_pretrained( + _bf16_name, + token = token, + trust_remote_code = trust_remote_code, + ) + print( + f"Unsloth: {_original_name} uses FP8 weights. " + f"Redirecting to {_bf16_name}." + ) + return _bf16_name, _bf16_config + except Exception: + raise RuntimeError( + f"Unsloth: {_original_name} uses FP8 weights but no BF16 version " + f"was found at {_bf16_name}.\n" + f"Loading FP8 weights with BitsAndBytes or in 16-bit will fail.\n" + f"Set load_in_fp8=True to use FP8 mode, or upload a BF16 version." + ) + + def verify_fp8_support_if_applicable(model_config): quant_method = get_quant_type(model_config) if quant_method in ["fbgemm_fp8", "fp8"] and DEVICE_TYPE != "cuda": diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index a3fc9ab49a..342366f02e 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -25,6 +25,7 @@ from ._utils import move_to_device from ._utils import ( _get_inference_mode_context_manager, _prepare_model_for_qat, + _redirect_fp8_to_bf16, ) from .loader_utils import _get_fp8_mode_and_check_settings from ..utils.packing import ( @@ -2228,6 +2229,15 @@ class FastLlamaModel: token = token, attn_implementation = "sdpa", ) + # Handle FP8 models: redirect to BF16 sibling when the model ships with + # FP8 weights. Redirect is skipped when load_in_fp8 is truthy (True or 'block'). + model_name, model_config = _redirect_fp8_to_bf16( + model_name, + model_config, + load_in_fp8, + token, + trust_remote_code, + ) model_config.model_name = model_name model_max_seq_length = model_config.max_position_embeddings diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index e7296291f1..ec7a7a8046 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -965,11 +965,18 @@ __INT_TO_FLOAT_MAPPER = \ "mistralai/Magistral-Small-2506", "unsloth/Magistral-Small-2506-bnb-4bit", ), - "unsloth/Mistral-Small-3.2-24B-Instruct-2506-unsloth-bnb-4bit" : ( - "unsloth/Mistral-Small-3.2-24B-Instruct-2506", - "mistralai/Mistral-Small-3.2-24B-Instruct-2506", - "unsloth/Mistral-Small-3.2-24B-Instruct-2506-bnb-4bit", - ), + "unsloth/Mistral-Small-3.2-24B-Instruct-2506-unsloth-bnb-4bit" : { + "8" : ( + "mistralai/Mistral-Small-3.2-24B-Instruct-2506", + "unsloth/Mistral-Small-3.2-24B-Instruct-2506-FP8", + "unsloth/Mistral-Small-3.2-24B-Instruct-2506-FP8", + ), + "16" : ( + "unsloth/Mistral-Small-3.2-24B-Instruct-2506", + "mistralai/Mistral-Small-3.2-24B-Instruct-2506", + "unsloth/Mistral-Small-3.2-24B-Instruct-2506-bnb-4bit", + ), + }, "unsloth/gemma-3n-E4B-it-unsloth-bnb-4bit" : ( "unsloth/gemma-3n-E4B-it", "google/gemma-3n-E4B-it", @@ -1056,11 +1063,18 @@ __INT_TO_FLOAT_MAPPER = \ "mistralai/Magistral-Small-2507", "unsloth/Magistral-Small-2507-bnb-4bit", ), - "unsloth/Magistral-Small-2509-unsloth-bnb-4bit" : ( - "unsloth/Magistral-Small-2509", - "mistralai/Magistral-Small-2509", - "unsloth/Magistral-Small-2509-bnb-4bit", - ), + "unsloth/Magistral-Small-2509-unsloth-bnb-4bit" : { + "8" : ( + "mistralai/Magistral-Small-2509", + "unsloth/Magistral-Small-2509-FP8-Dynamic", + "unsloth/Magistral-Small-2509-FP8-Dynamic", + ), + "16" : ( + "unsloth/Magistral-Small-2509", + "mistralai/Magistral-Small-2509", + "unsloth/Magistral-Small-2509-bnb-4bit", + ), + }, "unsloth/Apertus-70B-Instruct-2509-unsloth-bnb-4bit" : ( "unsloth/Apertus-70B-Instruct-2509", "swiss-ai/Apertus-70B-2509", @@ -1256,6 +1270,73 @@ __INT_TO_FLOAT_MAPPER = \ "google/functiongemma-270m-it", "unsloth/functiongemma-270m-it-unsloth-bnb-4bit", ), + # Ministral 3 models + "unsloth/Ministral-3-3B-Instruct-2512-unsloth-bnb-4bit" : { + "8" : ( + "mistralai/Ministral-3-3B-Instruct-2512", + "unsloth/Ministral-3-3B-Instruct-2512-FP8", + "unsloth/Ministral-3-3B-Instruct-2512-FP8", + ), + "16" : ( + "unsloth/Ministral-3-3B-Instruct-2512", + "mistralai/Ministral-3-3B-Instruct-2512", + "unsloth/Ministral-3-3B-Instruct-2512-bnb-4bit", + ), + }, + "unsloth/Ministral-3-3B-Base-2512-unsloth-bnb-4bit" : ( + "unsloth/Ministral-3-3B-Base-2512", + "mistralai/Ministral-3-3B-Base-2512", + "unsloth/Ministral-3-3B-Base-2512-bnb-4bit", + ), + "unsloth/Ministral-3-3B-Reasoning-2512-unsloth-bnb-4bit" : ( + "unsloth/Ministral-3-3B-Reasoning-2512", + "mistralai/Ministral-3-3B-Reasoning-2512", + "unsloth/Ministral-3-3B-Reasoning-2512-bnb-4bit", + ), + "unsloth/Ministral-3-8B-Instruct-2512-unsloth-bnb-4bit" : { + "8" : ( + "mistralai/Ministral-3-8B-Instruct-2512", + "unsloth/Ministral-3-8B-Instruct-2512-FP8", + "unsloth/Ministral-3-8B-Instruct-2512-FP8", + ), + "16" : ( + "unsloth/Ministral-3-8B-Instruct-2512", + "mistralai/Ministral-3-8B-Instruct-2512", + "unsloth/Ministral-3-8B-Instruct-2512-bnb-4bit", + ), + }, + "unsloth/Ministral-3-8B-Base-2512-unsloth-bnb-4bit" : ( + "unsloth/Ministral-3-8B-Base-2512", + "mistralai/Ministral-3-8B-Base-2512", + "unsloth/Ministral-3-8B-Base-2512-bnb-4bit", + ), + "unsloth/Ministral-3-8B-Reasoning-2512-unsloth-bnb-4bit" : ( + "unsloth/Ministral-3-8B-Reasoning-2512", + "mistralai/Ministral-3-8B-Reasoning-2512", + "unsloth/Ministral-3-8B-Reasoning-2512-bnb-4bit", + ), + "unsloth/Ministral-3-14B-Instruct-2512-unsloth-bnb-4bit" : { + "8" : ( + "mistralai/Ministral-3-14B-Instruct-2512", + "unsloth/Ministral-3-14B-Instruct-2512-FP8", + "unsloth/Ministral-3-14B-Instruct-2512-FP8", + ), + "16" : ( + "unsloth/Ministral-3-14B-Instruct-2512", + "mistralai/Ministral-3-14B-Instruct-2512", + "unsloth/Ministral-3-14B-Instruct-2512-bnb-4bit", + ), + }, + "unsloth/Ministral-3-14B-Base-2512-unsloth-bnb-4bit" : ( + "unsloth/Ministral-3-14B-Base-2512", + "mistralai/Ministral-3-14B-Base-2512", + "unsloth/Ministral-3-14B-Base-2512-bnb-4bit", + ), + "unsloth/Ministral-3-14B-Reasoning-2512-unsloth-bnb-4bit" : ( + "unsloth/Ministral-3-14B-Reasoning-2512", + "mistralai/Ministral-3-14B-Reasoning-2512", + "unsloth/Ministral-3-14B-Reasoning-2512-bnb-4bit", + ), } INT_TO_FLOAT_MAPPER = {} diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 2d6c7ac93f..56cd615a49 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -30,6 +30,7 @@ from ..kernels import ( post_patch_loss_function, ) from ._utils import __version__, importlib_version, _prepare_model_for_qat +from ._utils import _redirect_fp8_to_bf16 from ._utils import * from .loader_utils import _get_fp8_mode_and_check_settings from ..save import patch_saving_functions @@ -611,6 +612,24 @@ class FastBaseModel: model_class = None flex_attn_impl = prefer_flex_attn_if_supported(model_class, auto_config) + # Handle FP8 models: redirect to BF16 sibling when the model ships with + # FP8 weights (e.g. Ministral-3-3B-Instruct-2512). FP8 weights cannot be + # directly loaded by BNB, and the FP8 quantization config can cause issues + # even for 16-bit loading. + # Redirect is skipped when load_in_fp8 is truthy (True or 'block'). + model_name, auto_config = _redirect_fp8_to_bf16( + model_name, + auto_config, + load_in_fp8, + token, + trust_remote_code, + ) + # Re-resolve model_class after potential config change + try: + model_class = auto_model._model_mapping[auto_config.__class__] + except KeyError: + pass + default_attn_impl = "flex_attention" if flex_attn_impl else "sdpa" if not ("attn_implementation" in kwargs): kwargs["attn_implementation"] = default_attn_impl @@ -759,6 +778,7 @@ class FastBaseModel: if hasattr(auto_config, "attn_implementation"): setattr(auto_config, "attn_implementation", config_attn_impl) model_config = auto_config + verify_fp8_support_if_applicable(model_config) raise_handler = RaiseUninitialized() @@ -767,6 +787,7 @@ class FastBaseModel: load_in_fp8 = kwargs.pop("load_in_fp8", None) model = auto_model.from_pretrained( model_name, + config = model_config, device_map = device_map, # torch_dtype = torch_dtype, # Transformers removed torch_dtype # quantization_config = bnb_config,