diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 451f60091f..e7cc5ff1ef 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -74,6 +74,7 @@ __all__ = [ "dequantize_module_weight", "patch_hf_quantizer", "verify_fp8_support_if_applicable", + "_redirect_fp8_to_bf16", "_get_inference_mode_context_manager", "hf_login", "is_moe_model", @@ -2570,6 +2571,59 @@ def patch_hf_quantizer(): patch_hf_quantizer() +def _redirect_fp8_to_bf16( + model_name, auto_config, load_in_fp8, token, trust_remote_code +): + """ + Detect FP8 quantization in model config and redirect to BF16 sibling. + + Models shipping FP8 as default (e.g. mistralai/Ministral-3-*B-Instruct) + cannot be loaded with BNB 4-bit/8-bit or 16-bit mode. This detects + quant_method in ("fp8", "fbgemm_fp8") and redirects to {model_name}-BF16. + + Redirect is SKIPPED when load_in_fp8 is truthy (True or 'block'), + meaning the user explicitly wants FP8 loading. + + Returns (model_name, auto_config) -- possibly updated. + """ + if not hasattr(auto_config, "quantization_config"): + return model_name, auto_config + + _qc = auto_config.quantization_config + _qm = ( + _qc.get("quant_method", "") + if isinstance(_qc, dict) + else getattr(_qc, "quant_method", "") + ) + if _qm not in ("fp8", "fbgemm_fp8") or load_in_fp8: + return model_name, auto_config + + _bf16_name = model_name.rstrip("/") + "-BF16" + _original_name = model_name + try: + from huggingface_hub import model_info as _hf_model_info + from transformers import AutoConfig + + _hf_model_info(_bf16_name, token = token) + _bf16_config = AutoConfig.from_pretrained( + _bf16_name, + token = token, + trust_remote_code = trust_remote_code, + ) + print( + f"Unsloth: {_original_name} uses FP8 weights. " + f"Redirecting to {_bf16_name}." + ) + return _bf16_name, _bf16_config + except Exception: + raise RuntimeError( + f"Unsloth: {_original_name} uses FP8 weights but no BF16 version " + f"was found at {_bf16_name}.\n" + f"Loading FP8 weights with BitsAndBytes or in 16-bit will fail.\n" + f"Set load_in_fp8=True to use FP8 mode, or upload a BF16 version." + ) + + def verify_fp8_support_if_applicable(model_config): quant_method = get_quant_type(model_config) if quant_method in ["fbgemm_fp8", "fp8"] and DEVICE_TYPE != "cuda": diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index a3fc9ab49a..342366f02e 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -25,6 +25,7 @@ from ._utils import move_to_device from ._utils import ( _get_inference_mode_context_manager, _prepare_model_for_qat, + _redirect_fp8_to_bf16, ) from .loader_utils import _get_fp8_mode_and_check_settings from ..utils.packing import ( @@ -2228,6 +2229,15 @@ class FastLlamaModel: token = token, attn_implementation = "sdpa", ) + # Handle FP8 models: redirect to BF16 sibling when the model ships with + # FP8 weights. Redirect is skipped when load_in_fp8 is truthy (True or 'block'). + model_name, model_config = _redirect_fp8_to_bf16( + model_name, + model_config, + load_in_fp8, + token, + trust_remote_code, + ) model_config.model_name = model_name model_max_seq_length = model_config.max_position_embeddings diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index e7296291f1..ec7a7a8046 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -965,11 +965,18 @@ __INT_TO_FLOAT_MAPPER = \ "mistralai/Magistral-Small-2506", "unsloth/Magistral-Small-2506-bnb-4bit", ), - "unsloth/Mistral-Small-3.2-24B-Instruct-2506-unsloth-bnb-4bit" : ( - "unsloth/Mistral-Small-3.2-24B-Instruct-2506", - "mistralai/Mistral-Small-3.2-24B-Instruct-2506", - "unsloth/Mistral-Small-3.2-24B-Instruct-2506-bnb-4bit", - ), + "unsloth/Mistral-Small-3.2-24B-Instruct-2506-unsloth-bnb-4bit" : { + "8" : ( + "mistralai/Mistral-Small-3.2-24B-Instruct-2506", + "unsloth/Mistral-Small-3.2-24B-Instruct-2506-FP8", + "unsloth/Mistral-Small-3.2-24B-Instruct-2506-FP8", + ), + "16" : ( + "unsloth/Mistral-Small-3.2-24B-Instruct-2506", + "mistralai/Mistral-Small-3.2-24B-Instruct-2506", + "unsloth/Mistral-Small-3.2-24B-Instruct-2506-bnb-4bit", + ), + }, "unsloth/gemma-3n-E4B-it-unsloth-bnb-4bit" : ( "unsloth/gemma-3n-E4B-it", "google/gemma-3n-E4B-it", @@ -1056,11 +1063,18 @@ __INT_TO_FLOAT_MAPPER = \ "mistralai/Magistral-Small-2507", "unsloth/Magistral-Small-2507-bnb-4bit", ), - "unsloth/Magistral-Small-2509-unsloth-bnb-4bit" : ( - "unsloth/Magistral-Small-2509", - "mistralai/Magistral-Small-2509", - "unsloth/Magistral-Small-2509-bnb-4bit", - ), + "unsloth/Magistral-Small-2509-unsloth-bnb-4bit" : { + "8" : ( + "mistralai/Magistral-Small-2509", + "unsloth/Magistral-Small-2509-FP8-Dynamic", + "unsloth/Magistral-Small-2509-FP8-Dynamic", + ), + "16" : ( + "unsloth/Magistral-Small-2509", + "mistralai/Magistral-Small-2509", + "unsloth/Magistral-Small-2509-bnb-4bit", + ), + }, "unsloth/Apertus-70B-Instruct-2509-unsloth-bnb-4bit" : ( "unsloth/Apertus-70B-Instruct-2509", "swiss-ai/Apertus-70B-2509", @@ -1256,6 +1270,73 @@ __INT_TO_FLOAT_MAPPER = \ "google/functiongemma-270m-it", "unsloth/functiongemma-270m-it-unsloth-bnb-4bit", ), + # Ministral 3 models + "unsloth/Ministral-3-3B-Instruct-2512-unsloth-bnb-4bit" : { + "8" : ( + "mistralai/Ministral-3-3B-Instruct-2512", + "unsloth/Ministral-3-3B-Instruct-2512-FP8", + "unsloth/Ministral-3-3B-Instruct-2512-FP8", + ), + "16" : ( + "unsloth/Ministral-3-3B-Instruct-2512", + "mistralai/Ministral-3-3B-Instruct-2512", + "unsloth/Ministral-3-3B-Instruct-2512-bnb-4bit", + ), + }, + "unsloth/Ministral-3-3B-Base-2512-unsloth-bnb-4bit" : ( + "unsloth/Ministral-3-3B-Base-2512", + "mistralai/Ministral-3-3B-Base-2512", + "unsloth/Ministral-3-3B-Base-2512-bnb-4bit", + ), + "unsloth/Ministral-3-3B-Reasoning-2512-unsloth-bnb-4bit" : ( + "unsloth/Ministral-3-3B-Reasoning-2512", + "mistralai/Ministral-3-3B-Reasoning-2512", + "unsloth/Ministral-3-3B-Reasoning-2512-bnb-4bit", + ), + "unsloth/Ministral-3-8B-Instruct-2512-unsloth-bnb-4bit" : { + "8" : ( + "mistralai/Ministral-3-8B-Instruct-2512", + "unsloth/Ministral-3-8B-Instruct-2512-FP8", + "unsloth/Ministral-3-8B-Instruct-2512-FP8", + ), + "16" : ( + "unsloth/Ministral-3-8B-Instruct-2512", + "mistralai/Ministral-3-8B-Instruct-2512", + "unsloth/Ministral-3-8B-Instruct-2512-bnb-4bit", + ), + }, + "unsloth/Ministral-3-8B-Base-2512-unsloth-bnb-4bit" : ( + "unsloth/Ministral-3-8B-Base-2512", + "mistralai/Ministral-3-8B-Base-2512", + "unsloth/Ministral-3-8B-Base-2512-bnb-4bit", + ), + "unsloth/Ministral-3-8B-Reasoning-2512-unsloth-bnb-4bit" : ( + "unsloth/Ministral-3-8B-Reasoning-2512", + "mistralai/Ministral-3-8B-Reasoning-2512", + "unsloth/Ministral-3-8B-Reasoning-2512-bnb-4bit", + ), + "unsloth/Ministral-3-14B-Instruct-2512-unsloth-bnb-4bit" : { + "8" : ( + "mistralai/Ministral-3-14B-Instruct-2512", + "unsloth/Ministral-3-14B-Instruct-2512-FP8", + "unsloth/Ministral-3-14B-Instruct-2512-FP8", + ), + "16" : ( + "unsloth/Ministral-3-14B-Instruct-2512", + "mistralai/Ministral-3-14B-Instruct-2512", + "unsloth/Ministral-3-14B-Instruct-2512-bnb-4bit", + ), + }, + "unsloth/Ministral-3-14B-Base-2512-unsloth-bnb-4bit" : ( + "unsloth/Ministral-3-14B-Base-2512", + "mistralai/Ministral-3-14B-Base-2512", + "unsloth/Ministral-3-14B-Base-2512-bnb-4bit", + ), + "unsloth/Ministral-3-14B-Reasoning-2512-unsloth-bnb-4bit" : ( + "unsloth/Ministral-3-14B-Reasoning-2512", + "mistralai/Ministral-3-14B-Reasoning-2512", + "unsloth/Ministral-3-14B-Reasoning-2512-bnb-4bit", + ), } INT_TO_FLOAT_MAPPER = {} diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 2d6c7ac93f..56cd615a49 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -30,6 +30,7 @@ from ..kernels import ( post_patch_loss_function, ) from ._utils import __version__, importlib_version, _prepare_model_for_qat +from ._utils import _redirect_fp8_to_bf16 from ._utils import * from .loader_utils import _get_fp8_mode_and_check_settings from ..save import patch_saving_functions @@ -611,6 +612,24 @@ class FastBaseModel: model_class = None flex_attn_impl = prefer_flex_attn_if_supported(model_class, auto_config) + # Handle FP8 models: redirect to BF16 sibling when the model ships with + # FP8 weights (e.g. Ministral-3-3B-Instruct-2512). FP8 weights cannot be + # directly loaded by BNB, and the FP8 quantization config can cause issues + # even for 16-bit loading. + # Redirect is skipped when load_in_fp8 is truthy (True or 'block'). + model_name, auto_config = _redirect_fp8_to_bf16( + model_name, + auto_config, + load_in_fp8, + token, + trust_remote_code, + ) + # Re-resolve model_class after potential config change + try: + model_class = auto_model._model_mapping[auto_config.__class__] + except KeyError: + pass + default_attn_impl = "flex_attention" if flex_attn_impl else "sdpa" if not ("attn_implementation" in kwargs): kwargs["attn_implementation"] = default_attn_impl @@ -759,6 +778,7 @@ class FastBaseModel: if hasattr(auto_config, "attn_implementation"): setattr(auto_config, "attn_implementation", config_attn_impl) model_config = auto_config + verify_fp8_support_if_applicable(model_config) raise_handler = RaiseUninitialized() @@ -767,6 +787,7 @@ class FastBaseModel: load_in_fp8 = kwargs.pop("load_in_fp8", None) model = auto_model.from_pretrained( model_name, + config = model_config, device_map = device_map, # torch_dtype = torch_dtype, # Transformers removed torch_dtype # quantization_config = bnb_config,