From f6509e6939759da9b1c7dfcfdcf3e902d97cc133 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 25 Nov 2025 06:48:10 -0800 Subject: [PATCH] Float8 --- unsloth/kernels/fp8.py | 5 + unsloth/models/loader.py | 72 +++--- unsloth/models/loader_utils.py | 52 ++++- unsloth/models/mapper.py | 387 ++++++++++++++++++++++++--------- 4 files changed, 370 insertions(+), 146 deletions(-) diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py index 8d077cac0d..6fae46a22b 100644 --- a/unsloth/kernels/fp8.py +++ b/unsloth/kernels/fp8.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os import torch import torch.nn as nn import triton @@ -549,6 +550,8 @@ def test_has_fbgemm(): return has_fbgemm fp8_block_quant_linear = fp8_torch_block_quant_forward +if "UNSLOTH_HAS_FBGEMM" not in os.environ: + os.environ["UNSLOTH_HAS_FBGEMM"] = "0" try: import fbgemm_gpu @@ -559,8 +562,10 @@ try: # We must manually confirm if blockwise FBGEMM works! # This check is a must for consumer grade GPUs which fail if test_has_fbgemm(): + os.environ["UNSLOTH_HAS_FBGEMM"] = "1" logger.info(f"Using fbgemm_gpu block quantized FP8 matmul") fp8_block_quant_linear = fp8_fbgemm_block_linear + else: except: pass diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 03c10cd9a0..8a5af95a14 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -145,7 +145,7 @@ class FastLanguageModel(FastLlamaModel): max_lora_rank = 64, disable_log_stats = True, qat_scheme = None, - load_in_fp8 = False, # fp8 LoRA + load_in_fp8 = False, # fp8 LoRA (True, False, 'block') *args, **kwargs, ): @@ -219,25 +219,24 @@ class FastLanguageModel(FastLlamaModel): ) load_in_4bit = False - if load_in_fp8: - fp8_mode = _get_fp8_mode_and_check_settings( - load_in_fp8, - fast_inference, - full_finetuning, - load_in_4bit, - load_in_8bit, - load_in_16bit, - use_exact_model_name, - ) - else: - fp8_mode = None - + # Find FP8, BnB 4bit, other mapped names old_model_name = model_name if not use_exact_model_name: - if load_in_fp8: + new_model_name = get_model_name(model_name, load_in_4bit = load_in_4bit, load_in_fp8 = load_in_fp8) + if new_model_name is None and load_in_fp8 != False: + fp8_mode = _get_fp8_mode_and_check_settings( + load_in_fp8, + fast_inference, + full_finetuning, + load_in_4bit, + load_in_8bit, + load_in_16bit, + use_exact_model_name, + ) model_name = _offline_quantize_to_fp8(model_name, fp8_mode) else: - model_name = get_model_name(model_name, load_in_4bit) + assert new_model_name is not None + model_name = new_model_name # Check if pre-quantized models are allowed # For eg AMD GPUs need blocksize = 128, but our pre-quants are blocksize = 64 @@ -378,6 +377,7 @@ class FastLanguageModel(FastLlamaModel): if model_name.lower().endswith("-bf16"): load_in_4bit = False load_in_8bit = False + load_in_fp8 = False load_in_16bit = True model_config = AutoConfig.from_pretrained( @@ -580,7 +580,7 @@ class FastLanguageModel(FastLlamaModel): } model.config.update({"quantization_config": quantization_config}) - if load_in_fp8: + if load_in_fp8 != False: _tag_model_with_fp8_torchao_config(model, fp8_mode) if is_peft: @@ -663,7 +663,7 @@ class FastModel(FastBaseModel): max_lora_rank = 64, disable_log_stats = True, qat_scheme = None, - load_in_fp8 = False, # fp8 LoRA + load_in_fp8 = False, # fp8 LoRA (True, False, 'block') *args, **kwargs, ): @@ -690,6 +690,7 @@ class FastModel(FastBaseModel): ) dtype = torch.float16 assert dtype in (torch.float16, torch.bfloat16, torch.float32) + assert load_in_fp8 in (True, False, "block") patch_compiled_autograd() patch_compiling_bitsandbytes() @@ -724,25 +725,24 @@ class FastModel(FastBaseModel): ) load_in_4bit = False - if load_in_fp8: - fp8_mode = _get_fp8_mode_and_check_settings( - load_in_fp8, - fast_inference, - full_finetuning, - load_in_4bit, - load_in_8bit, - load_in_16bit, - use_exact_model_name, - ) - else: - fp8_mode = None - + # Find FP8, BnB 4bit, other mapped names old_model_name = model_name if not use_exact_model_name: - if load_in_fp8: + new_model_name = get_model_name(model_name, load_in_4bit = load_in_4bit, load_in_fp8 = load_in_fp8) + if new_model_name is None and load_in_fp8 != False: + fp8_mode = _get_fp8_mode_and_check_settings( + load_in_fp8, + fast_inference, + full_finetuning, + load_in_4bit, + load_in_8bit, + load_in_16bit, + use_exact_model_name, + ) model_name = _offline_quantize_to_fp8(model_name, fp8_mode) else: - model_name = get_model_name(model_name, load_in_4bit) + assert new_model_name is not None + model_name = new_model_name # Check if pre-quantized models are allowed # For eg AMD GPUs need blocksize = 128, but our pre-quants are blocksize = 64 @@ -755,6 +755,7 @@ class FastModel(FastBaseModel): if model_name.lower().endswith("-bf16"): load_in_4bit = False load_in_8bit = False + load_in_fp8 = False load_in_16bit = True # Check modelscope @@ -833,6 +834,8 @@ class FastModel(FastBaseModel): string += "_load_in_8bit_" if load_in_16bit: string += "_load_in_16bit_" + if load_in_fp8: + string += "load_in_fp8" os.environ["UNSLOTH_MODEL_NAME"] = string # Check versions @@ -1013,6 +1016,7 @@ class FastModel(FastBaseModel): if model_name.lower().endswith("-bf16"): load_in_4bit = False load_in_8bit = False + load_in_fp8 = False load_in_16bit = True model_config = AutoConfig.from_pretrained( @@ -1177,7 +1181,7 @@ class FastModel(FastBaseModel): } model.config.update({"quantization_config": quantization_config}) - if load_in_fp8: + if load_in_fp8 != False: _tag_model_with_fp8_torchao_config(model, fp8_mode) if is_peft: diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index bb6a5b5cc0..af31c9dcad 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -17,7 +17,13 @@ import os import re import tempfile from typing import Union -from .mapper import INT_TO_FLOAT_MAPPER, FLOAT_TO_INT_MAPPER, MAP_TO_UNSLOTH_16bit +from .mapper import ( + INT_TO_FLOAT_MAPPER, + FLOAT_TO_INT_MAPPER, + MAP_TO_UNSLOTH_16bit, + FLOAT_TO_FP8_BLOCK_MAPPER, + FLOAT_TO_FP8_ROW_MAPPER, +) # https://github.com/huggingface/transformers/pull/26037 allows 4 bit loading! from packaging.version import Version @@ -45,11 +51,28 @@ def __get_model_name( INT_TO_FLOAT_MAPPER = None, FLOAT_TO_INT_MAPPER = None, MAP_TO_UNSLOTH_16bit = None, + load_in_fp8 = False, + FLOAT_TO_FP8_BLOCK_MAPPER = None, + FLOAT_TO_FP8_ROW_MAPPER = None, ): model_name = str(model_name) lower_model_name = model_name.lower() - if not SUPPORTS_FOURBIT and lower_model_name in INT_TO_FLOAT_MAPPER: + assert load_in_fp8 in (True, False, "block") + if load_in_fp8 != False: + if load_in_fp8 == True and (os.environ.get("UNSLOTH_HAS_FBGEMM", "0") == "1"): + if lower_model_name in FLOAT_TO_FP8_ROW_MAPPER: + # Faster row scaling only works if FBGEMM works! + return FLOAT_TO_FP8_ROW_MAPPER[lower_model_name] + elif lower_model_name in FLOAT_TO_FP8_BLOCK_MAPPER: + # Otherwise we use the slower blockwise type + return FLOAT_TO_FP8_BLOCK_MAPPER[lower_model_name] + else: + if lower_model_name in FLOAT_TO_FP8_BLOCK_MAPPER: + return FLOAT_TO_FP8_BLOCK_MAPPER[lower_model_name] + return None + + elif not SUPPORTS_FOURBIT and lower_model_name in INT_TO_FLOAT_MAPPER: model_name = INT_TO_FLOAT_MAPPER[lower_model_name] print( f"Unsloth: Your transformers version of {transformers_version} does not support native " @@ -112,13 +135,17 @@ def _get_new_mapper(): return {}, {}, {} -def get_model_name(model_name, load_in_4bit = True): +def get_model_name(model_name, load_in_4bit = True, load_in_fp8 = False): + assert load_in_fp8 in (True, False, "block") new_model_name = __get_model_name( model_name = model_name, load_in_4bit = load_in_4bit, INT_TO_FLOAT_MAPPER = INT_TO_FLOAT_MAPPER, FLOAT_TO_INT_MAPPER = FLOAT_TO_INT_MAPPER, MAP_TO_UNSLOTH_16bit = MAP_TO_UNSLOTH_16bit, + load_in_fp8 = load_in_fp8, + FLOAT_TO_FP8_BLOCK_MAPPER = FLOAT_TO_FP8_BLOCK_MAPPER, + FLOAT_TO_FP8_ROW_MAPPER = FLOAT_TO_FP8_ROW_MAPPER, ) # In the rare case, we convert bad model names to other names # For eg too large dynamic quants or MoEs @@ -144,6 +171,9 @@ def get_model_name(model_name, load_in_4bit = True): INT_TO_FLOAT_MAPPER = NEW_INT_TO_FLOAT_MAPPER, FLOAT_TO_INT_MAPPER = NEW_FLOAT_TO_INT_MAPPER, MAP_TO_UNSLOTH_16bit = NEW_MAP_TO_UNSLOTH_16bit, + load_in_fp8 = load_in_fp8, + FLOAT_TO_FP8_BLOCK_MAPPER = FLOAT_TO_FP8_BLOCK_MAPPER, + FLOAT_TO_FP8_ROW_MAPPER = FLOAT_TO_FP8_ROW_MAPPER, ) if upgraded_model_name is not None: raise NotImplementedError( @@ -152,6 +182,9 @@ def get_model_name(model_name, load_in_4bit = True): 'pip install --upgrade --no-cache-dir "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"\n' 'pip install --upgrade --no-cache-dir "git+https://github.com/unslothai/unsloth-zoo.git"\n' ) + if load_in_fp8 != False: + # Handle on the fly TorchAO FP8 quantization + return new_model_name return new_model_name if new_model_name is not None else model_name @@ -239,11 +272,14 @@ def _tag_model_with_fp8_torchao_config(model: torch.nn.Module, fp8_mode: str): """ Tag a model with a `TorchAOConfig` so downstream callers will know what to do with it. """ - base_config = _get_torchao_fp8_config(fp8_mode) - model.torchao_config = TorchAOConfig( - qat_scheme = None, - base_config_and_filter_fns = [(base_config, None)], - ) + try: + base_config = _get_torchao_fp8_config(fp8_mode) + model.torchao_config = TorchAOConfig( + qat_scheme = None, + base_config_and_filter_fns = [(base_config, None)], + ) + except: + pass def _get_fp8_mode_and_check_settings( diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index f2dad24225..c81480355b 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -15,6 +15,9 @@ __all__ = [ "INT_TO_FLOAT_MAPPER", "FLOAT_TO_INT_MAPPER", + "MAP_TO_UNSLOTH_16bit", + "FLOAT_TO_FP8_BLOCK_MAPPER", + "FLOAT_TO_FP8_ROW_MAPPER", ] __INT_TO_FLOAT_MAPPER = \ @@ -233,21 +236,35 @@ __INT_TO_FLOAT_MAPPER = \ "meta-llama/Meta-Llama-3.1-8B", "unsloth/Meta-Llama-3.1-8B-bnb-4bit", ), - "unsloth/Meta-Llama-3.1-8B-Instruct-unsloth-bnb-4bit" : ( - "unsloth/Meta-Llama-3.1-8B-Instruct", - "meta-llama/Meta-Llama-3.1-8B-Instruct", - "unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit", - ), + "unsloth/Meta-Llama-3.1-8B-Instruct-unsloth-bnb-4bit" : { + "8" : ( + "RedHatAI/Llama-3.1-8B-Instruct-FP8", + "unsloth/Llama-3.1-8B-Instruct-FP8-Block", + "unsloth/Llama-3.1-8B-Instruct-FP8-Dynamic", + ), + "16" : ( + "unsloth/Meta-Llama-3.1-8B-Instruct", + "meta-llama/Meta-Llama-3.1-8B-Instruct", + "unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit", + ), + }, "unsloth/Llama-3.1-8B-unsloth-bnb-4bit" : ( "unsloth/Llama-3.1-8B", "meta-llama/Llama-3.1-8B", "unsloth/Llama-3.1-8B-bnb-4bit", ), - "unsloth/Llama-3.1-8B-Instruct-unsloth-bnb-4bit" : ( - "unsloth/Llama-3.1-8B-Instruct", - "meta-llama/Llama-3.1-8B-Instruct", - "unsloth/Llama-3.1-8B-Instruct-bnb-4bit", - ), + "unsloth/Llama-3.1-8B-Instruct-unsloth-bnb-4bit" : { + "8" : ( + "RedHatAI/Llama-3.1-8B-Instruct-FP8", + "unsloth/Llama-3.1-8B-Instruct-FP8-Block", + "unsloth/Llama-3.1-8B-Instruct-FP8-Dynamic", + ), + "16" : ( + "unsloth/Llama-3.1-8B-Instruct", + "meta-llama/Llama-3.1-8B-Instruct", + "unsloth/Llama-3.1-8B-Instruct-bnb-4bit", + ), + }, "unsloth/Meta-Llama-3.1-70B-bnb-4bit" : ( "unsloth/Meta-Llama-3.1-70B", "meta-llama/Meta-Llama-3.1-70B", @@ -474,16 +491,30 @@ __INT_TO_FLOAT_MAPPER = \ "meta-llama/Llama-3.2-3B", "unsloth/Llama-3.2-3B-bnb-4bit", ), - "unsloth/Llama-3.2-1B-Instruct-unsloth-bnb-4bit" : ( - "unsloth/Llama-3.2-1B-Instruct", - "meta-llama/Llama-3.2-1B-Instruct", - "unsloth/Llama-3.2-1B-Instruct-bnb-4bit", - ), - "unsloth/Llama-3.2-3B-Instruct-unsloth-bnb-4bit" : ( - "unsloth/Llama-3.2-3B-Instruct", - "meta-llama/Llama-3.2-3B-Instruct", - "unsloth/Llama-3.2-3B-Instruct-bnb-4bit", - ), + "unsloth/Llama-3.2-1B-Instruct-unsloth-bnb-4bit" : { + "8": ( + "RedHatAI/Llama-3.2-1B-Instruct-FP8", + "unsloth/Llama-3.2-1B-Instruct-FP8-Block", + "unsloth/Llama-3.2-1B-Instruct-FP8-Dynamic", + ), + "16" : ( + "unsloth/Llama-3.2-1B-Instruct", + "meta-llama/Llama-3.2-1B-Instruct", + "unsloth/Llama-3.2-1B-Instruct-bnb-4bit", + ), + }, + "unsloth/Llama-3.2-3B-Instruct-unsloth-bnb-4bit" : { + "8": ( + "RedHatAI/Llama-3.2-3B-Instruct-FP8", + "unsloth/Llama-3.2-3B-Instruct-FP8-Block", + "unsloth/Llama-3.2-3B-Instruct-FP8-Dynamic", + ), + "16" : ( + "unsloth/Llama-3.2-3B-Instruct", + "meta-llama/Llama-3.2-3B-Instruct", + "unsloth/Llama-3.2-3B-Instruct-bnb-4bit", + ), + }, "unsloth/Llama-3.1-Nemotron-70B-Instruct-bnb-4bit" : ( "unsloth/Llama-3.1-Nemotron-70B-Instruct", "nvidia/Llama-3.1-Nemotron-70B-Instruct-HF", @@ -561,10 +592,18 @@ __INT_TO_FLOAT_MAPPER = \ "unsloth/QwQ-32B-Preview", "Qwen/QwQ-32B-Preview", ), - "unsloth/Llama-3.3-70B-Instruct-bnb-4bit" : ( - "unsloth/Llama-3.3-70B-Instruct", - "meta-llama/Llama-3.3-70B-Instruct", - ), + "unsloth/Llama-3.3-70B-Instruct-unsloth-bnb-4bit" : { + "8" : ( + "RedHatAI/Llama-3.3-70B-Instruct-FP8", + "unsloth/Llama-3.3-70B-Instruct-FP8-Block", + "unsloth/Llama-3.3-70B-Instruct-FP8-Dynamic", + ), + "16" : ( + "unsloth/Llama-3.3-70B-Instruct", + "meta-llama/Llama-3.3-70B-Instruct", + "unsloth/Llama-3.3-70B-Instruct-bnb-4bit", + ), + }, "unsloth/phi-4-unsloth-bnb-4bit" : ( "unsloth/phi-4", "microsoft/phi-4", @@ -733,36 +772,78 @@ __INT_TO_FLOAT_MAPPER = \ "mistralai/Mistral-Small-3.1-24B-Base-2503", "unsloth/Mistral-Small-3.1-24B-Base-2503-bnb-4bit", ), - "unsloth/Qwen3-0.6B-unsloth-bnb-4bit" : ( - "unsloth/Qwen3-0.6B", - "Qwen/Qwen3-0.6B", - "unsloth/Qwen3-0.6B-bnb-4bit", - ), - "unsloth/Qwen3-1.7B-unsloth-bnb-4bit" : ( - "unsloth/Qwen3-1.7B", - "Qwen/Qwen3-1.7B", - "unsloth/Qwen3-1.7B-bnb-4bit", - ), - "unsloth/Qwen3-4B-unsloth-bnb-4bit" : ( - "unsloth/Qwen3-4B", - "Qwen/Qwen3-4B", - "unsloth/Qwen3-4B-bnb-4bit", - ), - "unsloth/Qwen3-8B-unsloth-bnb-4bit" : ( - "unsloth/Qwen3-8B", - "Qwen/Qwen3-8B", - "unsloth/Qwen3-8B-bnb-4bit", - ), - "unsloth/Qwen3-14B-unsloth-bnb-4bit" : ( - "unsloth/Qwen3-14B", - "Qwen/Qwen3-14B", - "unsloth/Qwen3-14B-bnb-4bit", - ), - "unsloth/Qwen3-32B-unsloth-bnb-4bit" : ( - "unsloth/Qwen3-32B", - "Qwen/Qwen3-32B", - "unsloth/Qwen3-32B-bnb-4bit", - ), + "unsloth/Qwen3-0.6B-unsloth-bnb-4bit" : { + "8" : ( + "Qwen/Qwen3-0.6B-FP8", + "unsloth/Qwen3-0.6B-FP8", + "unsloth/Qwen3-0.6B-FP8", + ), + "16" : ( + "unsloth/Qwen3-0.6B", + "Qwen/Qwen3-0.6B", + "unsloth/Qwen3-0.6B-bnb-4bit", + ), + }, + "unsloth/Qwen3-1.7B-unsloth-bnb-4bit" : { + "8" : ( + "Qwen/Qwen3-1.7B-FP8", + "unsloth/Qwen3-1.7B-FP8", + "unsloth/Qwen3-1.7B-FP8", + ), + "16" : ( + "unsloth/Qwen3-1.7B", + "Qwen/Qwen3-1.7B", + "unsloth/Qwen3-1.7B-bnb-4bit", + ), + }, + "unsloth/Qwen3-4B-unsloth-bnb-4bit" : { + "8" : ( + "Qwen/Qwen3-4B-FP8", + "unsloth/Qwen3-4B-FP8", + "unsloth/Qwen3-4B-FP8", + ), + "16" : ( + "unsloth/Qwen3-4B", + "Qwen/Qwen3-4B", + "unsloth/Qwen3-4B-bnb-4bit", + ), + }, + "unsloth/Qwen3-8B-unsloth-bnb-4bit" : { + "8" : ( + "Qwen/Qwen3-8B-FP8", + "unsloth/Qwen3-8B-FP8", + "unsloth/Qwen3-8B-FP8", + ), + "16" : ( + "unsloth/Qwen3-8B", + "Qwen/Qwen3-8B", + "unsloth/Qwen3-8B-bnb-4bit", + ), + }, + "unsloth/Qwen3-14B-unsloth-bnb-4bit" : { + "8" : ( + "Qwen/Qwen3-14B-FP8", + "unsloth/Qwen3-14B-FP8", + "unsloth/Qwen3-14B-FP8", + ), + "16" : ( + "unsloth/Qwen3-14B", + "Qwen/Qwen3-14B", + "unsloth/Qwen3-14B-bnb-4bit", + ), + }, + "unsloth/Qwen3-32B-unsloth-bnb-4bit" : { + "8" : ( + "Qwen/Qwen3-32B-FP8", + "unsloth/Qwen3-32B-FP8", + "unsloth/Qwen3-32B-FP8", + ), + "16" : ( + "unsloth/Qwen3-32B", + "Qwen/Qwen3-32B", + "unsloth/Qwen3-32B-bnb-4bit", + ), + }, "unsloth/Qwen3-30B-A3B-unsloth-bnb-4bit" : ( "unsloth/Qwen3-30B-A3B", "Qwen/Qwen3-30B-A3B", @@ -936,16 +1017,30 @@ __INT_TO_FLOAT_MAPPER = \ "openai/gpt-oss-120b", "unsloth/gpt-oss-120b-unsloth-bnb-4bit", ), - "unsloth/Qwen3-4B-Instruct-2507-unsloth-bnb-4bit" : ( - "unsloth/Qwen3-4B-Instruct-2507", - "Qwen/Qwen3-4B-Instruct-2507", - "unsloth/Qwen3-4B-Instruct-2507-bnb-4bit", - ), - "unsloth/Qwen3-4B-Thinking-2507-unsloth-bnb-4bit" : ( - "unsloth/Qwen3-4B-Thinking-2507", - "Qwen/Qwen3-4B-Thinking-2507", - "unsloth/Qwen3-4B-Thinking-2507-bnb-4bit", - ), + "unsloth/Qwen3-4B-Instruct-2507-unsloth-bnb-4bit" : { + "8" : ( + "Qwen/Qwen3-4B-Instruct-2507-FP8", + "unsloth/Qwen3-4B-Instruct-2507-FP8", + "unsloth/Qwen3-4B-Instruct-2507-FP8", + ), + "16" : ( + "unsloth/Qwen3-4B-Instruct-2507", + "Qwen/Qwen3-4B-Instruct-2507", + "unsloth/Qwen3-4B-Instruct-2507-bnb-4bit", + ), + }, + "unsloth/Qwen3-4B-Thinking-2507-unsloth-bnb-4bit" : { + "8" : ( + "Qwen/Qwen3-4B-Thinking-2507-FP8", + "unsloth/Qwen3-4B-Thinking-2507-FP8", + "unsloth/Qwen3-4B-Thinking-2507-FP8", + ), + "16" : ( + "unsloth/Qwen3-4B-Thinking-2507", + "Qwen/Qwen3-4B-Thinking-2507", + "unsloth/Qwen3-4B-Thinking-2507-bnb-4bit", + ), + }, "unsloth/gemma-3-270m-it-unsloth-bnb-4bit" : ( "unsloth/gemma-3-270m-it", "google/gemma-3-270m-it", @@ -1012,46 +1107,102 @@ __INT_TO_FLOAT_MAPPER = \ "unsloth/granite-4.0-h-small-base", "ibm-granite/granite-4.0-h-small-base", ), - "unsloth/Qwen3-VL-4B-Thinking-unsloth-bnb-4bit" : ( - "unsloth/Qwen3-VL-4B-Thinking", - "Qwen/Qwen3-VL-4B-Thinking", - "unsloth/Qwen3-VL-4B-Thinking-bnb-4bit", - ), - "unsloth/Qwen3-VL-8B-Thinking-unsloth-bnb-4bit" : ( - "unsloth/Qwen3-VL-8B-Thinking", - "Qwen/Qwen3-VL-8B-Thinking", - "unsloth/Qwen3-VL-8B-Thinking-bnb-4bit", - ), - "unsloth/Qwen3-VL-4B-Instruct-unsloth-bnb-4bit" : ( - "unsloth/Qwen3-VL-4B-Instruct", - "Qwen/Qwen3-VL-4B-Instruct", - "unsloth/Qwen3-VL-4B-Instruct-bnb-4bit", - ), - "unsloth/Qwen3-VL-8B-Instruct-unsloth-bnb-4bit" : ( - "unsloth/Qwen3-VL-8B-Instruct", - "Qwen/Qwen3-VL-8B-Instruct", - "unsloth/Qwen3-VL-8B-Instruct-bnb-4bit", - ), - "unsloth/Qwen3-VL-2B-Thinking-unsloth-bnb-4bit" : ( - "unsloth/Qwen3-VL-2B-Thinking", - "Qwen/Qwen3-VL-2B-Thinking", - "unsloth/Qwen3-VL-2B-Thinking-bnb-4bit", - ), - "unsloth/Qwen3-VL-32B-Thinking-unsloth-bnb-4bit" : ( - "unsloth/Qwen3-VL-32B-Thinking", - "Qwen/Qwen3-VL-32B-Thinking", - "unsloth/Qwen3-VL-32B-Thinking-bnb-4bit", - ), - "unsloth/Qwen3-VL-2B-Instruct-unsloth-bnb-4bit" : ( - "unsloth/Qwen3-VL-2B-Instruct", - "Qwen/Qwen3-VL-2B-Instruct", - "unsloth/Qwen3-VL-2B-Instruct-bnb-4bit", - ), - "unsloth/Qwen3-VL-32B-Instruct-unsloth-bnb-4bit" : ( - "unsloth/Qwen3-VL-32B-Instruct", - "Qwen/Qwen3-VL-32B-Instruct", - "unsloth/Qwen3-VL-32B-Instruct-bnb-4bit", - ), + "unsloth/Qwen3-VL-4B-Thinking-unsloth-bnb-4bit" : { + "8" : ( + "Qwen/Qwen3-VL-4B-Thinking-FP8", + "unsloth/Qwen3-VL-4B-Thinking-FP8", + "unsloth/Qwen3-VL-4B-Thinking-FP8", + ), + "16" : ( + "unsloth/Qwen3-VL-4B-Thinking", + "Qwen/Qwen3-VL-4B-Thinking", + "unsloth/Qwen3-VL-4B-Thinking-bnb-4bit", + ), + }, + "unsloth/Qwen3-VL-8B-Thinking-unsloth-bnb-4bit" : { + "8" : ( + "Qwen/Qwen3-VL-8B-Thinking-FP8", + "unsloth/Qwen3-VL-8B-Thinking-FP8", + "unsloth/Qwen3-VL-8B-Thinking-FP8", + ), + "16" : ( + "unsloth/Qwen3-VL-8B-Thinking", + "Qwen/Qwen3-VL-8B-Thinking", + "unsloth/Qwen3-VL-8B-Thinking-bnb-4bit", + ), + }, + "unsloth/Qwen3-VL-4B-Instruct-unsloth-bnb-4bit" : { + "8" : ( + "Qwen/Qwen3-VL-4B-Instruct-FP8", + "unsloth/Qwen3-VL-4B-Instruct-FP8", + "unsloth/Qwen3-VL-4B-Instruct-FP8", + ), + "16" : ( + "unsloth/Qwen3-VL-4B-Instruct", + "Qwen/Qwen3-VL-4B-Instruct", + "unsloth/Qwen3-VL-4B-Instruct-bnb-4bit", + ), + }, + "unsloth/Qwen3-VL-8B-Instruct-unsloth-bnb-4bit" : { + "8" : ( + "Qwen/Qwen3-VL-8B-Instruct-FP8", + "unsloth/Qwen3-VL-8B-Instruct-FP8", + "unsloth/Qwen3-VL-8B-Instruct-FP8", + ), + "16" : ( + "unsloth/Qwen3-VL-8B-Instruct", + "Qwen/Qwen3-VL-8B-Instruct", + "unsloth/Qwen3-VL-8B-Instruct-bnb-4bit", + ), + }, + "unsloth/Qwen3-VL-2B-Thinking-unsloth-bnb-4bit" : { + "8" : ( + "Qwen/Qwen3-VL-2B-Thinking-FP8", + "unsloth/Qwen3-VL-2B-Thinking-FP8", + "unsloth/Qwen3-VL-2B-Thinking-FP8", + ), + "16" : ( + "unsloth/Qwen3-VL-2B-Thinking", + "Qwen/Qwen3-VL-2B-Thinking", + "unsloth/Qwen3-VL-2B-Thinking-bnb-4bit", + ), + }, + "unsloth/Qwen3-VL-32B-Thinking-unsloth-bnb-4bit" : { + "8" : ( + "Qwen/Qwen3-VL-32B-Thinking-FP8", + "unsloth/Qwen3-VL-32B-Thinking-FP8", + "unsloth/Qwen3-VL-32B-Thinking-FP8", + ), + "16" : ( + "unsloth/Qwen3-VL-32B-Thinking", + "Qwen/Qwen3-VL-32B-Thinking", + "unsloth/Qwen3-VL-32B-Thinking-bnb-4bit", + ), + }, + "unsloth/Qwen3-VL-2B-Instruct-unsloth-bnb-4bit" : { + "8" : ( + "Qwen/Qwen3-VL-2B-Instruct-FP8", + "unsloth/Qwen3-VL-2B-Instruct-FP8", + "unsloth/Qwen3-VL-2B-Instruct-FP8", + ), + "16" : ( + "unsloth/Qwen3-VL-2B-Instruct", + "Qwen/Qwen3-VL-2B-Instruct", + "unsloth/Qwen3-VL-2B-Instruct-bnb-4bit", + ), + }, + "unsloth/Qwen3-VL-32B-Instruct-unsloth-bnb-4bit" : { + "8" : ( + "Qwen/Qwen3-VL-32B-Instruct-FP8", + "unsloth/Qwen3-VL-32B-Instruct-FP8", + "unsloth/Qwen3-VL-32B-Instruct-FP8", + ), + "16" : ( + "unsloth/Qwen3-VL-32B-Instruct", + "Qwen/Qwen3-VL-32B-Instruct", + "unsloth/Qwen3-VL-32B-Instruct-bnb-4bit", + ), + }, "unsloth/granite-4.0-350m-base-unsloth-bnb-4bit" : ( "unsloth/granite-4.0-350m-base-unsloth-bnb-4bit", "ibm-granite/granite-4.0-350m-base", @@ -1105,8 +1256,28 @@ __INT_TO_FLOAT_MAPPER = \ INT_TO_FLOAT_MAPPER = {} FLOAT_TO_INT_MAPPER = {} MAP_TO_UNSLOTH_16bit = {} +FLOAT_TO_FP8_BLOCK_MAPPER = {} +FLOAT_TO_FP8_ROW_MAPPER = {} for key, values in __INT_TO_FLOAT_MAPPER.items(): + block, row = None, None + if type(values) is dict: + assert "16" in values + # Float8 and other quantized types + if "8" in values: + float8_values = values["8"] + assert len(float8_values) == 3 + official, block, row = float8_values + FLOAT_TO_FP8_BLOCK_MAPPER[key.lower()] = block + FLOAT_TO_FP8_ROW_MAPPER[key.lower()] = row + FLOAT_TO_FP8_BLOCK_MAPPER[official.lower() + "-dynamic"] = block + FLOAT_TO_FP8_ROW_MAPPER[official.lower()] = row + FLOAT_TO_FP8_ROW_MAPPER[official.lower() + "-dynamic"] = row + for k in float8_values: + FLOAT_TO_FP8_BLOCK_MAPPER[k.lower()] = block + FLOAT_TO_FP8_ROW_MAPPER[k.lower()] = row + pass + values = values["16"] INT_TO_FLOAT_MAPPER[key] = values[0] for value in values: @@ -1117,6 +1288,10 @@ for key, values in __INT_TO_FLOAT_MAPPER.items(): if values[0].startswith("unsloth"): MAP_TO_UNSLOTH_16bit[values[1]] = values[0] MAP_TO_UNSLOTH_16bit[values[1].lower()] = values[0] + MAP_TO_UNSLOTH_16bit[block] = values[0] + MAP_TO_UNSLOTH_16bit[block.lower()] = values[0] + MAP_TO_UNSLOTH_16bit[row] = values[0] + MAP_TO_UNSLOTH_16bit[row.lower()] = values[0] elif len(values) == 3: # Dynamic Unsloth quantization if values[0].startswith("unsloth"): @@ -1124,6 +1299,10 @@ for key, values in __INT_TO_FLOAT_MAPPER.items(): MAP_TO_UNSLOTH_16bit[values[1].lower()] = values[0] MAP_TO_UNSLOTH_16bit[values[2]] = values[0] MAP_TO_UNSLOTH_16bit[values[2].lower()] = values[0] + MAP_TO_UNSLOTH_16bit[block] = values[0] + MAP_TO_UNSLOTH_16bit[block.lower()] = values[0] + MAP_TO_UNSLOTH_16bit[row] = values[0] + MAP_TO_UNSLOTH_16bit[row.lower()] = values[0] pass # Get lowercased