Float8
This commit is contained in:
Daniel Han
parent
fee3841c24
commit
cb00618e49
4 changed files with 370 additions and 146 deletions
|
|
@ -11,6 +11,7 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import os
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import triton
|
||||
|
|
@ -549,6 +550,8 @@ def test_has_fbgemm():
|
|||
return has_fbgemm
|
||||
|
||||
fp8_block_quant_linear = fp8_torch_block_quant_forward
|
||||
if "UNSLOTH_HAS_FBGEMM" not in os.environ:
|
||||
os.environ["UNSLOTH_HAS_FBGEMM"] = "0"
|
||||
try:
|
||||
import fbgemm_gpu
|
||||
|
||||
|
|
@ -559,8 +562,10 @@ try:
|
|||
# We must manually confirm if blockwise FBGEMM works!
|
||||
# This check is a must for consumer grade GPUs which fail
|
||||
if test_has_fbgemm():
|
||||
os.environ["UNSLOTH_HAS_FBGEMM"] = "1"
|
||||
logger.info(f"Using fbgemm_gpu block quantized FP8 matmul")
|
||||
fp8_block_quant_linear = fp8_fbgemm_block_linear
|
||||
else:
|
||||
except:
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -145,7 +145,7 @@ class FastLanguageModel(FastLlamaModel):
|
|||
max_lora_rank = 64,
|
||||
disable_log_stats = True,
|
||||
qat_scheme = None,
|
||||
load_in_fp8 = False, # fp8 LoRA
|
||||
load_in_fp8 = False, # fp8 LoRA (True, False, 'block')
|
||||
*args,
|
||||
**kwargs,
|
||||
):
|
||||
|
|
@ -219,25 +219,24 @@ class FastLanguageModel(FastLlamaModel):
|
|||
)
|
||||
load_in_4bit = False
|
||||
|
||||
if load_in_fp8:
|
||||
fp8_mode = _get_fp8_mode_and_check_settings(
|
||||
load_in_fp8,
|
||||
fast_inference,
|
||||
full_finetuning,
|
||||
load_in_4bit,
|
||||
load_in_8bit,
|
||||
load_in_16bit,
|
||||
use_exact_model_name,
|
||||
)
|
||||
else:
|
||||
fp8_mode = None
|
||||
|
||||
# Find FP8, BnB 4bit, other mapped names
|
||||
old_model_name = model_name
|
||||
if not use_exact_model_name:
|
||||
if load_in_fp8:
|
||||
new_model_name = get_model_name(model_name, load_in_4bit = load_in_4bit, load_in_fp8 = load_in_fp8)
|
||||
if new_model_name is None and load_in_fp8 != False:
|
||||
fp8_mode = _get_fp8_mode_and_check_settings(
|
||||
load_in_fp8,
|
||||
fast_inference,
|
||||
full_finetuning,
|
||||
load_in_4bit,
|
||||
load_in_8bit,
|
||||
load_in_16bit,
|
||||
use_exact_model_name,
|
||||
)
|
||||
model_name = _offline_quantize_to_fp8(model_name, fp8_mode)
|
||||
else:
|
||||
model_name = get_model_name(model_name, load_in_4bit)
|
||||
assert new_model_name is not None
|
||||
model_name = new_model_name
|
||||
|
||||
# Check if pre-quantized models are allowed
|
||||
# For eg AMD GPUs need blocksize = 128, but our pre-quants are blocksize = 64
|
||||
|
|
@ -378,6 +377,7 @@ class FastLanguageModel(FastLlamaModel):
|
|||
if model_name.lower().endswith("-bf16"):
|
||||
load_in_4bit = False
|
||||
load_in_8bit = False
|
||||
load_in_fp8 = False
|
||||
load_in_16bit = True
|
||||
|
||||
model_config = AutoConfig.from_pretrained(
|
||||
|
|
@ -580,7 +580,7 @@ class FastLanguageModel(FastLlamaModel):
|
|||
}
|
||||
model.config.update({"quantization_config": quantization_config})
|
||||
|
||||
if load_in_fp8:
|
||||
if load_in_fp8 != False:
|
||||
_tag_model_with_fp8_torchao_config(model, fp8_mode)
|
||||
|
||||
if is_peft:
|
||||
|
|
@ -663,7 +663,7 @@ class FastModel(FastBaseModel):
|
|||
max_lora_rank = 64,
|
||||
disable_log_stats = True,
|
||||
qat_scheme = None,
|
||||
load_in_fp8 = False, # fp8 LoRA
|
||||
load_in_fp8 = False, # fp8 LoRA (True, False, 'block')
|
||||
*args,
|
||||
**kwargs,
|
||||
):
|
||||
|
|
@ -690,6 +690,7 @@ class FastModel(FastBaseModel):
|
|||
)
|
||||
dtype = torch.float16
|
||||
assert dtype in (torch.float16, torch.bfloat16, torch.float32)
|
||||
assert load_in_fp8 in (True, False, "block")
|
||||
|
||||
patch_compiled_autograd()
|
||||
patch_compiling_bitsandbytes()
|
||||
|
|
@ -724,25 +725,24 @@ class FastModel(FastBaseModel):
|
|||
)
|
||||
load_in_4bit = False
|
||||
|
||||
if load_in_fp8:
|
||||
fp8_mode = _get_fp8_mode_and_check_settings(
|
||||
load_in_fp8,
|
||||
fast_inference,
|
||||
full_finetuning,
|
||||
load_in_4bit,
|
||||
load_in_8bit,
|
||||
load_in_16bit,
|
||||
use_exact_model_name,
|
||||
)
|
||||
else:
|
||||
fp8_mode = None
|
||||
|
||||
# Find FP8, BnB 4bit, other mapped names
|
||||
old_model_name = model_name
|
||||
if not use_exact_model_name:
|
||||
if load_in_fp8:
|
||||
new_model_name = get_model_name(model_name, load_in_4bit = load_in_4bit, load_in_fp8 = load_in_fp8)
|
||||
if new_model_name is None and load_in_fp8 != False:
|
||||
fp8_mode = _get_fp8_mode_and_check_settings(
|
||||
load_in_fp8,
|
||||
fast_inference,
|
||||
full_finetuning,
|
||||
load_in_4bit,
|
||||
load_in_8bit,
|
||||
load_in_16bit,
|
||||
use_exact_model_name,
|
||||
)
|
||||
model_name = _offline_quantize_to_fp8(model_name, fp8_mode)
|
||||
else:
|
||||
model_name = get_model_name(model_name, load_in_4bit)
|
||||
assert new_model_name is not None
|
||||
model_name = new_model_name
|
||||
|
||||
# Check if pre-quantized models are allowed
|
||||
# For eg AMD GPUs need blocksize = 128, but our pre-quants are blocksize = 64
|
||||
|
|
@ -755,6 +755,7 @@ class FastModel(FastBaseModel):
|
|||
if model_name.lower().endswith("-bf16"):
|
||||
load_in_4bit = False
|
||||
load_in_8bit = False
|
||||
load_in_fp8 = False
|
||||
load_in_16bit = True
|
||||
|
||||
# Check modelscope
|
||||
|
|
@ -833,6 +834,8 @@ class FastModel(FastBaseModel):
|
|||
string += "_load_in_8bit_"
|
||||
if load_in_16bit:
|
||||
string += "_load_in_16bit_"
|
||||
if load_in_fp8:
|
||||
string += "load_in_fp8"
|
||||
os.environ["UNSLOTH_MODEL_NAME"] = string
|
||||
|
||||
# Check versions
|
||||
|
|
@ -1013,6 +1016,7 @@ class FastModel(FastBaseModel):
|
|||
if model_name.lower().endswith("-bf16"):
|
||||
load_in_4bit = False
|
||||
load_in_8bit = False
|
||||
load_in_fp8 = False
|
||||
load_in_16bit = True
|
||||
|
||||
model_config = AutoConfig.from_pretrained(
|
||||
|
|
@ -1177,7 +1181,7 @@ class FastModel(FastBaseModel):
|
|||
}
|
||||
model.config.update({"quantization_config": quantization_config})
|
||||
|
||||
if load_in_fp8:
|
||||
if load_in_fp8 != False:
|
||||
_tag_model_with_fp8_torchao_config(model, fp8_mode)
|
||||
|
||||
if is_peft:
|
||||
|
|
|
|||
|
|
@ -17,7 +17,13 @@ import os
|
|||
import re
|
||||
import tempfile
|
||||
from typing import Union
|
||||
from .mapper import INT_TO_FLOAT_MAPPER, FLOAT_TO_INT_MAPPER, MAP_TO_UNSLOTH_16bit
|
||||
from .mapper import (
|
||||
INT_TO_FLOAT_MAPPER,
|
||||
FLOAT_TO_INT_MAPPER,
|
||||
MAP_TO_UNSLOTH_16bit,
|
||||
FLOAT_TO_FP8_BLOCK_MAPPER,
|
||||
FLOAT_TO_FP8_ROW_MAPPER,
|
||||
)
|
||||
|
||||
# https://github.com/huggingface/transformers/pull/26037 allows 4 bit loading!
|
||||
from packaging.version import Version
|
||||
|
|
@ -45,11 +51,28 @@ def __get_model_name(
|
|||
INT_TO_FLOAT_MAPPER = None,
|
||||
FLOAT_TO_INT_MAPPER = None,
|
||||
MAP_TO_UNSLOTH_16bit = None,
|
||||
load_in_fp8 = False,
|
||||
FLOAT_TO_FP8_BLOCK_MAPPER = None,
|
||||
FLOAT_TO_FP8_ROW_MAPPER = None,
|
||||
):
|
||||
model_name = str(model_name)
|
||||
lower_model_name = model_name.lower()
|
||||
|
||||
if not SUPPORTS_FOURBIT and lower_model_name in INT_TO_FLOAT_MAPPER:
|
||||
assert load_in_fp8 in (True, False, "block")
|
||||
if load_in_fp8 != False:
|
||||
if load_in_fp8 == True and (os.environ.get("UNSLOTH_HAS_FBGEMM", "0") == "1"):
|
||||
if lower_model_name in FLOAT_TO_FP8_ROW_MAPPER:
|
||||
# Faster row scaling only works if FBGEMM works!
|
||||
return FLOAT_TO_FP8_ROW_MAPPER[lower_model_name]
|
||||
elif lower_model_name in FLOAT_TO_FP8_BLOCK_MAPPER:
|
||||
# Otherwise we use the slower blockwise type
|
||||
return FLOAT_TO_FP8_BLOCK_MAPPER[lower_model_name]
|
||||
else:
|
||||
if lower_model_name in FLOAT_TO_FP8_BLOCK_MAPPER:
|
||||
return FLOAT_TO_FP8_BLOCK_MAPPER[lower_model_name]
|
||||
return None
|
||||
|
||||
elif not SUPPORTS_FOURBIT and lower_model_name in INT_TO_FLOAT_MAPPER:
|
||||
model_name = INT_TO_FLOAT_MAPPER[lower_model_name]
|
||||
print(
|
||||
f"Unsloth: Your transformers version of {transformers_version} does not support native "
|
||||
|
|
@ -112,13 +135,17 @@ def _get_new_mapper():
|
|||
return {}, {}, {}
|
||||
|
||||
|
||||
def get_model_name(model_name, load_in_4bit = True):
|
||||
def get_model_name(model_name, load_in_4bit = True, load_in_fp8 = False):
|
||||
assert load_in_fp8 in (True, False, "block")
|
||||
new_model_name = __get_model_name(
|
||||
model_name = model_name,
|
||||
load_in_4bit = load_in_4bit,
|
||||
INT_TO_FLOAT_MAPPER = INT_TO_FLOAT_MAPPER,
|
||||
FLOAT_TO_INT_MAPPER = FLOAT_TO_INT_MAPPER,
|
||||
MAP_TO_UNSLOTH_16bit = MAP_TO_UNSLOTH_16bit,
|
||||
load_in_fp8 = load_in_fp8,
|
||||
FLOAT_TO_FP8_BLOCK_MAPPER = FLOAT_TO_FP8_BLOCK_MAPPER,
|
||||
FLOAT_TO_FP8_ROW_MAPPER = FLOAT_TO_FP8_ROW_MAPPER,
|
||||
)
|
||||
# In the rare case, we convert bad model names to other names
|
||||
# For eg too large dynamic quants or MoEs
|
||||
|
|
@ -144,6 +171,9 @@ def get_model_name(model_name, load_in_4bit = True):
|
|||
INT_TO_FLOAT_MAPPER = NEW_INT_TO_FLOAT_MAPPER,
|
||||
FLOAT_TO_INT_MAPPER = NEW_FLOAT_TO_INT_MAPPER,
|
||||
MAP_TO_UNSLOTH_16bit = NEW_MAP_TO_UNSLOTH_16bit,
|
||||
load_in_fp8 = load_in_fp8,
|
||||
FLOAT_TO_FP8_BLOCK_MAPPER = FLOAT_TO_FP8_BLOCK_MAPPER,
|
||||
FLOAT_TO_FP8_ROW_MAPPER = FLOAT_TO_FP8_ROW_MAPPER,
|
||||
)
|
||||
if upgraded_model_name is not None:
|
||||
raise NotImplementedError(
|
||||
|
|
@ -152,6 +182,9 @@ def get_model_name(model_name, load_in_4bit = True):
|
|||
'pip install --upgrade --no-cache-dir "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"\n'
|
||||
'pip install --upgrade --no-cache-dir "git+https://github.com/unslothai/unsloth-zoo.git"\n'
|
||||
)
|
||||
if load_in_fp8 != False:
|
||||
# Handle on the fly TorchAO FP8 quantization
|
||||
return new_model_name
|
||||
return new_model_name if new_model_name is not None else model_name
|
||||
|
||||
|
||||
|
|
@ -239,11 +272,14 @@ def _tag_model_with_fp8_torchao_config(model: torch.nn.Module, fp8_mode: str):
|
|||
"""
|
||||
Tag a model with a `TorchAOConfig` so downstream callers will know what to do with it.
|
||||
"""
|
||||
base_config = _get_torchao_fp8_config(fp8_mode)
|
||||
model.torchao_config = TorchAOConfig(
|
||||
qat_scheme = None,
|
||||
base_config_and_filter_fns = [(base_config, None)],
|
||||
)
|
||||
try:
|
||||
base_config = _get_torchao_fp8_config(fp8_mode)
|
||||
model.torchao_config = TorchAOConfig(
|
||||
qat_scheme = None,
|
||||
base_config_and_filter_fns = [(base_config, None)],
|
||||
)
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
def _get_fp8_mode_and_check_settings(
|
||||
|
|
|
|||
|
|
@ -15,6 +15,9 @@
|
|||
__all__ = [
|
||||
"INT_TO_FLOAT_MAPPER",
|
||||
"FLOAT_TO_INT_MAPPER",
|
||||
"MAP_TO_UNSLOTH_16bit",
|
||||
"FLOAT_TO_FP8_BLOCK_MAPPER",
|
||||
"FLOAT_TO_FP8_ROW_MAPPER",
|
||||
]
|
||||
|
||||
__INT_TO_FLOAT_MAPPER = \
|
||||
|
|
@ -233,21 +236,35 @@ __INT_TO_FLOAT_MAPPER = \
|
|||
"meta-llama/Meta-Llama-3.1-8B",
|
||||
"unsloth/Meta-Llama-3.1-8B-bnb-4bit",
|
||||
),
|
||||
"unsloth/Meta-Llama-3.1-8B-Instruct-unsloth-bnb-4bit" : (
|
||||
"unsloth/Meta-Llama-3.1-8B-Instruct",
|
||||
"meta-llama/Meta-Llama-3.1-8B-Instruct",
|
||||
"unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit",
|
||||
),
|
||||
"unsloth/Meta-Llama-3.1-8B-Instruct-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"RedHatAI/Llama-3.1-8B-Instruct-FP8",
|
||||
"unsloth/Llama-3.1-8B-Instruct-FP8-Block",
|
||||
"unsloth/Llama-3.1-8B-Instruct-FP8-Dynamic",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Meta-Llama-3.1-8B-Instruct",
|
||||
"meta-llama/Meta-Llama-3.1-8B-Instruct",
|
||||
"unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Llama-3.1-8B-unsloth-bnb-4bit" : (
|
||||
"unsloth/Llama-3.1-8B",
|
||||
"meta-llama/Llama-3.1-8B",
|
||||
"unsloth/Llama-3.1-8B-bnb-4bit",
|
||||
),
|
||||
"unsloth/Llama-3.1-8B-Instruct-unsloth-bnb-4bit" : (
|
||||
"unsloth/Llama-3.1-8B-Instruct",
|
||||
"meta-llama/Llama-3.1-8B-Instruct",
|
||||
"unsloth/Llama-3.1-8B-Instruct-bnb-4bit",
|
||||
),
|
||||
"unsloth/Llama-3.1-8B-Instruct-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"RedHatAI/Llama-3.1-8B-Instruct-FP8",
|
||||
"unsloth/Llama-3.1-8B-Instruct-FP8-Block",
|
||||
"unsloth/Llama-3.1-8B-Instruct-FP8-Dynamic",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Llama-3.1-8B-Instruct",
|
||||
"meta-llama/Llama-3.1-8B-Instruct",
|
||||
"unsloth/Llama-3.1-8B-Instruct-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Meta-Llama-3.1-70B-bnb-4bit" : (
|
||||
"unsloth/Meta-Llama-3.1-70B",
|
||||
"meta-llama/Meta-Llama-3.1-70B",
|
||||
|
|
@ -474,16 +491,30 @@ __INT_TO_FLOAT_MAPPER = \
|
|||
"meta-llama/Llama-3.2-3B",
|
||||
"unsloth/Llama-3.2-3B-bnb-4bit",
|
||||
),
|
||||
"unsloth/Llama-3.2-1B-Instruct-unsloth-bnb-4bit" : (
|
||||
"unsloth/Llama-3.2-1B-Instruct",
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
"unsloth/Llama-3.2-1B-Instruct-bnb-4bit",
|
||||
),
|
||||
"unsloth/Llama-3.2-3B-Instruct-unsloth-bnb-4bit" : (
|
||||
"unsloth/Llama-3.2-3B-Instruct",
|
||||
"meta-llama/Llama-3.2-3B-Instruct",
|
||||
"unsloth/Llama-3.2-3B-Instruct-bnb-4bit",
|
||||
),
|
||||
"unsloth/Llama-3.2-1B-Instruct-unsloth-bnb-4bit" : {
|
||||
"8": (
|
||||
"RedHatAI/Llama-3.2-1B-Instruct-FP8",
|
||||
"unsloth/Llama-3.2-1B-Instruct-FP8-Block",
|
||||
"unsloth/Llama-3.2-1B-Instruct-FP8-Dynamic",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Llama-3.2-1B-Instruct",
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
"unsloth/Llama-3.2-1B-Instruct-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Llama-3.2-3B-Instruct-unsloth-bnb-4bit" : {
|
||||
"8": (
|
||||
"RedHatAI/Llama-3.2-3B-Instruct-FP8",
|
||||
"unsloth/Llama-3.2-3B-Instruct-FP8-Block",
|
||||
"unsloth/Llama-3.2-3B-Instruct-FP8-Dynamic",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Llama-3.2-3B-Instruct",
|
||||
"meta-llama/Llama-3.2-3B-Instruct",
|
||||
"unsloth/Llama-3.2-3B-Instruct-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Llama-3.1-Nemotron-70B-Instruct-bnb-4bit" : (
|
||||
"unsloth/Llama-3.1-Nemotron-70B-Instruct",
|
||||
"nvidia/Llama-3.1-Nemotron-70B-Instruct-HF",
|
||||
|
|
@ -561,10 +592,18 @@ __INT_TO_FLOAT_MAPPER = \
|
|||
"unsloth/QwQ-32B-Preview",
|
||||
"Qwen/QwQ-32B-Preview",
|
||||
),
|
||||
"unsloth/Llama-3.3-70B-Instruct-bnb-4bit" : (
|
||||
"unsloth/Llama-3.3-70B-Instruct",
|
||||
"meta-llama/Llama-3.3-70B-Instruct",
|
||||
),
|
||||
"unsloth/Llama-3.3-70B-Instruct-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"RedHatAI/Llama-3.3-70B-Instruct-FP8",
|
||||
"unsloth/Llama-3.3-70B-Instruct-FP8-Block",
|
||||
"unsloth/Llama-3.3-70B-Instruct-FP8-Dynamic",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Llama-3.3-70B-Instruct",
|
||||
"meta-llama/Llama-3.3-70B-Instruct",
|
||||
"unsloth/Llama-3.3-70B-Instruct-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/phi-4-unsloth-bnb-4bit" : (
|
||||
"unsloth/phi-4",
|
||||
"microsoft/phi-4",
|
||||
|
|
@ -733,36 +772,78 @@ __INT_TO_FLOAT_MAPPER = \
|
|||
"mistralai/Mistral-Small-3.1-24B-Base-2503",
|
||||
"unsloth/Mistral-Small-3.1-24B-Base-2503-bnb-4bit",
|
||||
),
|
||||
"unsloth/Qwen3-0.6B-unsloth-bnb-4bit" : (
|
||||
"unsloth/Qwen3-0.6B",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"unsloth/Qwen3-0.6B-bnb-4bit",
|
||||
),
|
||||
"unsloth/Qwen3-1.7B-unsloth-bnb-4bit" : (
|
||||
"unsloth/Qwen3-1.7B",
|
||||
"Qwen/Qwen3-1.7B",
|
||||
"unsloth/Qwen3-1.7B-bnb-4bit",
|
||||
),
|
||||
"unsloth/Qwen3-4B-unsloth-bnb-4bit" : (
|
||||
"unsloth/Qwen3-4B",
|
||||
"Qwen/Qwen3-4B",
|
||||
"unsloth/Qwen3-4B-bnb-4bit",
|
||||
),
|
||||
"unsloth/Qwen3-8B-unsloth-bnb-4bit" : (
|
||||
"unsloth/Qwen3-8B",
|
||||
"Qwen/Qwen3-8B",
|
||||
"unsloth/Qwen3-8B-bnb-4bit",
|
||||
),
|
||||
"unsloth/Qwen3-14B-unsloth-bnb-4bit" : (
|
||||
"unsloth/Qwen3-14B",
|
||||
"Qwen/Qwen3-14B",
|
||||
"unsloth/Qwen3-14B-bnb-4bit",
|
||||
),
|
||||
"unsloth/Qwen3-32B-unsloth-bnb-4bit" : (
|
||||
"unsloth/Qwen3-32B",
|
||||
"Qwen/Qwen3-32B",
|
||||
"unsloth/Qwen3-32B-bnb-4bit",
|
||||
),
|
||||
"unsloth/Qwen3-0.6B-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"Qwen/Qwen3-0.6B-FP8",
|
||||
"unsloth/Qwen3-0.6B-FP8",
|
||||
"unsloth/Qwen3-0.6B-FP8",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Qwen3-0.6B",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"unsloth/Qwen3-0.6B-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Qwen3-1.7B-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"Qwen/Qwen3-1.7B-FP8",
|
||||
"unsloth/Qwen3-1.7B-FP8",
|
||||
"unsloth/Qwen3-1.7B-FP8",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Qwen3-1.7B",
|
||||
"Qwen/Qwen3-1.7B",
|
||||
"unsloth/Qwen3-1.7B-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Qwen3-4B-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"Qwen/Qwen3-4B-FP8",
|
||||
"unsloth/Qwen3-4B-FP8",
|
||||
"unsloth/Qwen3-4B-FP8",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Qwen3-4B",
|
||||
"Qwen/Qwen3-4B",
|
||||
"unsloth/Qwen3-4B-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Qwen3-8B-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"Qwen/Qwen3-8B-FP8",
|
||||
"unsloth/Qwen3-8B-FP8",
|
||||
"unsloth/Qwen3-8B-FP8",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Qwen3-8B",
|
||||
"Qwen/Qwen3-8B",
|
||||
"unsloth/Qwen3-8B-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Qwen3-14B-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"Qwen/Qwen3-14B-FP8",
|
||||
"unsloth/Qwen3-14B-FP8",
|
||||
"unsloth/Qwen3-14B-FP8",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Qwen3-14B",
|
||||
"Qwen/Qwen3-14B",
|
||||
"unsloth/Qwen3-14B-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Qwen3-32B-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"Qwen/Qwen3-32B-FP8",
|
||||
"unsloth/Qwen3-32B-FP8",
|
||||
"unsloth/Qwen3-32B-FP8",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Qwen3-32B",
|
||||
"Qwen/Qwen3-32B",
|
||||
"unsloth/Qwen3-32B-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Qwen3-30B-A3B-unsloth-bnb-4bit" : (
|
||||
"unsloth/Qwen3-30B-A3B",
|
||||
"Qwen/Qwen3-30B-A3B",
|
||||
|
|
@ -936,16 +1017,30 @@ __INT_TO_FLOAT_MAPPER = \
|
|||
"openai/gpt-oss-120b",
|
||||
"unsloth/gpt-oss-120b-unsloth-bnb-4bit",
|
||||
),
|
||||
"unsloth/Qwen3-4B-Instruct-2507-unsloth-bnb-4bit" : (
|
||||
"unsloth/Qwen3-4B-Instruct-2507",
|
||||
"Qwen/Qwen3-4B-Instruct-2507",
|
||||
"unsloth/Qwen3-4B-Instruct-2507-bnb-4bit",
|
||||
),
|
||||
"unsloth/Qwen3-4B-Thinking-2507-unsloth-bnb-4bit" : (
|
||||
"unsloth/Qwen3-4B-Thinking-2507",
|
||||
"Qwen/Qwen3-4B-Thinking-2507",
|
||||
"unsloth/Qwen3-4B-Thinking-2507-bnb-4bit",
|
||||
),
|
||||
"unsloth/Qwen3-4B-Instruct-2507-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"Qwen/Qwen3-4B-Instruct-2507-FP8",
|
||||
"unsloth/Qwen3-4B-Instruct-2507-FP8",
|
||||
"unsloth/Qwen3-4B-Instruct-2507-FP8",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Qwen3-4B-Instruct-2507",
|
||||
"Qwen/Qwen3-4B-Instruct-2507",
|
||||
"unsloth/Qwen3-4B-Instruct-2507-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Qwen3-4B-Thinking-2507-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"Qwen/Qwen3-4B-Thinking-2507-FP8",
|
||||
"unsloth/Qwen3-4B-Thinking-2507-FP8",
|
||||
"unsloth/Qwen3-4B-Thinking-2507-FP8",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Qwen3-4B-Thinking-2507",
|
||||
"Qwen/Qwen3-4B-Thinking-2507",
|
||||
"unsloth/Qwen3-4B-Thinking-2507-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/gemma-3-270m-it-unsloth-bnb-4bit" : (
|
||||
"unsloth/gemma-3-270m-it",
|
||||
"google/gemma-3-270m-it",
|
||||
|
|
@ -1012,46 +1107,102 @@ __INT_TO_FLOAT_MAPPER = \
|
|||
"unsloth/granite-4.0-h-small-base",
|
||||
"ibm-granite/granite-4.0-h-small-base",
|
||||
),
|
||||
"unsloth/Qwen3-VL-4B-Thinking-unsloth-bnb-4bit" : (
|
||||
"unsloth/Qwen3-VL-4B-Thinking",
|
||||
"Qwen/Qwen3-VL-4B-Thinking",
|
||||
"unsloth/Qwen3-VL-4B-Thinking-bnb-4bit",
|
||||
),
|
||||
"unsloth/Qwen3-VL-8B-Thinking-unsloth-bnb-4bit" : (
|
||||
"unsloth/Qwen3-VL-8B-Thinking",
|
||||
"Qwen/Qwen3-VL-8B-Thinking",
|
||||
"unsloth/Qwen3-VL-8B-Thinking-bnb-4bit",
|
||||
),
|
||||
"unsloth/Qwen3-VL-4B-Instruct-unsloth-bnb-4bit" : (
|
||||
"unsloth/Qwen3-VL-4B-Instruct",
|
||||
"Qwen/Qwen3-VL-4B-Instruct",
|
||||
"unsloth/Qwen3-VL-4B-Instruct-bnb-4bit",
|
||||
),
|
||||
"unsloth/Qwen3-VL-8B-Instruct-unsloth-bnb-4bit" : (
|
||||
"unsloth/Qwen3-VL-8B-Instruct",
|
||||
"Qwen/Qwen3-VL-8B-Instruct",
|
||||
"unsloth/Qwen3-VL-8B-Instruct-bnb-4bit",
|
||||
),
|
||||
"unsloth/Qwen3-VL-2B-Thinking-unsloth-bnb-4bit" : (
|
||||
"unsloth/Qwen3-VL-2B-Thinking",
|
||||
"Qwen/Qwen3-VL-2B-Thinking",
|
||||
"unsloth/Qwen3-VL-2B-Thinking-bnb-4bit",
|
||||
),
|
||||
"unsloth/Qwen3-VL-32B-Thinking-unsloth-bnb-4bit" : (
|
||||
"unsloth/Qwen3-VL-32B-Thinking",
|
||||
"Qwen/Qwen3-VL-32B-Thinking",
|
||||
"unsloth/Qwen3-VL-32B-Thinking-bnb-4bit",
|
||||
),
|
||||
"unsloth/Qwen3-VL-2B-Instruct-unsloth-bnb-4bit" : (
|
||||
"unsloth/Qwen3-VL-2B-Instruct",
|
||||
"Qwen/Qwen3-VL-2B-Instruct",
|
||||
"unsloth/Qwen3-VL-2B-Instruct-bnb-4bit",
|
||||
),
|
||||
"unsloth/Qwen3-VL-32B-Instruct-unsloth-bnb-4bit" : (
|
||||
"unsloth/Qwen3-VL-32B-Instruct",
|
||||
"Qwen/Qwen3-VL-32B-Instruct",
|
||||
"unsloth/Qwen3-VL-32B-Instruct-bnb-4bit",
|
||||
),
|
||||
"unsloth/Qwen3-VL-4B-Thinking-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"Qwen/Qwen3-VL-4B-Thinking-FP8",
|
||||
"unsloth/Qwen3-VL-4B-Thinking-FP8",
|
||||
"unsloth/Qwen3-VL-4B-Thinking-FP8",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Qwen3-VL-4B-Thinking",
|
||||
"Qwen/Qwen3-VL-4B-Thinking",
|
||||
"unsloth/Qwen3-VL-4B-Thinking-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Qwen3-VL-8B-Thinking-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"Qwen/Qwen3-VL-8B-Thinking-FP8",
|
||||
"unsloth/Qwen3-VL-8B-Thinking-FP8",
|
||||
"unsloth/Qwen3-VL-8B-Thinking-FP8",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Qwen3-VL-8B-Thinking",
|
||||
"Qwen/Qwen3-VL-8B-Thinking",
|
||||
"unsloth/Qwen3-VL-8B-Thinking-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Qwen3-VL-4B-Instruct-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"Qwen/Qwen3-VL-4B-Instruct-FP8",
|
||||
"unsloth/Qwen3-VL-4B-Instruct-FP8",
|
||||
"unsloth/Qwen3-VL-4B-Instruct-FP8",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Qwen3-VL-4B-Instruct",
|
||||
"Qwen/Qwen3-VL-4B-Instruct",
|
||||
"unsloth/Qwen3-VL-4B-Instruct-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Qwen3-VL-8B-Instruct-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"Qwen/Qwen3-VL-8B-Instruct-FP8",
|
||||
"unsloth/Qwen3-VL-8B-Instruct-FP8",
|
||||
"unsloth/Qwen3-VL-8B-Instruct-FP8",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Qwen3-VL-8B-Instruct",
|
||||
"Qwen/Qwen3-VL-8B-Instruct",
|
||||
"unsloth/Qwen3-VL-8B-Instruct-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Qwen3-VL-2B-Thinking-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"Qwen/Qwen3-VL-2B-Thinking-FP8",
|
||||
"unsloth/Qwen3-VL-2B-Thinking-FP8",
|
||||
"unsloth/Qwen3-VL-2B-Thinking-FP8",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Qwen3-VL-2B-Thinking",
|
||||
"Qwen/Qwen3-VL-2B-Thinking",
|
||||
"unsloth/Qwen3-VL-2B-Thinking-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Qwen3-VL-32B-Thinking-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"Qwen/Qwen3-VL-32B-Thinking-FP8",
|
||||
"unsloth/Qwen3-VL-32B-Thinking-FP8",
|
||||
"unsloth/Qwen3-VL-32B-Thinking-FP8",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Qwen3-VL-32B-Thinking",
|
||||
"Qwen/Qwen3-VL-32B-Thinking",
|
||||
"unsloth/Qwen3-VL-32B-Thinking-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Qwen3-VL-2B-Instruct-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"Qwen/Qwen3-VL-2B-Instruct-FP8",
|
||||
"unsloth/Qwen3-VL-2B-Instruct-FP8",
|
||||
"unsloth/Qwen3-VL-2B-Instruct-FP8",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Qwen3-VL-2B-Instruct",
|
||||
"Qwen/Qwen3-VL-2B-Instruct",
|
||||
"unsloth/Qwen3-VL-2B-Instruct-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/Qwen3-VL-32B-Instruct-unsloth-bnb-4bit" : {
|
||||
"8" : (
|
||||
"Qwen/Qwen3-VL-32B-Instruct-FP8",
|
||||
"unsloth/Qwen3-VL-32B-Instruct-FP8",
|
||||
"unsloth/Qwen3-VL-32B-Instruct-FP8",
|
||||
),
|
||||
"16" : (
|
||||
"unsloth/Qwen3-VL-32B-Instruct",
|
||||
"Qwen/Qwen3-VL-32B-Instruct",
|
||||
"unsloth/Qwen3-VL-32B-Instruct-bnb-4bit",
|
||||
),
|
||||
},
|
||||
"unsloth/granite-4.0-350m-base-unsloth-bnb-4bit" : (
|
||||
"unsloth/granite-4.0-350m-base-unsloth-bnb-4bit",
|
||||
"ibm-granite/granite-4.0-350m-base",
|
||||
|
|
@ -1105,8 +1256,28 @@ __INT_TO_FLOAT_MAPPER = \
|
|||
INT_TO_FLOAT_MAPPER = {}
|
||||
FLOAT_TO_INT_MAPPER = {}
|
||||
MAP_TO_UNSLOTH_16bit = {}
|
||||
FLOAT_TO_FP8_BLOCK_MAPPER = {}
|
||||
FLOAT_TO_FP8_ROW_MAPPER = {}
|
||||
|
||||
for key, values in __INT_TO_FLOAT_MAPPER.items():
|
||||
block, row = None, None
|
||||
if type(values) is dict:
|
||||
assert "16" in values
|
||||
# Float8 and other quantized types
|
||||
if "8" in values:
|
||||
float8_values = values["8"]
|
||||
assert len(float8_values) == 3
|
||||
official, block, row = float8_values
|
||||
FLOAT_TO_FP8_BLOCK_MAPPER[key.lower()] = block
|
||||
FLOAT_TO_FP8_ROW_MAPPER[key.lower()] = row
|
||||
FLOAT_TO_FP8_BLOCK_MAPPER[official.lower() + "-dynamic"] = block
|
||||
FLOAT_TO_FP8_ROW_MAPPER[official.lower()] = row
|
||||
FLOAT_TO_FP8_ROW_MAPPER[official.lower() + "-dynamic"] = row
|
||||
for k in float8_values:
|
||||
FLOAT_TO_FP8_BLOCK_MAPPER[k.lower()] = block
|
||||
FLOAT_TO_FP8_ROW_MAPPER[k.lower()] = row
|
||||
pass
|
||||
values = values["16"]
|
||||
INT_TO_FLOAT_MAPPER[key] = values[0]
|
||||
|
||||
for value in values:
|
||||
|
|
@ -1117,6 +1288,10 @@ for key, values in __INT_TO_FLOAT_MAPPER.items():
|
|||
if values[0].startswith("unsloth"):
|
||||
MAP_TO_UNSLOTH_16bit[values[1]] = values[0]
|
||||
MAP_TO_UNSLOTH_16bit[values[1].lower()] = values[0]
|
||||
MAP_TO_UNSLOTH_16bit[block] = values[0]
|
||||
MAP_TO_UNSLOTH_16bit[block.lower()] = values[0]
|
||||
MAP_TO_UNSLOTH_16bit[row] = values[0]
|
||||
MAP_TO_UNSLOTH_16bit[row.lower()] = values[0]
|
||||
elif len(values) == 3:
|
||||
# Dynamic Unsloth quantization
|
||||
if values[0].startswith("unsloth"):
|
||||
|
|
@ -1124,6 +1299,10 @@ for key, values in __INT_TO_FLOAT_MAPPER.items():
|
|||
MAP_TO_UNSLOTH_16bit[values[1].lower()] = values[0]
|
||||
MAP_TO_UNSLOTH_16bit[values[2]] = values[0]
|
||||
MAP_TO_UNSLOTH_16bit[values[2].lower()] = values[0]
|
||||
MAP_TO_UNSLOTH_16bit[block] = values[0]
|
||||
MAP_TO_UNSLOTH_16bit[block.lower()] = values[0]
|
||||
MAP_TO_UNSLOTH_16bit[row] = values[0]
|
||||
MAP_TO_UNSLOTH_16bit[row.lower()] = values[0]
|
||||
pass
|
||||
|
||||
# Get lowercased
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue