ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Qwen3 bug fixes

Browse source
This commit is contained in:
Daniel Han 2025-05-02 07:18:33 -07:00
commit 2249a5ff88
3 changed files with 20 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
unsloth/models/llama.py
View file

@ -23,6 +23,7 @@ from ._utils import __version__
from torch.nn.functional import scaled_dot_product_attention
from transformers import __version__ as transformers_version
from unsloth_zoo.utils import Version, _get_dtype
from unsloth_zoo.peft_utils import SKIP_QUANTIZATION_MODULES
transformers_version = Version(transformers_version)
# Transformers moved rotary embeddings out of all attention layers
IS_ATTENTION_REFACTOR = transformers_version > Version("4.47.1")
@ -1771,6 +1772,7 @@ class FastLlamaModel:
bnb_4bit_use_double_quant = True,
bnb_4bit_quant_type = "nf4",
bnb_4bit_compute_dtype = dtype,
llm_int8_skip_modules = SKIP_QUANTIZATION_MODULES.copy(),
)
pass

11
unsloth/models/loader_utils.py
View file

@ -19,6 +19,12 @@ from transformers import __version__ as transformers_version
transformers_version = Version(transformers_version)
SUPPORTS_FOURBIT = transformers_version >= Version("4.37")
BAD_MAPPINGS = \
{
"unsloth/qwen3-32B-unsloth-bnb-4bit" : "unsloth/Qwen3-32B-bnb-4bit", # 32B dynamic quant is way too big
"unsloth/qwen3-30B-A3B-unsloth-bnb-4bit" : "unsloth/qwen3-30B-A3B", # HF loads MoEs too slowly
"unsloth/qwen3-30B-A3B-bnb-4bit" : "unsloth/qwen3-30B-A3B", # We rather do it on the fly
}
def __get_model_name(
model_name,
@ -102,6 +108,11 @@ def get_model_name(model_name, load_in_4bit = True):
FLOAT_TO_INT_MAPPER = FLOAT_TO_INT_MAPPER,
MAP_TO_UNSLOTH_16bit = MAP_TO_UNSLOTH_16bit,
)
# In the rare case, we convert bad model names to other names
# For eg too large dynamic quants or MoEs
if new_model_name.lower() in BAD_MAPPINGS:
new_model_name = BAD_MAPPINGS[new_model_name.lower()]
if new_model_name is None and model_name.count("/") == 1 and model_name[0].isalnum():
# Try checking if a new Unsloth version allows it!
NEW_INT_TO_FLOAT_MAPPER, NEW_FLOAT_TO_INT_MAPPER, NEW_MAP_TO_UNSLOTH_16bit = _get_new_mapper()

8
unsloth/models/mapper.py
View file

@ -763,9 +763,15 @@ __INT_TO_FLOAT_MAPPER = \
"Qwen/Qwen3-14B",
"unsloth/Qwen3-14B-bnb-4bit",
),
"unsloth/Qwen3-32B-bnb-4bit" : (
"unsloth/Qwen3-32B-unsloth-bnb-4bit" : (
"unsloth/Qwen3-32B",
"Qwen/Qwen3-32B",
"unsloth/Qwen3-32B-bnb-4bit",
),
"unsloth/Qwen3-30B-A3B-unsloth-bnb-4bit" : (
"unsloth/Qwen3-30B-A3B",
"Qwen/Qwen3-30B-A3B",
"unsloth/Qwen3-30B-A3B-bnb-4bit",
),
"unsloth/Qwen3-0.6B-Base-unsloth-bnb-4bit" : (
"unsloth/Qwen3-0.6B-Base",