ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Quant Method missing

Browse source
This commit is contained in:
Daniel Han 2025-10-28 05:26:51 -07:00
commit 0116242177
2 changed files with 12 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

6
unsloth/__init__.py
View file

@ -89,8 +89,10 @@ try:
# except:
# raise ImportError("Unsloth: Please update unsloth_zoo via `pip install --upgrade --no-cache-dir --no-deps unsloth_zoo`")
import unsloth_zoo
except:
raise ImportError("Unsloth: Please install unsloth_zoo via `pip install unsloth_zoo`")
except NotImplementedError as e:
raise NotImplementedError(str(e))
except Exception as e:
raise ImportError(f"Unsloth: Please install unsloth_zoo via `pip install unsloth_zoo` Also error = {str(e)}")
pass
from unsloth_zoo.device_type import (

9
unsloth/models/vision.py
View file

@ -538,7 +538,14 @@ class FastBaseModel:
if hasattr(auto_config, "quantization_config"):
from transformers.quantizers.auto import AUTO_QUANTIZATION_CONFIG_MAPPING
quantization_config = auto_config.quantization_config
quantizer = AUTO_QUANTIZATION_CONFIG_MAPPING[quantization_config["quant_method"]]
quant_method = quantization_config["quant_method"]
# Sometimes bitsandbytes_4bit + bitsandbytes_8bit is provided
if quant_method == "bitsandbytes" and "bitsandbytes" not in AUTO_QUANTIZATION_CONFIG_MAPPING:
if "bitsandbytes_4bit" not in AUTO_QUANTIZATION_CONFIG_MAPPING:
raise KeyError("Unsloth: AUTO_QUANTIZATION_CONFIG_MAPPING does not have `bitsandbytes_4bit`")
quantizer = AUTO_QUANTIZATION_CONFIG_MAPPING["bitsandbytes_4bit"]
else:
quantizer = AUTO_QUANTIZATION_CONFIG_MAPPING[quant_method]
quantizer_kwargs = {}
# We cannot dequantize since gpt-oss-20b MXFP4 will now be gpt-oss-20b-BF16
if load_in_16bit and "dequantize" in inspect.signature(quantizer).parameters: