ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update vision.py

Browse source
This commit is contained in:
Daniel Han 2025-11-24 05:50:22 -08:00
commit 6f8fe138a5
1 changed files with 18 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

28
unsloth/models/vision.py
View file

@ -618,16 +618,24 @@ class FastBaseModel:
else:
quantizer = AUTO_QUANTIZATION_CONFIG_MAPPING[quant_method]
quantizer_kwargs = {}
# We cannot dequantize since gpt-oss-20b MXFP4 will now be gpt-oss-20b-BF16
if (
load_in_16bit
and "dequantize" in inspect.signature(quantizer).parameters
):
quantizer_kwargs["dequantize"] = True
quantization_config = quantizer.from_dict(
quantization_config, **quantizer_kwargs
)
kwargs["quantization_config"] = quantization_config
if quant_method == "compressed-tensors":
# Ignore these
pass
else:
# We cannot dequantize since gpt-oss-20b MXFP4 will now be gpt-oss-20b-BF16
if (
load_in_16bit
and "dequantize" in inspect.signature(quantizer).parameters
):
quantizer_kwargs["dequantize"] = True
try:
# Sometimes this fails so we wrap it in a try except
quantization_config = quantizer.from_dict(
quantization_config, **quantizer_kwargs
)
except:
pass
kwargs["quantization_config"] = quantization_config
# Check if using forced float32 - we load it in bfloat16, then cast to float16!
torch_dtype = dtype