ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Conditionally enable 4bit on CDNA for bitsandbytes>=v0.49.2 (#4161)

Browse source
This commit is contained in:
Strahinja Stamenkovic 2026-03-07 10:33:40 +01:00 • committed by GitHub
commit 69330957ee
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 15 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

17
unsloth/device_type.py
View file

@ -79,16 +79,19 @@ def get_device_count():
DEVICE_COUNT: int = get_device_count()
# 4-bit quantization requires a block size of 64
# this is not supported on AMD Instinct GPUs currently
# | Device Type | Warp Size | Block Size |
# |-----------------|-----------|------------|
# | CUDA | 32 | 64 |
# | Radeon (Navi) | 32 | 64 |
# | Instinct (MI) | 64 | 128 |
# | CUDA | 32 | 32 |
# | Radeon (Navi) | 32 | 32 |
# | Instinct (MI) | 64 | 32 |
#
# Since bitsandbytes 0.49.0, pre-quantized models with 64 blockwise now works
# on Radeon GPUs, but not Instinct MI300x for eg [WIP]
# on Radeon GPUs, but not Instinct MI300x for eg
# See https://github.com/bitsandbytes-foundation/bitsandbytes/pull/1748
#
# Since bitsandbytes 0.49.2, blocksize=64 4-bit quantization is supported on
# CDNA (MI Instinct / gfx9xx) GPUs as well
# See https://github.com/bitsandbytes-foundation/bitsandbytes/pull/1856
ALLOW_PREQUANTIZED_MODELS: bool = True
# HSA_STATUS_ERROR_EXCEPTION checks - sometimes AMD fails for BnB
@ -104,7 +107,9 @@ if DEVICE_TYPE == "hip":
ALLOW_BITSANDBYTES = False
if ALLOW_BITSANDBYTES:
ALLOW_BITSANDBYTES = Version(bitsandbytes.__version__) > Version("0.48.2.dev0")
if Version(bitsandbytes.__version__) > Version("0.49.0"):
if Version(bitsandbytes.__version__) >= Version("0.49.2"):
pass
elif Version(bitsandbytes.__version__) >= Version("0.49.0"):
try:
# Pre-quantized bitsandbytes models use blocksize 64, so we need to check the GPU
from bitsandbytes.cextension import ROCM_WARP_SIZE_64

8
unsloth/models/loader.py
View file

@ -397,7 +397,7 @@ class FastLanguageModel(FastLlamaModel):
load_in_fp8 = False
# Check if pre-quantized models are allowed
# For eg AMD Instinct GPUs need blocksize = 128, but our pre-quants are blocksize = 64
# AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64)
if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(
("-unsloth-bnb-4bit", "-bnb-4bit")
):
@ -537,7 +537,7 @@ class FastLanguageModel(FastLlamaModel):
trust_remote_code = trust_remote_code,
)
# Check if pre-quantized models are allowed
# For eg AMD Instinct GPUs need blocksize = 128, but our pre-quants are blocksize = 64
# AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64)
if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(
("-unsloth-bnb-4bit", "-bnb-4bit")
):
@ -1005,7 +1005,7 @@ class FastModel(FastBaseModel):
load_in_fp8 = False
# Check if pre-quantized models are allowed
# For eg AMD Instinct GPUs need blocksize = 128, but our pre-quants are blocksize = 64
# AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64)
if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(
("-unsloth-bnb-4bit", "-bnb-4bit")
):
@ -1288,7 +1288,7 @@ class FastModel(FastBaseModel):
if not use_exact_model_name:
model_name = get_model_name(model_name, load_in_4bit)
# Check if pre-quantized models are allowed
# For eg AMD Instinct GPUs need blocksize = 128, but our pre-quants are blocksize = 64
# AMD Instinct GPUs need blocksize = 128 on bitsandbytes < 0.49.2 (our pre-quants use blocksize = 64)
if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(
("-unsloth-bnb-4bit", "-bnb-4bit")
):