ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Keep 4bit flag for fast_inference

Browse source
This commit is contained in:
danielhanchen 2026-01-04 13:18:15 +00:00
commit 15052dc8e7
1 changed files with 12 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

24
unsloth/models/loader.py
View file

@ -167,12 +167,6 @@ class FastLanguageModel(FastLlamaModel):
load_in_8bit = True
load_in_4bit = False
load_in_4bit_kwargs = load_in_4bit
load_in_8bit_kwargs = load_in_8bit
if quantization_config is not None:
load_in_4bit_kwargs = False
load_in_8bit_kwargs = False
# Login to allow private models
token = hf_login(token)
# Align dtype with bnb_4bit_compute_dtype if provided and dtype is unset.
@ -581,6 +575,12 @@ class FastLanguageModel(FastLlamaModel):
if fast_inference:
fast_inference, model_name = fast_inference_setup(model_name, model_config)
load_in_4bit_kwargs = load_in_4bit
load_in_8bit_kwargs = load_in_8bit
if quantization_config is not None and not fast_inference:
load_in_4bit_kwargs = False
load_in_8bit_kwargs = False
model, tokenizer = dispatch_model.from_pretrained(
model_name = model_name,
max_seq_length = max_seq_length,
@ -753,12 +753,6 @@ class FastModel(FastBaseModel):
load_in_8bit = True
load_in_4bit = False
load_in_4bit_kwargs = load_in_4bit
load_in_8bit_kwargs = load_in_8bit
if quantization_config is not None:
load_in_4bit_kwargs = False
load_in_8bit_kwargs = False
# Login to allow private models
token = hf_login(token)
if whisper_language is not None:
@ -1255,6 +1249,12 @@ class FastModel(FastBaseModel):
if auto_model is None:
auto_model = AutoModelForVision2Seq if is_vlm else AutoModelForCausalLM
load_in_4bit_kwargs = load_in_4bit
load_in_8bit_kwargs = load_in_8bit
if quantization_config is not None and not fast_inference:
load_in_4bit_kwargs = False
load_in_8bit_kwargs = False
model, tokenizer = FastBaseModel.from_pretrained(
model_name = model_name,
max_seq_length = max_seq_length,