From 15052dc8e7e507244a487664d3211e6776f45f4b Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Sun, 4 Jan 2026 13:18:15 +0000 Subject: [PATCH] Keep 4bit flag for fast_inference --- unsloth/models/loader.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 4488ad9a07..eb3b21e206 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -167,12 +167,6 @@ class FastLanguageModel(FastLlamaModel): load_in_8bit = True load_in_4bit = False - load_in_4bit_kwargs = load_in_4bit - load_in_8bit_kwargs = load_in_8bit - if quantization_config is not None: - load_in_4bit_kwargs = False - load_in_8bit_kwargs = False - # Login to allow private models token = hf_login(token) # Align dtype with bnb_4bit_compute_dtype if provided and dtype is unset. @@ -581,6 +575,12 @@ class FastLanguageModel(FastLlamaModel): if fast_inference: fast_inference, model_name = fast_inference_setup(model_name, model_config) + load_in_4bit_kwargs = load_in_4bit + load_in_8bit_kwargs = load_in_8bit + if quantization_config is not None and not fast_inference: + load_in_4bit_kwargs = False + load_in_8bit_kwargs = False + model, tokenizer = dispatch_model.from_pretrained( model_name = model_name, max_seq_length = max_seq_length, @@ -753,12 +753,6 @@ class FastModel(FastBaseModel): load_in_8bit = True load_in_4bit = False - load_in_4bit_kwargs = load_in_4bit - load_in_8bit_kwargs = load_in_8bit - if quantization_config is not None: - load_in_4bit_kwargs = False - load_in_8bit_kwargs = False - # Login to allow private models token = hf_login(token) if whisper_language is not None: @@ -1255,6 +1249,12 @@ class FastModel(FastBaseModel): if auto_model is None: auto_model = AutoModelForVision2Seq if is_vlm else AutoModelForCausalLM + load_in_4bit_kwargs = load_in_4bit + load_in_8bit_kwargs = load_in_8bit + if quantization_config is not None and not fast_inference: + load_in_4bit_kwargs = False + load_in_8bit_kwargs = False + model, tokenizer = FastBaseModel.from_pretrained( model_name = model_name, max_seq_length = max_seq_length,