diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 4488ad9a07..eb3b21e206 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -167,12 +167,6 @@ class FastLanguageModel(FastLlamaModel):
                 load_in_8bit = True
                 load_in_4bit = False
 
-        load_in_4bit_kwargs = load_in_4bit
-        load_in_8bit_kwargs = load_in_8bit
-        if quantization_config is not None:
-            load_in_4bit_kwargs = False
-            load_in_8bit_kwargs = False
-
         # Login to allow private models
         token = hf_login(token)
         # Align dtype with bnb_4bit_compute_dtype if provided and dtype is unset.
@@ -581,6 +575,12 @@ class FastLanguageModel(FastLlamaModel):
         if fast_inference:
             fast_inference, model_name = fast_inference_setup(model_name, model_config)
 
+        load_in_4bit_kwargs = load_in_4bit
+        load_in_8bit_kwargs = load_in_8bit
+        if quantization_config is not None and not fast_inference:
+            load_in_4bit_kwargs = False
+            load_in_8bit_kwargs = False
+
         model, tokenizer = dispatch_model.from_pretrained(
             model_name = model_name,
             max_seq_length = max_seq_length,
@@ -753,12 +753,6 @@ class FastModel(FastBaseModel):
                 load_in_8bit = True
                 load_in_4bit = False
 
-        load_in_4bit_kwargs = load_in_4bit
-        load_in_8bit_kwargs = load_in_8bit
-        if quantization_config is not None:
-            load_in_4bit_kwargs = False
-            load_in_8bit_kwargs = False
-
         # Login to allow private models
         token = hf_login(token)
         if whisper_language is not None:
@@ -1255,6 +1249,12 @@ class FastModel(FastBaseModel):
         if auto_model is None:
             auto_model = AutoModelForVision2Seq if is_vlm else AutoModelForCausalLM
 
+        load_in_4bit_kwargs = load_in_4bit
+        load_in_8bit_kwargs = load_in_8bit
+        if quantization_config is not None and not fast_inference:
+            load_in_4bit_kwargs = False
+            load_in_8bit_kwargs = False
+
         model, tokenizer = FastBaseModel.from_pretrained(
             model_name = model_name,
             max_seq_length = max_seq_length,