diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index e7cc5ff1ef..5f4927b92f 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -2567,6 +2567,14 @@ def patch_hf_quantizer():
     except Exception as e:
         logger.warning(f"Failed to patch FbgemmFp8HfQuantizer. Error {e}")
 
+    try:
+        from transformers.quantizers.quantizer_torchao import TorchAoHfQuantizer
+
+        TorchAoHfQuantizer.is_trainable = property(make_trainable)
+        TorchAoHfQuantizer.is_qat_trainable = property(make_trainable)
+    except Exception as e:
+        logger.warning(f"Failed to patch TorchAoHfQuantizer. Error {e}")
+
 
 patch_hf_quantizer()
 
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 04c15bb2c3..72a6d7d10f 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -261,12 +261,6 @@ class FastLanguageModel(FastLlamaModel):
                         fast_inference = False
                         break
 
-        # [TODO] For now fast_inference only works with fast_inference ie vLLM
-        if load_in_fp8 != False:
-            if not fast_inference:
-                raise NotImplementedError(
-                    "Unsloth: set `fast_inference = True` when doing `load_in_fp8`."
-                )
         # Check if 4bit is allowed specifically for AMD
         if not ALLOW_BITSANDBYTES and not use_exact_model_name:
             if load_in_4bit or load_in_8bit or model_name.lower().endswith("-bnb-4bit"):
@@ -876,13 +870,6 @@ class FastModel(FastBaseModel):
                         fast_inference = False
                         break
 
-        # [TODO] For now fast_inference only works with fast_inference ie vLLM
-        if load_in_fp8 != False:
-            if not fast_inference:
-                raise NotImplementedError(
-                    "Unsloth: set `fast_inference = True` when doing `load_in_fp8`."
-                )
-
         # Find FP8, BnB 4bit, other mapped names
         old_model_name = model_name
         fp8_mode = None
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index f17ebc53e3..40ac49ca78 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -350,10 +350,6 @@ def _get_fp8_mode_and_check_settings(
         raise ValueError(
             f"Unsloth: `load_in_fp8` can only be 'row' or 'block', got '{fp8_mode}'"
         )
-    if not fast_inference:
-        raise ValueError(
-            "Unsloth: `load_in_fp8` is only supported for `fast_inference` for now"
-        )
     if full_finetuning:
         raise ValueError(
             "Unsloth: `load_in_fp8` is not compatible with full finetuning"