Allow fp8 for non fast inference (#3904)

* Allow fp8 for non fast inference

* Extensive fp8 alow and quantizer patch

* Clean up commented-out code, duplicate import, and revert unnecessary Version() changes

- Delete commented-out FP8 fast_inference guard in FastModel (loader.py)
  instead of leaving it commented -- matches FastLanguageModel which was
  properly deleted
- Delete commented-out fast_inference guard in loader_utils.py
- Remove duplicate `from transformers import GenerationConfig, CompileConfig`
  in vision.py (line 112 already imports both plus AutoConfig)
- Revert Version(trl.__version__) back to Version(trl) in trainer.py --
  trainer.py imports Version from unsloth_zoo.utils which already handles
  module objects

---------

Co-authored-by: Daniel Han <danielhanchen@gmail.com>

unsloth/models/_utils.py

@@ -2567,6 +2567,14 @@ def patch_hf_quantizer():
     except Exception as e:
         logger.warning(f"Failed to patch FbgemmFp8HfQuantizer. Error {e}")
 
+    try:
+        from transformers.quantizers.quantizer_torchao import TorchAoHfQuantizer
+
+        TorchAoHfQuantizer.is_trainable = property(make_trainable)
+        TorchAoHfQuantizer.is_qat_trainable = property(make_trainable)
+    except Exception as e:
+        logger.warning(f"Failed to patch TorchAoHfQuantizer. Error {e}")
+
 
 patch_hf_quantizer()
 

unsloth/models/loader.py

@@ -261,12 +261,6 @@ class FastLanguageModel(FastLlamaModel):
                         fast_inference = False
                         break
 
-        # [TODO] For now fast_inference only works with fast_inference ie vLLM
-        if load_in_fp8 != False:
-            if not fast_inference:
-                raise NotImplementedError(
-                    "Unsloth: set `fast_inference = True` when doing `load_in_fp8`."
-                )
         # Check if 4bit is allowed specifically for AMD
         if not ALLOW_BITSANDBYTES and not use_exact_model_name:
             if load_in_4bit or load_in_8bit or model_name.lower().endswith("-bnb-4bit"):
@@ -876,13 +870,6 @@ class FastModel(FastBaseModel):
                         fast_inference = False
                         break
 
-        # [TODO] For now fast_inference only works with fast_inference ie vLLM
-        if load_in_fp8 != False:
-            if not fast_inference:
-                raise NotImplementedError(
-                    "Unsloth: set `fast_inference = True` when doing `load_in_fp8`."
-                )
-
         # Find FP8, BnB 4bit, other mapped names
         old_model_name = model_name
         fp8_mode = None

unsloth/models/loader_utils.py

@@ -350,10 +350,6 @@ def _get_fp8_mode_and_check_settings(
         raise ValueError(
             f"Unsloth: `load_in_fp8` can only be 'row' or 'block', got '{fp8_mode}'"
         )
-    if not fast_inference:
-        raise ValueError(
-            "Unsloth: `load_in_fp8` is only supported for `fast_inference` for now"
-        )
     if full_finetuning:
         raise ValueError(
             "Unsloth: `load_in_fp8` is not compatible with full finetuning"