From 3f9e03ff1b0a3e0d2dd8277283c33b13638ed522 Mon Sep 17 00:00:00 2001 From: Datta Nimmaturi Date: Wed, 25 Feb 2026 20:22:18 +0530 Subject: [PATCH] Allow fp8 for non fast inference (#3904) * Allow fp8 for non fast inference * Extensive fp8 alow and quantizer patch * Clean up commented-out code, duplicate import, and revert unnecessary Version() changes - Delete commented-out FP8 fast_inference guard in FastModel (loader.py) instead of leaving it commented -- matches FastLanguageModel which was properly deleted - Delete commented-out fast_inference guard in loader_utils.py - Remove duplicate `from transformers import GenerationConfig, CompileConfig` in vision.py (line 112 already imports both plus AutoConfig) - Revert Version(trl.__version__) back to Version(trl) in trainer.py -- trainer.py imports Version from unsloth_zoo.utils which already handles module objects --------- Co-authored-by: Daniel Han --- unsloth/models/_utils.py | 8 ++++++++ unsloth/models/loader.py | 13 ------------- unsloth/models/loader_utils.py | 4 ---- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index e7cc5ff1ef..5f4927b92f 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -2567,6 +2567,14 @@ def patch_hf_quantizer(): except Exception as e: logger.warning(f"Failed to patch FbgemmFp8HfQuantizer. Error {e}") + try: + from transformers.quantizers.quantizer_torchao import TorchAoHfQuantizer + + TorchAoHfQuantizer.is_trainable = property(make_trainable) + TorchAoHfQuantizer.is_qat_trainable = property(make_trainable) + except Exception as e: + logger.warning(f"Failed to patch TorchAoHfQuantizer. Error {e}") + patch_hf_quantizer() diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 04c15bb2c3..72a6d7d10f 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -261,12 +261,6 @@ class FastLanguageModel(FastLlamaModel): fast_inference = False break - # [TODO] For now fast_inference only works with fast_inference ie vLLM - if load_in_fp8 != False: - if not fast_inference: - raise NotImplementedError( - "Unsloth: set `fast_inference = True` when doing `load_in_fp8`." - ) # Check if 4bit is allowed specifically for AMD if not ALLOW_BITSANDBYTES and not use_exact_model_name: if load_in_4bit or load_in_8bit or model_name.lower().endswith("-bnb-4bit"): @@ -876,13 +870,6 @@ class FastModel(FastBaseModel): fast_inference = False break - # [TODO] For now fast_inference only works with fast_inference ie vLLM - if load_in_fp8 != False: - if not fast_inference: - raise NotImplementedError( - "Unsloth: set `fast_inference = True` when doing `load_in_fp8`." - ) - # Find FP8, BnB 4bit, other mapped names old_model_name = model_name fp8_mode = None diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index f17ebc53e3..40ac49ca78 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -350,10 +350,6 @@ def _get_fp8_mode_and_check_settings( raise ValueError( f"Unsloth: `load_in_fp8` can only be 'row' or 'block', got '{fp8_mode}'" ) - if not fast_inference: - raise ValueError( - "Unsloth: `load_in_fp8` is only supported for `fast_inference` for now" - ) if full_finetuning: raise ValueError( "Unsloth: `load_in_fp8` is not compatible with full finetuning"