From a66b828a98d8abbdfa6248fd977bdfcf9c71dfd5 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 25 Nov 2025 07:38:51 -0800 Subject: [PATCH] Update loader.py --- unsloth/models/loader.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 951bcb1e0c..1bc8e265a2 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -211,6 +211,12 @@ class FastLanguageModel(FastLlamaModel): "Unsloth: Please install vLLM before enabling `fast_inference`!\n" "You can do this in a terminal via `pip install vllm`" ) + # [TODO] For now fast_inference only works with fast_inference ie vLLM + if load_in_fp8 != False: + if not fast_inference: + raise NotImplementedError( + "Unsloth: set `fast_inference = True` when doing `load_in_fp8`." + ) # Check if 4bit is allowed specifically for AMD if not ALLOW_BITSANDBYTES and not use_exact_model_name: if load_in_4bit or load_in_8bit or model_name.lower().endswith("-bnb-4bit"): @@ -728,6 +734,19 @@ class FastModel(FastBaseModel): ) load_in_4bit = False + if fast_inference: + if importlib.util.find_spec("vllm") is None: + raise ImportError( + "Unsloth: Please install vLLM before enabling `fast_inference`!\n" + "You can do this in a terminal via `pip install vllm`" + ) + # [TODO] For now fast_inference only works with fast_inference ie vLLM + if load_in_fp8 != False: + if not fast_inference: + raise NotImplementedError( + "Unsloth: set `fast_inference = True` when doing `load_in_fp8`." + ) + # Find FP8, BnB 4bit, other mapped names old_model_name = model_name fp8_mode = None