From f95033833026353fe640f8343df469de8b6d4901 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Thu, 23 Apr 2026 10:40:45 +0000 Subject: [PATCH] vision: enable fast_inference for Gemma 4 via vLLM nightly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add "gemma4" to VLLM_SUPPORTED_VLM so FastLanguageModel(fast_inference=True) stops rejecting unsloth/gemma-4-E2B-it / gemma-4-26b-a4b-it at the vision-model arch gate. vLLM nightly (vllm#39291, merged 2026-04-17) registers Gemma4ForConditionalGeneration as SupportsLoRA; the unsloth-zoo#603 vLLM patches (patch_gemma4_vllm_lora_support + patch_gemma4_vllm_k_eq_v_support) wire LoRA + attention_k_eq_v support on top of that. Also force compilation_config=0 and enforce_eager=True for any gemma4 variant. vLLM's -O3 backend currently hits "Tried to erase Node size_N but it still had N users" in _decompose_size_nodes when splitting the Gemma 4 FX graph (both dense E2B audio path and MoE 26B-A4B language path). Until the upstream FX splitter grows support for these size() users, eager execution preserves correctness — bitwise token parity with HF naive confirmed at 25/25, 4/4, 5/5 on 3 chat prompts × 32 tokens. --- unsloth/models/vision.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 93a1069c3f..59f577c215 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -245,6 +245,7 @@ VLLM_SUPPORTED_VLM = [ "mistral3", "qwen3_vl", "qwen3_vl_moe", + "gemma4", ] VLLM_NON_LORA_VLM = [ "mllama", @@ -1053,6 +1054,16 @@ class FastBaseModel: is_vision_model = is_vlm, fp8_mode = fp8_mode, ) + # why: vLLM's -O3 compile backend (compilation_config=3) hits + # _decompose_size_nodes "Tried to erase Node size_N but it still + # had N users" on Gemma 4 multimodal models (both dense E2B and + # MoE 26B-A4B). Until the upstream FX splitter is fixed, fall + # back to compilation_config=0 (no piecewise compile) for gemma4 + # so fast_inference still loads. Runtime is slower than -O3 but + # correctness is preserved. + if any(arch == "gemma4" for arch in (model_types or [])): + load_vllm_kwargs.setdefault("compilation_config", 0) + load_vllm_kwargs.setdefault("enforce_eager", True) for allowed_arg in allowed_args: if allowed_arg not in load_vllm_kwargs and allowed_arg in kwargs: load_vllm_kwargs[allowed_arg] = kwargs[allowed_arg]