From 4c06c1dcc771f3d4f2aecc8d3ab77f2e01aa9107 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 4 Jun 2026 00:56:53 -0700 Subject: [PATCH] Studio: enable audio input for Gemma 4 GGUFs; default chat model to Qwen3.5-4B-MTP (#6000) * Studio: enable audio input for Gemma 4 GGUF models Audio file upload was disabled for Gemma 4 vision+audio GGUFs (e.g. gemma-4-12b-it-GGUF) even though their mmproj carries an audio encoder (clip.has_audio_encoder, gemma4ua). Two causes: - Audio-input detection only matched Gemma 3n's ; Gemma 4 uses <|audio|>, so audio_vlm was never detected. - The GGUF load/status responses hardcoded has_audio_input=False, so the flag was dropped even when audio_vlm was detected (affected Gemma 3n GGUFs too). Changes: - Recognize <|audio|> alongside in the llama-server token probe and the tokenizer-config pattern. - Read clip.has_audio_encoder from the mmproj as an independent, model-agnostic signal (read_mmproj_audio_capability). - Emit the computed has_audio_input on the GGUF load/status responses. - Tests for the new pattern and the mmproj reader. * Studio: default chat model and dataset helper to Qwen3.5-4B-MTP Switch the auto-loaded chat default and the dataset-analysis helper GGUF from gemma-4-E2B-it to unsloth/Qwen3.5-4B-MTP-GGUF (UD-Q4_K_XL). * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- studio/backend/core/inference/llama_cpp.py | 37 ++++++++- studio/backend/routes/inference.py | 6 +- .../tests/test_audio_token_detection.py | 46 +++++++++++ studio/backend/tests/test_gguf_metadata.py | 67 +++++++++++++++- studio/backend/utils/datasets/llm_assist.py | 2 +- studio/backend/utils/models/gguf_metadata.py | 78 +++++++++++++++++++ studio/backend/utils/models/model_config.py | 3 +- .../src/features/chat/api/chat-adapter.ts | 16 ++-- 8 files changed, 240 insertions(+), 15 deletions(-) create mode 100644 studio/backend/tests/test_audio_token_detection.py diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 7bcf02dc35..0f23549138 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -697,6 +697,9 @@ class LlamaCppBackend: self._is_audio: bool = False self._audio_type: Optional[str] = None self._audio_probed: bool = False + # Audio INPUT capability (distinct from _is_audio, which is TTS output). + self._has_audio_input: bool = False + self._mmproj_has_audio: bool = False # clip.has_audio_encoder, set at load # Monotonic timestamp set in _kill_process; read by load_model # to decide whether to wait for the VRAM reclaim to finish. self._last_kill_monotonic: float = 0.0 @@ -2782,6 +2785,12 @@ class LlamaCppBackend: if not self._healthy: return False self._audio_type = detected + # Re-derive after a retried probe (_mmproj_has_audio persists). + from utils.models.model_config import is_audio_input_type + + self._has_audio_input = bool( + is_audio_input_type(self._audio_type) + ) or bool(self._mmproj_has_audio) if not self._healthy: return False return True @@ -3095,6 +3104,21 @@ class LlamaCppBackend: "image input will be disabled for this session" ) + # Audio input straight from the mmproj (clip.has_audio_encoder), + # independent of token names. + self._mmproj_has_audio = False + if launch_mmproj_path: + try: + from utils.models.gguf_metadata import ( + read_mmproj_audio_capability, + ) + + self._mmproj_has_audio = bool( + read_mmproj_audio_capability(launch_mmproj_path) + ) + except Exception as e: + logger.debug(f"mmproj audio-capability read failed: {e}") + cmd = [ binary, "-m", @@ -3527,6 +3551,7 @@ class LlamaCppBackend: self._is_audio = False self._audio_type = None self._audio_probed = False + self._has_audio_input = False try: detected = self._detect_audio_type_strict() self._audio_probed = True @@ -3558,6 +3583,13 @@ class LlamaCppBackend: return False self._audio_type = detected + # Audio input = token probe (audio_vlm/whisper) OR mmproj audio encoder. + from utils.models.model_config import is_audio_input_type + + self._has_audio_input = bool(is_audio_input_type(self._audio_type)) or bool( + self._mmproj_has_audio + ) + if not self._healthy: return False return True @@ -3901,6 +3933,8 @@ class LlamaCppBackend: self._is_audio = False self._audio_type = None self._audio_probed = False + self._has_audio_input = False + self._mmproj_has_audio = False self._port = None self._healthy = False self._context_length = None @@ -5591,7 +5625,8 @@ class LlamaCppBackend: return "csm" if len(_tok("<|startoftranscript|>")) == 1: return "whisper" - if len(_tok("")) == 1: + # Gemma 3n: ; Gemma 4: <|audio|> (not csm's <|AUDIO|>). + if len(_tok("")) == 1 or len(_tok("<|audio|>")) == 1: return "audio_vlm" if ( len(_tok("<|bicodec_semantic_0|>")) == 1 diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 2964a0801f..15d73405cb 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -794,7 +794,7 @@ async def load_model( is_gguf = True, is_audio = _gguf_is_audio, audio_type = _gguf_audio, - has_audio_input = False, + has_audio_input = getattr(llama_backend, "_has_audio_input", False), inference = inference_config, requires_trust_remote_code = bool( inference_config.get("trust_remote_code", False) @@ -1044,7 +1044,7 @@ async def load_model( is_gguf = True, is_audio = _gguf_is_audio, audio_type = _gguf_audio, - has_audio_input = False, + has_audio_input = llama_backend._has_audio_input, inference = inference_config, requires_trust_remote_code = bool( inference_config.get("trust_remote_code", False) @@ -1531,7 +1531,7 @@ async def get_status( gguf_variant = llama_backend.hf_variant, is_audio = getattr(llama_backend, "_is_audio", False), audio_type = _audio_type, - has_audio_input = False, + has_audio_input = getattr(llama_backend, "_has_audio_input", False), loading = [], loaded = [_display_model_id] if _display_model_id else [], inference = _inference_cfg, diff --git a/studio/backend/tests/test_audio_token_detection.py b/studio/backend/tests/test_audio_token_detection.py new file mode 100644 index 0000000000..a3ea7c89a7 --- /dev/null +++ b/studio/backend/tests/test_audio_token_detection.py @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Tests for tokenizer-based audio_type detection patterns, covering both +Gemma 3n () and Gemma 4 (<|audio|>) audio-input tokens.""" + +from __future__ import annotations + +from utils.models.model_config import _AUDIO_TOKEN_PATTERNS, is_audio_input_type + + +def _classify(tokens: list[str]) -> str | None: + """Mirror _detect_audio_from_tokenizer._check_token_patterns: first match + in dict order wins.""" + for audio_type, check in _AUDIO_TOKEN_PATTERNS.items(): + if check(tokens): + return audio_type + return None + + +def test_gemma3n_audio_soft_token_is_audio_vlm(): + assert ( + _classify(["", "", ""]) == "audio_vlm" + ) + + +def test_gemma4_pipe_audio_token_is_audio_vlm(): + # Gemma 4 uses <|audio|> (and <|image|>) instead of *_soft_token. + assert _classify(["", "<|image|>", "<|audio|>"]) == "audio_vlm" + + +def test_csm_uppercase_audio_not_classified_as_audio_vlm(): + # csm uses uppercase <|AUDIO|> + <|audio_eos|>; must stay csm, not audio_vlm. + tokens = ["<|AUDIO|>", "<|audio_eos|>"] + assert _classify(tokens) == "csm" + + +def test_audio_vlm_and_whisper_accept_audio_input(): + assert is_audio_input_type("audio_vlm") is True + assert is_audio_input_type("whisper") is True + assert is_audio_input_type("snac") is False + assert is_audio_input_type(None) is False + + +def test_non_audio_tokens_classify_none(): + assert _classify(["", "", ""]) is None diff --git a/studio/backend/tests/test_gguf_metadata.py b/studio/backend/tests/test_gguf_metadata.py index cf1a17347f..e5040e306c 100644 --- a/studio/backend/tests/test_gguf_metadata.py +++ b/studio/backend/tests/test_gguf_metadata.py @@ -14,6 +14,7 @@ from utils.models.gguf_metadata import ( is_mmproj_by_metadata, pairing_score, read_gguf_general_metadata, + read_mmproj_audio_capability, ) @@ -21,6 +22,7 @@ _GGUF_MAGIC = 0x46554747 _VTYPE_STRING = 8 _VTYPE_UINT32 = 4 _VTYPE_ARRAY = 9 +_VTYPE_BOOL = 7 def _enc_string(s: str) -> bytes: @@ -38,6 +40,14 @@ def _enc_kv_uint32(key: str, value: int) -> bytes: ) +def _enc_kv_bool(key: str, value: bool) -> bytes: + return ( + _enc_string(key) + + struct.pack(" bytes: vals = list(values) out = _enc_string(key) + struct.pack(" Path: """Minimal GGUF: header + KV body, no tensors.""" extra_uint32 = extra_uint32 or {} extra_string_arrays = extra_string_arrays or {} - kv_count = len(general_strings) + len(extra_uint32) + len(extra_string_arrays) + extra_bools = extra_bools or {} + kv_count = ( + len(general_strings) + + len(extra_uint32) + + len(extra_string_arrays) + + len(extra_bools) + ) body = b"" for k, v in general_strings.items(): body += _enc_kv_string(k, v) @@ -65,6 +82,8 @@ def _write_synthetic_gguf( body += _enc_kv_uint32(k, v) for k, v in extra_string_arrays.items(): body += _enc_kv_string_array(k, v) + for k, v in extra_bools.items(): + body += _enc_kv_bool(k, v) header = struct.pack( " Optional[_CacheKey]: try: @@ -193,6 +197,80 @@ def _skip_gguf_value(f, vtype: int) -> bool: return True +def _parse_gguf_bool(path: str, wanted_key: str) -> Optional[bool]: + """Bool value of ``wanted_key`` (GGUF vtype 7), or ``None`` if absent / + unreadable. Mirrors ``_parse_gguf_header`` for a single bool key.""" + try: + with open(path, "rb") as f: + head = f.read(24) + if len(head) < 24: + return None + magic, _version, _tcount, kv_count = struct.unpack(" 1 << 20: # 1 MB sanity bound + break + kbytes = f.read(klen) + if len(kbytes) < klen: + break + key = kbytes.decode("utf-8", "replace") + vt_bytes = f.read(4) + if len(vt_bytes) < 4: + break + vtype = struct.unpack(" Optional[bool]: + """Cached single-bool-key read, keyed by (path, mtime, size, wanted_key).""" + fkey = _cache_key(path) + if fkey is None: + return None + ckey = (fkey, wanted_key) + with _CACHE_LOCK: + if ckey in _BOOL_CACHE: + return _BOOL_CACHE[ckey] + result = _parse_gguf_bool(path, wanted_key) + with _CACHE_LOCK: + while len(_BOOL_CACHE) >= _CACHE_MAX_ENTRIES: + try: + _BOOL_CACHE.pop(next(iter(_BOOL_CACHE))) + except StopIteration: + break + _BOOL_CACHE[ckey] = result + return result + + +def read_mmproj_audio_capability(path: str) -> Optional[bool]: + """``clip.has_audio_encoder`` from an mmproj GGUF (e.g. Gemma 4's gemma4ua): + ``True``/``False`` if present, ``None`` if absent / unreadable. Flags + audio-input models independently of tokenizer token names.""" + return _read_gguf_bool(path, "clip.has_audio_encoder") + + def is_mmproj_by_metadata(meta: Optional[Dict[str, str]]) -> Optional[bool]: """True/False from ``general.type``; None means fall back to filename.""" if not meta: diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index dc34444ccb..b488a19953 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -814,7 +814,8 @@ _audio_detection_cache: Dict[str, Optional[str]] = {} _AUDIO_TOKEN_PATTERNS = { "csm": lambda tokens: "<|AUDIO|>" in tokens and "<|audio_eos|>" in tokens, "whisper": lambda tokens: "<|startoftranscript|>" in tokens, - "audio_vlm": lambda tokens: "" in tokens, + # Gemma 3n: ; Gemma 4: <|audio|> (not csm's <|AUDIO|>). + "audio_vlm": lambda tokens: "" in tokens or "<|audio|>" in tokens, "bicodec": lambda tokens: any(t.startswith("<|bicodec_") for t in tokens), "dac": lambda tokens: ( "<|audio_start|>" in tokens diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index 1fcbe7ff70..9a7f95df87 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -1149,13 +1149,13 @@ async function autoLoadSmallestModel(): Promise<{ toast("Downloading a small model…", { id: toastId, description: - "No downloaded models found. Fetching Gemma-4-E2B-it (UD-Q4_K_XL).", + "No downloaded models found. Fetching Qwen3.5-4B-MTP (UD-Q4_K_XL).", duration: 30000, }); try { if ( !(await canAutoLoad({ - model_path: "unsloth/gemma-4-E2B-it-GGUF", + model_path: "unsloth/Qwen3.5-4B-MTP-GGUF", max_seq_length: 0, is_lora: false, gguf_variant: "UD-Q4_K_XL", @@ -1166,7 +1166,7 @@ async function autoLoadSmallestModel(): Promise<{ } loadAttempts += 1; const loadResp = await loadModel({ - model_path: "unsloth/gemma-4-E2B-it-GGUF", + model_path: "unsloth/Qwen3.5-4B-MTP-GGUF", hf_token: hfToken, max_seq_length: 0, load_in_4bit: true, @@ -1176,7 +1176,7 @@ async function autoLoadSmallestModel(): Promise<{ }); useChatRuntimeStore .getState() - .setCheckpoint("unsloth/gemma-4-E2B-it-GGUF", "UD-Q4_K_XL"); + .setCheckpoint("unsloth/Qwen3.5-4B-MTP-GGUF", "UD-Q4_K_XL"); const store = useChatRuntimeStore.getState(); store.setModelRequiresTrustRemoteCode( loadResp.requires_trust_remote_code ?? false, @@ -1186,13 +1186,13 @@ async function autoLoadSmallestModel(): Promise<{ maxTokens: loadResp.context_length ?? 131072, }); const defaultModel: ChatModelSummary = { - id: "unsloth/gemma-4-E2B-it-GGUF", - name: loadResp.display_name ?? "gemma-4-E2B-it-GGUF", + id: "unsloth/Qwen3.5-4B-MTP-GGUF", + name: loadResp.display_name ?? "Qwen3.5-4B-MTP-GGUF", isVision: loadResp.is_vision ?? false, isLora: false, isGguf: true, }; - if (!store.models.some((m) => m.id === "unsloth/gemma-4-E2B-it-GGUF")) { + if (!store.models.some((m) => m.id === "unsloth/Qwen3.5-4B-MTP-GGUF")) { store.setModels([...store.models, defaultModel]); } useChatRuntimeStore.setState({ @@ -1212,7 +1212,7 @@ async function autoLoadSmallestModel(): Promise<{ chatTemplateOverride: null, loadedIsMultimodal: isMultimodalResponse(loadResp), }); - toast.success("Loaded Gemma-4-E2B-it (UD-Q4_K_XL)", { id: toastId }); + toast.success("Loaded Qwen3.5-4B-MTP (UD-Q4_K_XL)", { id: toastId }); return { loaded: true, blockedByTrustRemoteCode: false }; } catch { toast.dismiss(toastId);