Studio: enable audio input for Gemma 4 GGUFs; default chat model to Qwen3.5-4B-MTP (#6000)

* Studio: enable audio input for Gemma 4 GGUF models

Audio file upload was disabled for Gemma 4 vision+audio GGUFs (e.g.
gemma-4-12b-it-GGUF) even though their mmproj carries an audio encoder
(clip.has_audio_encoder, gemma4ua). Two causes:

- Audio-input detection only matched Gemma 3n's <audio_soft_token>;
  Gemma 4 uses <|audio|>, so audio_vlm was never detected.
- The GGUF load/status responses hardcoded has_audio_input=False, so the
  flag was dropped even when audio_vlm was detected (affected Gemma 3n
  GGUFs too).

Changes:
- Recognize <|audio|> alongside <audio_soft_token> in the llama-server
  token probe and the tokenizer-config pattern.
- Read clip.has_audio_encoder from the mmproj as an independent,
  model-agnostic signal (read_mmproj_audio_capability).
- Emit the computed has_audio_input on the GGUF load/status responses.
- Tests for the new pattern and the mmproj reader.

* Studio: default chat model and dataset helper to Qwen3.5-4B-MTP

Switch the auto-loaded chat default and the dataset-analysis helper GGUF
from gemma-4-E2B-it to unsloth/Qwen3.5-4B-MTP-GGUF (UD-Q4_K_XL).

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

studio/backend/core/inference/llama_cpp.py

@@ -697,6 +697,9 @@ class LlamaCppBackend:
         self._is_audio: bool = False
         self._audio_type: Optional[str] = None
         self._audio_probed: bool = False
+        # Audio INPUT capability (distinct from _is_audio, which is TTS output).
+        self._has_audio_input: bool = False
+        self._mmproj_has_audio: bool = False  # clip.has_audio_encoder, set at load
         # Monotonic timestamp set in _kill_process; read by load_model
         # to decide whether to wait for the VRAM reclaim to finish.
         self._last_kill_monotonic: float = 0.0
@@ -2782,6 +2785,12 @@ class LlamaCppBackend:
                             if not self._healthy:
                                 return False
                             self._audio_type = detected
+                    # Re-derive after a retried probe (_mmproj_has_audio persists).
+                    from utils.models.model_config import is_audio_input_type
+
+                    self._has_audio_input = bool(
+                        is_audio_input_type(self._audio_type)
+                    ) or bool(self._mmproj_has_audio)
                 if not self._healthy:
                     return False
                 return True
@@ -3095,6 +3104,21 @@ class LlamaCppBackend:
                         "image input will be disabled for this session"
                     )
 
+                # Audio input straight from the mmproj (clip.has_audio_encoder),
+                # independent of token names.
+                self._mmproj_has_audio = False
+                if launch_mmproj_path:
+                    try:
+                        from utils.models.gguf_metadata import (
+                            read_mmproj_audio_capability,
+                        )
+
+                        self._mmproj_has_audio = bool(
+                            read_mmproj_audio_capability(launch_mmproj_path)
+                        )
+                    except Exception as e:
+                        logger.debug(f"mmproj audio-capability read failed: {e}")
+
                 cmd = [
                     binary,
                     "-m",
@@ -3527,6 +3551,7 @@ class LlamaCppBackend:
             self._is_audio = False
             self._audio_type = None
             self._audio_probed = False
+            self._has_audio_input = False
             try:
                 detected = self._detect_audio_type_strict()
                 self._audio_probed = True
@@ -3558,6 +3583,13 @@ class LlamaCppBackend:
                         return False
                     self._audio_type = detected
 
+            # Audio input = token probe (audio_vlm/whisper) OR mmproj audio encoder.
+            from utils.models.model_config import is_audio_input_type
+
+            self._has_audio_input = bool(is_audio_input_type(self._audio_type)) or bool(
+                self._mmproj_has_audio
+            )
+
             if not self._healthy:
                 return False
             return True
@@ -3901,6 +3933,8 @@ class LlamaCppBackend:
             self._is_audio = False
             self._audio_type = None
             self._audio_probed = False
+            self._has_audio_input = False
+            self._mmproj_has_audio = False
             self._port = None
             self._healthy = False
             self._context_length = None
@@ -5591,7 +5625,8 @@ class LlamaCppBackend:
                 return "csm"
             if len(_tok("<|startoftranscript|>")) == 1:
                 return "whisper"
-            if len(_tok("<audio_soft_token>")) == 1:
+            # Gemma 3n: <audio_soft_token>; Gemma 4: <|audio|> (not csm's <|AUDIO|>).
+            if len(_tok("<audio_soft_token>")) == 1 or len(_tok("<|audio|>")) == 1:
                 return "audio_vlm"
             if (
                 len(_tok("<|bicodec_semantic_0|>")) == 1

studio/backend/routes/inference.py

@@ -794,7 +794,7 @@ async def load_model(
                     is_gguf = True,
                     is_audio = _gguf_is_audio,
                     audio_type = _gguf_audio,
-                    has_audio_input = False,
+                    has_audio_input = getattr(llama_backend, "_has_audio_input", False),
                     inference = inference_config,
                     requires_trust_remote_code = bool(
                         inference_config.get("trust_remote_code", False)
@@ -1044,7 +1044,7 @@ async def load_model(
                 is_gguf = True,
                 is_audio = _gguf_is_audio,
                 audio_type = _gguf_audio,
-                has_audio_input = False,
+                has_audio_input = llama_backend._has_audio_input,
                 inference = inference_config,
                 requires_trust_remote_code = bool(
                     inference_config.get("trust_remote_code", False)
@@ -1531,7 +1531,7 @@ async def get_status(
                 gguf_variant = llama_backend.hf_variant,
                 is_audio = getattr(llama_backend, "_is_audio", False),
                 audio_type = _audio_type,
-                has_audio_input = False,
+                has_audio_input = getattr(llama_backend, "_has_audio_input", False),
                 loading = [],
                 loaded = [_display_model_id] if _display_model_id else [],
                 inference = _inference_cfg,

studio/backend/tests/test_audio_token_detection.py

@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Tests for tokenizer-based audio_type detection patterns, covering both
+Gemma 3n (<audio_soft_token>) and Gemma 4 (<|audio|>) audio-input tokens."""
+
+from __future__ import annotations
+
+from utils.models.model_config import _AUDIO_TOKEN_PATTERNS, is_audio_input_type
+
+
+def _classify(tokens: list[str]) -> str | None:
+    """Mirror _detect_audio_from_tokenizer._check_token_patterns: first match
+    in dict order wins."""
+    for audio_type, check in _AUDIO_TOKEN_PATTERNS.items():
+        if check(tokens):
+            return audio_type
+    return None
+
+
+def test_gemma3n_audio_soft_token_is_audio_vlm():
+    assert (
+        _classify(["<bos>", "<audio_soft_token>", "<image_soft_token>"]) == "audio_vlm"
+    )
+
+
+def test_gemma4_pipe_audio_token_is_audio_vlm():
+    # Gemma 4 uses <|audio|> (and <|image|>) instead of *_soft_token.
+    assert _classify(["<bos>", "<|image|>", "<|audio|>"]) == "audio_vlm"
+
+
+def test_csm_uppercase_audio_not_classified_as_audio_vlm():
+    # csm uses uppercase <|AUDIO|> + <|audio_eos|>; must stay csm, not audio_vlm.
+    tokens = ["<|AUDIO|>", "<|audio_eos|>"]
+    assert _classify(tokens) == "csm"
+
+
+def test_audio_vlm_and_whisper_accept_audio_input():
+    assert is_audio_input_type("audio_vlm") is True
+    assert is_audio_input_type("whisper") is True
+    assert is_audio_input_type("snac") is False
+    assert is_audio_input_type(None) is False
+
+
+def test_non_audio_tokens_classify_none():
+    assert _classify(["<bos>", "<eos>", "<pad>"]) is None

studio/backend/tests/test_gguf_metadata.py

@@ -14,6 +14,7 @@ from utils.models.gguf_metadata import (
     is_mmproj_by_metadata,
     pairing_score,
     read_gguf_general_metadata,
+    read_mmproj_audio_capability,
 )
 
 
@@ -21,6 +22,7 @@ _GGUF_MAGIC = 0x46554747
 _VTYPE_STRING = 8
 _VTYPE_UINT32 = 4
 _VTYPE_ARRAY = 9
+_VTYPE_BOOL = 7
 
 
 def _enc_string(s: str) -> bytes:
@@ -38,6 +40,14 @@ def _enc_kv_uint32(key: str, value: int) -> bytes:
     )
 
 
+def _enc_kv_bool(key: str, value: bool) -> bytes:
+    return (
+        _enc_string(key)
+        + struct.pack("<I", _VTYPE_BOOL)
+        + struct.pack("<B", 1 if value else 0)
+    )
+
+
 def _enc_kv_string_array(key: str, values: Iterable[str]) -> bytes:
     vals = list(values)
     out = _enc_string(key) + struct.pack("<I", _VTYPE_ARRAY)
@@ -53,11 +63,18 @@ def _write_synthetic_gguf(
     *,
     extra_uint32: Mapping[str, int] | None = None,
     extra_string_arrays: Mapping[str, Iterable[str]] | None = None,
+    extra_bools: Mapping[str, bool] | None = None,
 ) -> Path:
     """Minimal GGUF: header + KV body, no tensors."""
     extra_uint32 = extra_uint32 or {}
     extra_string_arrays = extra_string_arrays or {}
-    kv_count = len(general_strings) + len(extra_uint32) + len(extra_string_arrays)
+    extra_bools = extra_bools or {}
+    kv_count = (
+        len(general_strings)
+        + len(extra_uint32)
+        + len(extra_string_arrays)
+        + len(extra_bools)
+    )
     body = b""
     for k, v in general_strings.items():
         body += _enc_kv_string(k, v)
@@ -65,6 +82,8 @@ def _write_synthetic_gguf(
         body += _enc_kv_uint32(k, v)
     for k, v in extra_string_arrays.items():
         body += _enc_kv_string_array(k, v)
+    for k, v in extra_bools.items():
+        body += _enc_kv_bool(k, v)
     header = struct.pack(
         "<IIQQ",
         _GGUF_MAGIC,
@@ -214,3 +233,49 @@ def test_pairing_score_no_overlap_returns_zero():
     assert pairing_score({"general.basename": "Foo"}, {}) == 0
     assert pairing_score({}, {"general.basename": "Foo"}) == 0
     assert pairing_score(None, {"general.basename": "Foo"}) == 0
+
+
+# --- read_mmproj_audio_capability --------------------------------------
+
+
+def test_mmproj_audio_capability_true(tmp_path: Path):
+    """clip.has_audio_encoder=True (e.g. Gemma 4's gemma4ua projector)."""
+    p = _write_synthetic_gguf(
+        tmp_path / "mmproj.gguf",
+        {"general.type": "mmproj"},
+        extra_bools = {
+            "clip.has_vision_encoder": True,
+            "clip.has_audio_encoder": True,
+        },
+    )
+    assert read_mmproj_audio_capability(str(p)) is True
+
+
+def test_mmproj_audio_capability_false(tmp_path: Path):
+    """Vision-only projector: key present but false."""
+    p = _write_synthetic_gguf(
+        tmp_path / "mmproj.gguf",
+        {"general.type": "mmproj"},
+        extra_bools = {
+            "clip.has_vision_encoder": True,
+            "clip.has_audio_encoder": False,
+        },
+    )
+    assert read_mmproj_audio_capability(str(p)) is False
+
+
+def test_mmproj_audio_capability_absent_returns_none(tmp_path: Path):
+    """Key absent (older/vision-only mmproj): None, not False."""
+    p = _write_synthetic_gguf(
+        tmp_path / "mmproj.gguf",
+        {"general.type": "mmproj"},
+        extra_bools = {"clip.has_vision_encoder": True},
+    )
+    assert read_mmproj_audio_capability(str(p)) is None
+
+
+def test_mmproj_audio_capability_missing_or_non_gguf(tmp_path: Path):
+    assert read_mmproj_audio_capability(str(tmp_path / "nope.gguf")) is None
+    junk = tmp_path / "garbage.gguf"
+    junk.write_bytes(b"not a gguf header at all")
+    assert read_mmproj_audio_capability(str(junk)) is None

studio/backend/utils/datasets/llm_assist.py

@@ -26,7 +26,7 @@ from loggers import get_logger
 
 logger = get_logger(__name__)
 
-DEFAULT_HELPER_MODEL_REPO = "unsloth/gemma-4-E2B-it-GGUF"
+DEFAULT_HELPER_MODEL_REPO = "unsloth/Qwen3.5-4B-MTP-GGUF"
 DEFAULT_HELPER_MODEL_VARIANT = "UD-Q4_K_XL"
 
 README_MAX_CHARS = 1500

studio/backend/utils/models/gguf_metadata.py

@@ -47,6 +47,10 @@ _METADATA_CACHE: Dict[_CacheKey, Optional[Dict[str, str]]] = {}
 _CACHE_LOCK = threading.Lock()
 _CACHE_MAX_ENTRIES = 4096
 
+# Separate cache for single bool capability keys (e.g. clip.has_audio_encoder),
+# keyed by (file cache key, wanted key). None = key absent / file unreadable.
+_BOOL_CACHE: Dict[Tuple[_CacheKey, str], Optional[bool]] = {}
+
 
 def _cache_key(path: str) -> Optional[_CacheKey]:
     try:
@@ -193,6 +197,80 @@ def _skip_gguf_value(f, vtype: int) -> bool:
     return True
 
 
+def _parse_gguf_bool(path: str, wanted_key: str) -> Optional[bool]:
+    """Bool value of ``wanted_key`` (GGUF vtype 7), or ``None`` if absent /
+    unreadable. Mirrors ``_parse_gguf_header`` for a single bool key."""
+    try:
+        with open(path, "rb") as f:
+            head = f.read(24)
+            if len(head) < 24:
+                return None
+            magic, _version, _tcount, kv_count = struct.unpack("<IIQQ", head)
+            if magic != _GGUF_MAGIC:
+                return None
+
+            for _ in range(kv_count):
+                try:
+                    klen_bytes = f.read(8)
+                    if len(klen_bytes) < 8:
+                        break
+                    klen = struct.unpack("<Q", klen_bytes)[0]
+                    if klen > 1 << 20:  # 1 MB sanity bound
+                        break
+                    kbytes = f.read(klen)
+                    if len(kbytes) < klen:
+                        break
+                    key = kbytes.decode("utf-8", "replace")
+                    vt_bytes = f.read(4)
+                    if len(vt_bytes) < 4:
+                        break
+                    vtype = struct.unpack("<I", vt_bytes)[0]
+
+                    if key == wanted_key and vtype == 7:  # BOOL (1 byte)
+                        bbyte = f.read(1)
+                        if len(bbyte) < 1:
+                            break
+                        return bbyte[0] != 0
+                    if not _skip_gguf_value(f, vtype):
+                        break
+                except (struct.error, UnicodeDecodeError):
+                    break
+    except OSError as e:
+        logger.debug(f"_parse_gguf_bool: cannot open {path}: {e}")
+        return None
+    except Exception as e:
+        logger.debug(f"_parse_gguf_bool: parse failure on {path}: {e}")
+        return None
+    return None
+
+
+def _read_gguf_bool(path: str, wanted_key: str) -> Optional[bool]:
+    """Cached single-bool-key read, keyed by (path, mtime, size, wanted_key)."""
+    fkey = _cache_key(path)
+    if fkey is None:
+        return None
+    ckey = (fkey, wanted_key)
+    with _CACHE_LOCK:
+        if ckey in _BOOL_CACHE:
+            return _BOOL_CACHE[ckey]
+    result = _parse_gguf_bool(path, wanted_key)
+    with _CACHE_LOCK:
+        while len(_BOOL_CACHE) >= _CACHE_MAX_ENTRIES:
+            try:
+                _BOOL_CACHE.pop(next(iter(_BOOL_CACHE)))
+            except StopIteration:
+                break
+        _BOOL_CACHE[ckey] = result
+    return result
+
+
+def read_mmproj_audio_capability(path: str) -> Optional[bool]:
+    """``clip.has_audio_encoder`` from an mmproj GGUF (e.g. Gemma 4's gemma4ua):
+    ``True``/``False`` if present, ``None`` if absent / unreadable. Flags
+    audio-input models independently of tokenizer token names."""
+    return _read_gguf_bool(path, "clip.has_audio_encoder")
+
+
 def is_mmproj_by_metadata(meta: Optional[Dict[str, str]]) -> Optional[bool]:
     """True/False from ``general.type``; None means fall back to filename."""
     if not meta:

studio/backend/utils/models/model_config.py

@@ -814,7 +814,8 @@ _audio_detection_cache: Dict[str, Optional[str]] = {}
 _AUDIO_TOKEN_PATTERNS = {
     "csm": lambda tokens: "<|AUDIO|>" in tokens and "<|audio_eos|>" in tokens,
     "whisper": lambda tokens: "<|startoftranscript|>" in tokens,
-    "audio_vlm": lambda tokens: "<audio_soft_token>" in tokens,
+    # Gemma 3n: <audio_soft_token>; Gemma 4: <|audio|> (not csm's <|AUDIO|>).
+    "audio_vlm": lambda tokens: "<audio_soft_token>" in tokens or "<|audio|>" in tokens,
     "bicodec": lambda tokens: any(t.startswith("<|bicodec_") for t in tokens),
     "dac": lambda tokens: (
         "<|audio_start|>" in tokens

studio/frontend/src/features/chat/api/chat-adapter.ts

@@ -1149,13 +1149,13 @@ async function autoLoadSmallestModel(): Promise<{
     toast("Downloading a small model…", {
       id: toastId,
       description:
-        "No downloaded models found. Fetching Gemma-4-E2B-it (UD-Q4_K_XL).",
+        "No downloaded models found. Fetching Qwen3.5-4B-MTP (UD-Q4_K_XL).",
       duration: 30000,
     });
     try {
       if (
         !(await canAutoLoad({
-          model_path: "unsloth/gemma-4-E2B-it-GGUF",
+          model_path: "unsloth/Qwen3.5-4B-MTP-GGUF",
           max_seq_length: 0,
           is_lora: false,
           gguf_variant: "UD-Q4_K_XL",
@@ -1166,7 +1166,7 @@ async function autoLoadSmallestModel(): Promise<{
       }
       loadAttempts += 1;
       const loadResp = await loadModel({
-        model_path: "unsloth/gemma-4-E2B-it-GGUF",
+        model_path: "unsloth/Qwen3.5-4B-MTP-GGUF",
         hf_token: hfToken,
         max_seq_length: 0,
         load_in_4bit: true,
@@ -1176,7 +1176,7 @@ async function autoLoadSmallestModel(): Promise<{
       });
       useChatRuntimeStore
         .getState()
-        .setCheckpoint("unsloth/gemma-4-E2B-it-GGUF", "UD-Q4_K_XL");
+        .setCheckpoint("unsloth/Qwen3.5-4B-MTP-GGUF", "UD-Q4_K_XL");
       const store = useChatRuntimeStore.getState();
       store.setModelRequiresTrustRemoteCode(
         loadResp.requires_trust_remote_code ?? false,
@@ -1186,13 +1186,13 @@ async function autoLoadSmallestModel(): Promise<{
         maxTokens: loadResp.context_length ?? 131072,
       });
       const defaultModel: ChatModelSummary = {
-        id: "unsloth/gemma-4-E2B-it-GGUF",
-        name: loadResp.display_name ?? "gemma-4-E2B-it-GGUF",
+        id: "unsloth/Qwen3.5-4B-MTP-GGUF",
+        name: loadResp.display_name ?? "Qwen3.5-4B-MTP-GGUF",
         isVision: loadResp.is_vision ?? false,
         isLora: false,
         isGguf: true,
       };
-      if (!store.models.some((m) => m.id === "unsloth/gemma-4-E2B-it-GGUF")) {
+      if (!store.models.some((m) => m.id === "unsloth/Qwen3.5-4B-MTP-GGUF")) {
         store.setModels([...store.models, defaultModel]);
       }
       useChatRuntimeStore.setState({
@@ -1212,7 +1212,7 @@ async function autoLoadSmallestModel(): Promise<{
         chatTemplateOverride: null,
         loadedIsMultimodal: isMultimodalResponse(loadResp),
       });
-      toast.success("Loaded Gemma-4-E2B-it (UD-Q4_K_XL)", { id: toastId });
+      toast.success("Loaded Qwen3.5-4B-MTP (UD-Q4_K_XL)", { id: toastId });
       return { loaded: true, blockedByTrustRemoteCode: false };
     } catch {
       toast.dismiss(toastId);