Studio: enable audio input for Gemma 4 GGUFs; default chat model to Qwen3.5-4B-MTP (#6000)

* Studio: enable audio input for Gemma 4 GGUF models

Audio file upload was disabled for Gemma 4 vision+audio GGUFs (e.g.
gemma-4-12b-it-GGUF) even though their mmproj carries an audio encoder
(clip.has_audio_encoder, gemma4ua). Two causes:

- Audio-input detection only matched Gemma 3n's <audio_soft_token>;
  Gemma 4 uses <|audio|>, so audio_vlm was never detected.
- The GGUF load/status responses hardcoded has_audio_input=False, so the
  flag was dropped even when audio_vlm was detected (affected Gemma 3n
  GGUFs too).

Changes:
- Recognize <|audio|> alongside <audio_soft_token> in the llama-server
  token probe and the tokenizer-config pattern.
- Read clip.has_audio_encoder from the mmproj as an independent,
  model-agnostic signal (read_mmproj_audio_capability).
- Emit the computed has_audio_input on the GGUF load/status responses.
- Tests for the new pattern and the mmproj reader.

* Studio: default chat model and dataset helper to Qwen3.5-4B-MTP

Switch the auto-loaded chat default and the dataset-analysis helper GGUF
from gemma-4-E2B-it to unsloth/Qwen3.5-4B-MTP-GGUF (UD-Q4_K_XL).

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Daniel Han 2026-06-04 00:56:53 -07:00 committed by GitHub
commit 4c06c1dcc7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 240 additions and 15 deletions

View file

@ -697,6 +697,9 @@ class LlamaCppBackend:
self._is_audio: bool = False
self._audio_type: Optional[str] = None
self._audio_probed: bool = False
# Audio INPUT capability (distinct from _is_audio, which is TTS output).
self._has_audio_input: bool = False
self._mmproj_has_audio: bool = False # clip.has_audio_encoder, set at load
# Monotonic timestamp set in _kill_process; read by load_model
# to decide whether to wait for the VRAM reclaim to finish.
self._last_kill_monotonic: float = 0.0
@ -2782,6 +2785,12 @@ class LlamaCppBackend:
if not self._healthy:
return False
self._audio_type = detected
# Re-derive after a retried probe (_mmproj_has_audio persists).
from utils.models.model_config import is_audio_input_type
self._has_audio_input = bool(
is_audio_input_type(self._audio_type)
) or bool(self._mmproj_has_audio)
if not self._healthy:
return False
return True
@ -3095,6 +3104,21 @@ class LlamaCppBackend:
"image input will be disabled for this session"
)
# Audio input straight from the mmproj (clip.has_audio_encoder),
# independent of token names.
self._mmproj_has_audio = False
if launch_mmproj_path:
try:
from utils.models.gguf_metadata import (
read_mmproj_audio_capability,
)
self._mmproj_has_audio = bool(
read_mmproj_audio_capability(launch_mmproj_path)
)
except Exception as e:
logger.debug(f"mmproj audio-capability read failed: {e}")
cmd = [
binary,
"-m",
@ -3527,6 +3551,7 @@ class LlamaCppBackend:
self._is_audio = False
self._audio_type = None
self._audio_probed = False
self._has_audio_input = False
try:
detected = self._detect_audio_type_strict()
self._audio_probed = True
@ -3558,6 +3583,13 @@ class LlamaCppBackend:
return False
self._audio_type = detected
# Audio input = token probe (audio_vlm/whisper) OR mmproj audio encoder.
from utils.models.model_config import is_audio_input_type
self._has_audio_input = bool(is_audio_input_type(self._audio_type)) or bool(
self._mmproj_has_audio
)
if not self._healthy:
return False
return True
@ -3901,6 +3933,8 @@ class LlamaCppBackend:
self._is_audio = False
self._audio_type = None
self._audio_probed = False
self._has_audio_input = False
self._mmproj_has_audio = False
self._port = None
self._healthy = False
self._context_length = None
@ -5591,7 +5625,8 @@ class LlamaCppBackend:
return "csm"
if len(_tok("<|startoftranscript|>")) == 1:
return "whisper"
if len(_tok("<audio_soft_token>")) == 1:
# Gemma 3n: <audio_soft_token>; Gemma 4: <|audio|> (not csm's <|AUDIO|>).
if len(_tok("<audio_soft_token>")) == 1 or len(_tok("<|audio|>")) == 1:
return "audio_vlm"
if (
len(_tok("<|bicodec_semantic_0|>")) == 1