4c06c1dcc7* Studio: enable audio input for Gemma 4 GGUF models Audio file upload was disabled for Gemma 4 vision+audio GGUFs (e.g. gemma-4-12b-it-GGUF) even though their mmproj carries an audio encoder (clip.has_audio_encoder, gemma4ua). Two causes: - Audio-input detection only matched Gemma 3n's <audio_soft_token>; Gemma 4 uses <|audio|>, so audio_vlm was never detected. - The GGUF load/status responses hardcoded has_audio_input=False, so the flag was dropped even when audio_vlm was detected (affected Gemma 3n GGUFs too). Changes: - Recognize <|audio|> alongside <audio_soft_token> in the llama-server token probe and the tokenizer-config pattern. - Read clip.has_audio_encoder from the mmproj as an independent, model-agnostic signal (read_mmproj_audio_capability). - Emit the computed has_audio_input on the GGUF load/status responses. - Tests for the new pattern and the mmproj reader. * Studio: default chat model and dataset helper to Qwen3.5-4B-MTP Switch the auto-loaded chat default and the dataset-analysis helper GGUF from gemma-4-E2B-it to unsloth/Qwen3.5-4B-MTP-GGUF (UD-Q4_K_XL). * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
46 lines
1.6 KiB
Python
46 lines
1.6 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-only
|
|
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
|
|
|
"""Tests for tokenizer-based audio_type detection patterns, covering both
|
|
Gemma 3n (<audio_soft_token>) and Gemma 4 (<|audio|>) audio-input tokens."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from utils.models.model_config import _AUDIO_TOKEN_PATTERNS, is_audio_input_type
|
|
|
|
|
|
def _classify(tokens: list[str]) -> str | None:
|
|
"""Mirror _detect_audio_from_tokenizer._check_token_patterns: first match
|
|
in dict order wins."""
|
|
for audio_type, check in _AUDIO_TOKEN_PATTERNS.items():
|
|
if check(tokens):
|
|
return audio_type
|
|
return None
|
|
|
|
|
|
def test_gemma3n_audio_soft_token_is_audio_vlm():
|
|
assert (
|
|
_classify(["<bos>", "<audio_soft_token>", "<image_soft_token>"]) == "audio_vlm"
|
|
)
|
|
|
|
|
|
def test_gemma4_pipe_audio_token_is_audio_vlm():
|
|
# Gemma 4 uses <|audio|> (and <|image|>) instead of *_soft_token.
|
|
assert _classify(["<bos>", "<|image|>", "<|audio|>"]) == "audio_vlm"
|
|
|
|
|
|
def test_csm_uppercase_audio_not_classified_as_audio_vlm():
|
|
# csm uses uppercase <|AUDIO|> + <|audio_eos|>; must stay csm, not audio_vlm.
|
|
tokens = ["<|AUDIO|>", "<|audio_eos|>"]
|
|
assert _classify(tokens) == "csm"
|
|
|
|
|
|
def test_audio_vlm_and_whisper_accept_audio_input():
|
|
assert is_audio_input_type("audio_vlm") is True
|
|
assert is_audio_input_type("whisper") is True
|
|
assert is_audio_input_type("snac") is False
|
|
assert is_audio_input_type(None) is False
|
|
|
|
|
|
def test_non_audio_tokens_classify_none():
|
|
assert _classify(["<bos>", "<eos>", "<pad>"]) is None
|