From fe379212232b9649cdf96cafa226e6fa54626851 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 5 Jun 2026 07:15:45 -0700 Subject: [PATCH] Studio: fix load_freeze audio-type tests for #6000's Gemma 4 `<|audio|>` probe (#6018) * Studio: fix load_freeze audio-type tests for #6000 Gemma 4 <|audio|> probe #6000 extended LlamaCppBackend._detect_audio_type_strict audio_vlm arm to also probe Gemma 4 `<|audio|>` (alongside Gemma 3n ``), but did not update the load_freeze simulation suite (last touched by #5922). Its "no-match" and "bicodec" fixtures only defeat ``; the unmapped `<|audio|>` probe falls through to FakeLlamaServer 1-token default, so detect_audio_type now returns audio_vlm where these tests expect None / bicodec: - test_functional_equivalence_no_match - test_functional_equivalence_bicodec_match - test_response_shape_matches_pre_fix_for_no_match main push-CI does not run "Repo tests (CPU)" (pull_request-only), so this surfaces in every open PR merge-ref (e.g. #5940, which is unrelated to audio). Fix: map `<|audio|>` to a 2-token response in the three fixtures that intend a non-audio_vlm result (restoring their original semantics), and add a positive test_functional_equivalence_audio_vlm_match locking in #6000 new `<|audio|>` detection. Co-Authored-By: Claude Opus 4.8 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: Claude Opus 4.8 Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../load_freeze/test_load_orchestrator.py | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/studio/load_freeze/test_load_orchestrator.py b/tests/studio/load_freeze/test_load_orchestrator.py index 8d32932d13..91b70f0b03 100644 --- a/tests/studio/load_freeze/test_load_orchestrator.py +++ b/tests/studio/load_freeze/test_load_orchestrator.py @@ -3,7 +3,7 @@ Covers: 1. Behavioural canary (the bug class) — 2 tests 2. Behavioural fix-validation — 1 test - 3. Functional equivalence (sync == to_thread) — 5 tests, one per codec branch + 3. Functional equivalence (sync == to_thread) — 6 tests, one per codec branch 4. Failure modes (HTTP 500, malformed JSON, connection reset, unreachable, not-loaded) — 5 tests 5. Stress (50 concurrent probes / 100 healths) — 2 tests @@ -254,6 +254,7 @@ def shim_no_match(): "<|audio_eos|>": [0, 1], "<|startoftranscript|>": [0, 1], "": [0, 1], + "<|audio|>": [0, 1], "<|bicodec_semantic_0|>": [0, 1], "<|bicodec_global_0|>": [0, 1], "<|c1_0|>": [0, 1], @@ -314,6 +315,28 @@ def test_functional_equivalence_whisper_match(): assert sync_result == threaded +def test_functional_equivalence_audio_vlm_match(): + # audio_vlm: snac/csm/whisper fail first, then the Gemma 4 <|audio|> + # probe tokenises to a single token. #6000 added this arm alongside + # Gemma 3n's ; keep at 2 tokens so + # it is specifically the new <|audio|> arm that triggers the match. + with FakeLlamaServer( + detok_map = {128258: "non-snac", 128259: "non-snac"}, + tok_response_map = { + "<|AUDIO|>": [0, 1], # csm fails (>1 token) + "<|audio_eos|>": [0, 1], + "<|startoftranscript|>": [0, 1], # whisper fails + "": [0, 1], # Gemma 3n arm fails ... + "<|audio|>": [0], # ... Gemma 4 arm matches (#6000) + }, + ) as srv: + backend = _make_backend(srv.port) + sync_result = backend.detect_audio_type() + threaded = asyncio.run(asyncio.to_thread(backend.detect_audio_type)) + assert sync_result == "audio_vlm" + assert sync_result == threaded + + def test_functional_equivalence_bicodec_match(): # bicodec: snac/csm/whisper/audio_vlm all fail first, then both # bicodec_semantic_0 and bicodec_global_0 are single tokens. @@ -324,6 +347,7 @@ def test_functional_equivalence_bicodec_match(): "<|audio_eos|>": [0, 1], "<|startoftranscript|>": [0, 1], "": [0, 1], + "<|audio|>": [0, 1], "<|bicodec_semantic_0|>": [0], "<|bicodec_global_0|>": [0], }, @@ -644,6 +668,7 @@ def test_response_shape_matches_pre_fix_for_no_match(): "<|audio_eos|>": [0, 1], "<|startoftranscript|>": [0, 1], "": [0, 1], + "<|audio|>": [0, 1], "<|bicodec_semantic_0|>": [0, 1], "<|bicodec_global_0|>": [0, 1], "<|c1_0|>": [0, 1],