diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 02270ab405..bf92055929 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -427,9 +427,17 @@ _TOOL_ACTION_NUDGE = ( " Do NOT output code blocks -- use the python tool instead." ) -# Regex for stripping leaked tool-call XML from assistant messages/stream +# Strip tool-call XML the speculative buffer in core/inference/llama_cpp.py +# split across the visible/DRAIN boundary. Four leak shapes: +# 1. well-formed `...` / `...` +# 2. orphan opening to EOF (close was DRAINED) +# 3. bare orphan close (open was DRAINED) +# 4. tail-only `` (outer close truncated by EOS); anchored to +# `\Z` so mid-text `` in user code samples survives. _TOOL_XML_RE = _re.compile( - r".*?|.*?", + r"<(?:tool_call|function=\w+)>.*?(?:|\Z)" + r"|" + r"|\s*\Z", _re.DOTALL, ) logger = get_logger(__name__) diff --git a/studio/backend/tests/test_tool_xml_strip.py b/studio/backend/tests/test_tool_xml_strip.py new file mode 100644 index 0000000000..8b90a46d5a --- /dev/null +++ b/studio/backend/tests/test_tool_xml_strip.py @@ -0,0 +1,263 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Tests for `_TOOL_XML_RE` (routes/inference.py) -- strips tool-call +XML that leaks past the speculative buffer in core/inference/llama_cpp.py +when the open/close pair is split across the visible/DRAIN boundary. +""" + +from __future__ import annotations + +import sys +import types as _types +from pathlib import Path + +import pytest + +_BACKEND_DIR = str(Path(__file__).resolve().parent.parent) +if _BACKEND_DIR not in sys.path: + sys.path.insert(0, _BACKEND_DIR) + +# Extract the regex from source (routes module needs heavy stubbing to import). +import re as _re + +_src = (Path(_BACKEND_DIR) / "routes" / "inference.py").read_text() +_m = _re.search(r"_TOOL_XML_RE = _re\.compile\((.*?)\n\)", _src, _re.DOTALL) +assert _m, "could not extract _TOOL_XML_RE source" +_ns = {"_re": _re} +exec(f"_TOOL_XML_RE = _re.compile({_m.group(1)})", _ns) +_TOOL_XML_RE = _ns["_TOOL_XML_RE"] + + +# ── Well-formed pairs ───────────────────────────────────────────── + + +def test_strips_well_formed_tool_call(): + text = ( + "Let me search.\n" + "\n" + "\n" + "\nBillboard 2015\n\n" + "\n" + "\n" + "Here are the songs:" + ) + cleaned = _TOOL_XML_RE.sub("", text) + assert "" not in cleaned + assert "" not in cleaned + assert "" not in cleaned + assert "Here are the songs:" in cleaned, "non-XML content must survive" + assert "Let me search." in cleaned + + +def test_strips_function_only_well_formed(): + text = "Setup.\n\n\nprint(1)\n\n\nDone." + cleaned = _TOOL_XML_RE.sub("", text) + assert "" + "\n" + "\n" + "\nBillboard 2015\n\n" + "" not in cleaned + assert "\n\nprint(1)\n" + ) + cleaned = _TOOL_XML_RE.sub("", text) + assert "") + assert "" not in cleaned + assert "Search starting." in cleaned + + +def test_strips_multiple_orphans(): + text = ( + "First call:\n\n\n\nx=1\n" + "Second call:\n\n\nhi\n" + ) + cleaned = _TOOL_XML_RE.sub("", text) + assert "" not in cleaned + assert "" not in cleaned + assert "" not in cleaned + # Mid-string intentionally preserved (see preserve test). + + +# ── Tail-only (PR #5735 follow-up) ─────────────────── + + +def test_strips_tail_only_parameter_orphan(): + # Outer truncated by EOS, inner DRAINED. + cleaned = _TOOL_XML_RE.sub("", "and the text is not readable.\n\n\n") + assert "" not in cleaned + assert "and the text is not readable." in cleaned + + +def test_strips_tail_only_parameter_orphan_single_newline(): + cleaned = _TOOL_XML_RE.sub("", "Global Economic Prospects\n\n") + assert "" not in cleaned + assert "Global Economic Prospects" in cleaned + + +def test_strips_tail_only_parameter_orphan_no_trailing_ws(): + cleaned = _TOOL_XML_RE.sub("", "Final answer.") + assert "" not in cleaned + assert "Final answer." in cleaned + + +def test_preserves_mid_string_parameter_in_code_sample(): + # Tail-anchor on `` is required so doc/example prose survives. + text = ( + "Here is the Qwen tool-call format:\n" + "```xml\n" + "value\n" + "```\n" + "Note the closing sits inside ." + ) + cleaned = _TOOL_XML_RE.sub("", text) + assert "Note the closing sits inside" in cleaned + + +def test_strips_well_formed_then_orphan(): + text = ( + "Round one:\n\n\n\n1\n" + "\n\n\n" + "Now round two:\n\n\n\n" + "what is X\n\n" not in cleaned + assert "\n\n\n"Billboard Hot 100" "2015" "weekly" "chart" "position" "3"\n\n\n\n\n"peaked at number 3" Billboard Hot 100 2015 list\n\n\n\n\n"List of Billboard Hot 100 top-ten singles in 2015" wikipedia\n\n\n\nThe user wants me to list and categorize all songs that charted #3 on the Billboard Hot 100 in 2015. I have been trying to get this data", + # Qwen3.6-35B-A3B Q8_0 billboard s21 -- orphan close + "parse it more carefully.\n\n\nThe user wants a list of songs that charted #3 on the Billboard Hot 100 in 2015, categorized.", +] + + +@pytest.mark.parametrize( + "leak", REAL_LEAKS, ids = [f"sweep_sample_{i}" for i in range(len(REAL_LEAKS))] +) +def test_real_world_sweep_leaks_get_stripped(leak): + cleaned = _TOOL_XML_RE.sub("", leak) + assert "" not in cleaned, f"leak survived: {cleaned!r}" + assert " from gdpval sweep ────────── + + +# All end-anchored: outer truncated by EOS, +# inner open DRAINED, leaving bare tail. +GDPVAL_PARAMETER_LEAKS = [ + # Qwen3.5-27B Q8_0 / worldbank s00 + "the page contains image data and the text is not readable.\n\n\n", + # Qwen3.5-27B Q8_0 / worldbank s42 (preceded by mojibake) + "...some mojibake content here...\n\n\n", + # Qwen3.5-27B UD-Q4_K_XL / coppa s07 + "blocked, while others may still be in effect. The law is currently under further review by the Ninth Circuit.\n\n\n", + # Qwen3.5-27B UD-Q4_K_XL / police_training s00 + "comprehensive training report\n\n\n", + # Qwen3.5-27B UD-Q4_K_XL / worldbank s00 + "Global Economic Prospects\nJune 2025\nGlobal Economic Prospects\n\n", + # Qwen3.6-27B Q8_0 / overpass s07 + "Let me create a comprehensive query and instructions document.\n\n\n", +] + + +@pytest.mark.parametrize( + "leak", + GDPVAL_PARAMETER_LEAKS, + ids = [f"gdpval_param_orphan_{i}" for i in range(len(GDPVAL_PARAMETER_LEAKS))], +) +def test_gdpval_parameter_orphans_get_stripped(leak): + cleaned = _TOOL_XML_RE.sub("", leak) + assert "" not in cleaned, f"leak survived: {cleaned!r}" + + +# ── Backtracking guards ────────────────────────────────────────── + + +def test_no_catastrophic_backtracking_on_open_bracket_spam(): + # 256KB of '<' must fail fast (literal mismatch char 2), not backtrack. + import time + + adv = "<" * (1024 * 256) + "X" + t0 = time.perf_counter() + _TOOL_XML_RE.sub("", adv) + elapsed = time.perf_counter() - t0 + assert elapsed < 0.5, f"regex took {elapsed*1000:.0f}ms on 256KB '<' spam" + + +def test_no_catastrophic_backtracking_on_orphan_opening_spam(): + # 1000 unclosed openings: first alt must consume them all greedily. + import time + + adv = "X" * 1000 + t0 = time.perf_counter() + cleaned = _TOOL_XML_RE.sub("", adv) + elapsed = time.perf_counter() - t0 + assert elapsed < 0.1, f"regex took {elapsed*1000:.0f}ms on 1000x orphan opens" + assert "" not in cleaned