Studio: trim redundant comments (comment-only, AST-verified)

This commit is contained in:
danielhanchen 2026-07-02 07:25:40 +00:00
commit ade6613e99
3 changed files with 19 additions and 32 deletions

View file

@ -26,8 +26,7 @@ import re
from typing import Any
# Markers that flip the streaming buffer from STREAMING to DRAINING so
# partial markup never leaks before the parser sees it.
# Flip the stream buffer to DRAINING so partial markup never leaks.
TOOL_XML_SIGNALS = (
"<tool_call>",
"<function=",
@ -37,8 +36,7 @@ TOOL_XML_SIGNALS = (
)
# Closed pairs only (mid-stream); _TOOL_ALL_PATS also eats unclosed
# tails for end-of-turn cleanup.
# Closed pairs only; _TOOL_ALL_PATS also eats unclosed tails at end-of-turn.
_TOOL_CLOSED_PATS = [
re.compile(r"<tool_call>.*?</tool_call>", re.DOTALL),
re.compile(r"<function=\w+>.*?</function>", re.DOTALL),
@ -102,11 +100,8 @@ _LLAMA3_KV_RE = re.compile(
re.VERBOSE,
)
# Mistral ``[TOOL_CALLS]`` trigger. V11+ Tekken chains them, each
# followed by a bare name plus ``{json}`` (Magistral) or
# ``[ARGS]{json}`` (V13 Tekken: Devstral, Magistral-Small-2509).
# V3 tokenizer models (v0.3, Nemo, Small, Ministral-8B-2410,
# Large-2411) emit the ``[TOOL_CALLS] [...]`` array form instead.
# Mistral ``[TOOL_CALLS]``: V11+ Tekken chains ``name{json}`` / ``name[ARGS]{json}``
# (Magistral, Devstral); V3 tokenizer models emit the ``[...]`` array form.
_MISTRAL_TRIGGER = "[TOOL_CALLS]"
_MISTRAL_ARGS_MARKER = "[ARGS]"
_MISTRAL_V11_NAME_RE = re.compile(r"\s*([\w\.\-]+)\s*")

View file

@ -260,11 +260,9 @@ def _detect_safetensors_features(backend, chat_template: Optional[str]) -> dict:
"supports_tools": False,
}
)
# Markers the safetensors / MLX parser recognises. If the template
# advertises tools but uses none of them, drop the pill (parser
# can't honour the emission). The two ``{"name":`` variants cover
# Llama-3.2 ``custom_tools`` whose template prompts the bare-JSON
# form without a ``<|python_tag|>`` prefix.
# Markers the safetensors / MLX parser recognises; drop the pill if the
# template advertises tools but emits none. The ``{"name":`` variants cover
# Llama-3.2 ``custom_tools`` bare-JSON (no ``<|python_tag|>`` prefix).
_PARSER_MARKERS = (
"<tool_call>",
"<function=",
@ -435,12 +433,10 @@ _TOOL_ACTION_NUDGE = (
" Do NOT output code blocks -- use the python tool instead."
)
# Strip leaked tool-call markup. Covers every shared-parser format AND
# the four leak shapes the speculative buffer in ``llama_cpp.py`` splits
# across the visible/DRAIN boundary (closed pair, orphan open to EOF,
# bare orphan close, tail-only ``</parameter>``). Mistral ``[TOOL_CALLS]``
# is delegated to the parser's balanced-brace helper -- a non-greedy
# ``\{.*?\}`` here would truncate nested JSON at the first ``}``.
# Strip leaked tool-call markup: every shared-parser format plus the four
# leak shapes the llama_cpp.py buffer splits across the visible/DRAIN boundary.
# Mistral ``[TOOL_CALLS]`` goes via the parser's balanced-brace helper (a
# non-greedy ``\{.*?\}`` would truncate nested JSON at the first ``}``).
_TOOL_XML_RE = _re.compile(
"|".join(
[
@ -450,10 +446,8 @@ _TOOL_XML_RE = _re.compile(
r"</(?:tool_call|function)>",
# Gemma 4.
r"<\|tool_call>.*?<tool_call\|>",
# Llama-3 ``<|python_tag|>...`` to the next ``<|`` sentinel
# or EOF. ``(?:[^<]|<(?!\|))*`` (not ``[^\n<]*`` or
# ``[^\n]*``) keeps literal ``<``, newlines, and embedded
# JSON inside the strip.
# Llama-3 ``<|python_tag|>...`` to the next ``<|`` sentinel or EOF;
# ``(?:[^<]|<(?!\|))*`` keeps literal ``<``, newlines, embedded JSON.
r"<\|python_tag\|>(?:[^<]|<(?!\|))*",
# Tail-only ``</parameter>`` (anchored so mid-text survives).
r"</parameter>\s*\Z",

View file

@ -139,7 +139,7 @@ class TestParserMultiFormat:
agentic loop is family-agnostic.
"""
# ── Llama-3 ────────────────────────────────────────────────────
# Llama-3
def test_llama3_python_tag_dot_call(self):
# Llama-3 built-in tools: <|python_tag|>NAME.call(k="v", ...).
@ -193,7 +193,7 @@ class TestParserMultiFormat:
text = '<|python_tag|>brave_search.call(query="x")'
assert strip_tool_markup(text, final = True) == ""
# ── Llama-3.2 bare JSON ``custom_tools`` ─────────────────────
# Llama-3.2 bare JSON ``custom_tools``
def test_llama3_2_bare_json_parameters(self):
# Llama-3.2-Instruct emits bare JSON directly as content; no
@ -291,7 +291,7 @@ class TestParserMultiFormat:
):
assert parse_tool_calls_from_text(bad) == [], bad
# ── Mistral pre-v11 ───────────────────────────────────────────
# Mistral pre-v11
def test_mistral_pre_v11_array(self):
import json
@ -324,7 +324,7 @@ class TestParserMultiFormat:
assert len(result) == 1
assert result[0]["function"]["name"] == "web_search"
# ── Mistral v11+ ───────────────────────────────────────────────
# Mistral v11+
def test_mistral_v11_single(self):
# Magistral / Mistral Small 3.1: bare ``name{json}`` after trigger.
@ -358,7 +358,7 @@ class TestParserMultiFormat:
text = '[TOOL_CALLS]add{"a":1}'
assert strip_tool_markup(text, final = True) == ""
# ── Gemma 4 ───────────────────────────────────────────────────
# Gemma 4
def test_gemma4_simple_call(self):
import json
@ -423,7 +423,7 @@ class TestParserMultiFormat:
text = "<|tool_call>call:foo{x:1}<tool_call|>"
assert strip_tool_markup(text, final = True) == ""
# ── Cross-format sentinels ────────────────────────────────────
# Cross-format sentinels
def test_all_markers_in_tool_xml_signals(self):
# Streaming buffer wakes up on every emission marker.
@ -1166,9 +1166,7 @@ class TestGptOssNameDetection:
assert is_gpt_oss_model_name(None) is False
# ────────────────────────────────────────────────────────────────────
# Routes-level python_tag strip (multi-line; stop on next sentinel)
# ────────────────────────────────────────────────────────────────────
class TestRoutesPythonTagStrip: