Studio: trim redundant comments (comment-only, AST-verified)
This commit is contained in:
parent
615b860803
commit
ade6613e99
3 changed files with 19 additions and 32 deletions
|
|
@ -26,8 +26,7 @@ import re
|
|||
from typing import Any
|
||||
|
||||
|
||||
# Markers that flip the streaming buffer from STREAMING to DRAINING so
|
||||
# partial markup never leaks before the parser sees it.
|
||||
# Flip the stream buffer to DRAINING so partial markup never leaks.
|
||||
TOOL_XML_SIGNALS = (
|
||||
"<tool_call>",
|
||||
"<function=",
|
||||
|
|
@ -37,8 +36,7 @@ TOOL_XML_SIGNALS = (
|
|||
)
|
||||
|
||||
|
||||
# Closed pairs only (mid-stream); _TOOL_ALL_PATS also eats unclosed
|
||||
# tails for end-of-turn cleanup.
|
||||
# Closed pairs only; _TOOL_ALL_PATS also eats unclosed tails at end-of-turn.
|
||||
_TOOL_CLOSED_PATS = [
|
||||
re.compile(r"<tool_call>.*?</tool_call>", re.DOTALL),
|
||||
re.compile(r"<function=\w+>.*?</function>", re.DOTALL),
|
||||
|
|
@ -102,11 +100,8 @@ _LLAMA3_KV_RE = re.compile(
|
|||
re.VERBOSE,
|
||||
)
|
||||
|
||||
# Mistral ``[TOOL_CALLS]`` trigger. V11+ Tekken chains them, each
|
||||
# followed by a bare name plus ``{json}`` (Magistral) or
|
||||
# ``[ARGS]{json}`` (V13 Tekken: Devstral, Magistral-Small-2509).
|
||||
# V3 tokenizer models (v0.3, Nemo, Small, Ministral-8B-2410,
|
||||
# Large-2411) emit the ``[TOOL_CALLS] [...]`` array form instead.
|
||||
# Mistral ``[TOOL_CALLS]``: V11+ Tekken chains ``name{json}`` / ``name[ARGS]{json}``
|
||||
# (Magistral, Devstral); V3 tokenizer models emit the ``[...]`` array form.
|
||||
_MISTRAL_TRIGGER = "[TOOL_CALLS]"
|
||||
_MISTRAL_ARGS_MARKER = "[ARGS]"
|
||||
_MISTRAL_V11_NAME_RE = re.compile(r"\s*([\w\.\-]+)\s*")
|
||||
|
|
|
|||
|
|
@ -260,11 +260,9 @@ def _detect_safetensors_features(backend, chat_template: Optional[str]) -> dict:
|
|||
"supports_tools": False,
|
||||
}
|
||||
)
|
||||
# Markers the safetensors / MLX parser recognises. If the template
|
||||
# advertises tools but uses none of them, drop the pill (parser
|
||||
# can't honour the emission). The two ``{"name":`` variants cover
|
||||
# Llama-3.2 ``custom_tools`` whose template prompts the bare-JSON
|
||||
# form without a ``<|python_tag|>`` prefix.
|
||||
# Markers the safetensors / MLX parser recognises; drop the pill if the
|
||||
# template advertises tools but emits none. The ``{"name":`` variants cover
|
||||
# Llama-3.2 ``custom_tools`` bare-JSON (no ``<|python_tag|>`` prefix).
|
||||
_PARSER_MARKERS = (
|
||||
"<tool_call>",
|
||||
"<function=",
|
||||
|
|
@ -435,12 +433,10 @@ _TOOL_ACTION_NUDGE = (
|
|||
" Do NOT output code blocks -- use the python tool instead."
|
||||
)
|
||||
|
||||
# Strip leaked tool-call markup. Covers every shared-parser format AND
|
||||
# the four leak shapes the speculative buffer in ``llama_cpp.py`` splits
|
||||
# across the visible/DRAIN boundary (closed pair, orphan open to EOF,
|
||||
# bare orphan close, tail-only ``</parameter>``). Mistral ``[TOOL_CALLS]``
|
||||
# is delegated to the parser's balanced-brace helper -- a non-greedy
|
||||
# ``\{.*?\}`` here would truncate nested JSON at the first ``}``.
|
||||
# Strip leaked tool-call markup: every shared-parser format plus the four
|
||||
# leak shapes the llama_cpp.py buffer splits across the visible/DRAIN boundary.
|
||||
# Mistral ``[TOOL_CALLS]`` goes via the parser's balanced-brace helper (a
|
||||
# non-greedy ``\{.*?\}`` would truncate nested JSON at the first ``}``).
|
||||
_TOOL_XML_RE = _re.compile(
|
||||
"|".join(
|
||||
[
|
||||
|
|
@ -450,10 +446,8 @@ _TOOL_XML_RE = _re.compile(
|
|||
r"</(?:tool_call|function)>",
|
||||
# Gemma 4.
|
||||
r"<\|tool_call>.*?<tool_call\|>",
|
||||
# Llama-3 ``<|python_tag|>...`` to the next ``<|`` sentinel
|
||||
# or EOF. ``(?:[^<]|<(?!\|))*`` (not ``[^\n<]*`` or
|
||||
# ``[^\n]*``) keeps literal ``<``, newlines, and embedded
|
||||
# JSON inside the strip.
|
||||
# Llama-3 ``<|python_tag|>...`` to the next ``<|`` sentinel or EOF;
|
||||
# ``(?:[^<]|<(?!\|))*`` keeps literal ``<``, newlines, embedded JSON.
|
||||
r"<\|python_tag\|>(?:[^<]|<(?!\|))*",
|
||||
# Tail-only ``</parameter>`` (anchored so mid-text survives).
|
||||
r"</parameter>\s*\Z",
|
||||
|
|
|
|||
|
|
@ -139,7 +139,7 @@ class TestParserMultiFormat:
|
|||
agentic loop is family-agnostic.
|
||||
"""
|
||||
|
||||
# ── Llama-3 ────────────────────────────────────────────────────
|
||||
# Llama-3
|
||||
|
||||
def test_llama3_python_tag_dot_call(self):
|
||||
# Llama-3 built-in tools: <|python_tag|>NAME.call(k="v", ...).
|
||||
|
|
@ -193,7 +193,7 @@ class TestParserMultiFormat:
|
|||
text = '<|python_tag|>brave_search.call(query="x")'
|
||||
assert strip_tool_markup(text, final = True) == ""
|
||||
|
||||
# ── Llama-3.2 bare JSON ``custom_tools`` ─────────────────────
|
||||
# Llama-3.2 bare JSON ``custom_tools``
|
||||
|
||||
def test_llama3_2_bare_json_parameters(self):
|
||||
# Llama-3.2-Instruct emits bare JSON directly as content; no
|
||||
|
|
@ -291,7 +291,7 @@ class TestParserMultiFormat:
|
|||
):
|
||||
assert parse_tool_calls_from_text(bad) == [], bad
|
||||
|
||||
# ── Mistral pre-v11 ───────────────────────────────────────────
|
||||
# Mistral pre-v11
|
||||
|
||||
def test_mistral_pre_v11_array(self):
|
||||
import json
|
||||
|
|
@ -324,7 +324,7 @@ class TestParserMultiFormat:
|
|||
assert len(result) == 1
|
||||
assert result[0]["function"]["name"] == "web_search"
|
||||
|
||||
# ── Mistral v11+ ───────────────────────────────────────────────
|
||||
# Mistral v11+
|
||||
|
||||
def test_mistral_v11_single(self):
|
||||
# Magistral / Mistral Small 3.1: bare ``name{json}`` after trigger.
|
||||
|
|
@ -358,7 +358,7 @@ class TestParserMultiFormat:
|
|||
text = '[TOOL_CALLS]add{"a":1}'
|
||||
assert strip_tool_markup(text, final = True) == ""
|
||||
|
||||
# ── Gemma 4 ───────────────────────────────────────────────────
|
||||
# Gemma 4
|
||||
|
||||
def test_gemma4_simple_call(self):
|
||||
import json
|
||||
|
|
@ -423,7 +423,7 @@ class TestParserMultiFormat:
|
|||
text = "<|tool_call>call:foo{x:1}<tool_call|>"
|
||||
assert strip_tool_markup(text, final = True) == ""
|
||||
|
||||
# ── Cross-format sentinels ────────────────────────────────────
|
||||
# Cross-format sentinels
|
||||
|
||||
def test_all_markers_in_tool_xml_signals(self):
|
||||
# Streaming buffer wakes up on every emission marker.
|
||||
|
|
@ -1166,9 +1166,7 @@ class TestGptOssNameDetection:
|
|||
assert is_gpt_oss_model_name(None) is False
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────────
|
||||
# Routes-level python_tag strip (multi-line; stop on next sentinel)
|
||||
# ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestRoutesPythonTagStrip:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue