Studio: r21 fixes - strip orphan tool-call XML before artifact, broaden first-person plan verbs
- Re-prompt path calls _strip_tool_markup(final=True) on content_accum
before measuring intent / artifact / length. An orphan
``<tool_call>...</tool_call>`` block containing a code fence no
longer hides the intent-only visible answer from the artifact check.
- _DIRECT_NUMBERED_PLAN_FRAMING splits into two branches:
* First-person intent ("I'll", "Let me", "I will", etc.) accepts a
broader work-verb set (open, read, search, check, review, inspect,
examine, etc.). Direct first-person announcements are strong
plan-like signals.
* Bare "First, ..." / "Step N: ..." keeps the narrow verb set so
algorithmic answers ("First, use binary search:") stay valid.
Catches stalls like "I will check the docs:\n1. Gather..." and
"Let me read the uploaded file:\n1. Identify the columns..." that
previously slipped past the freshness-gated lookup verbs.
This commit is contained in:
parent
64a400be2d
commit
834b34c68d
2 changed files with 62 additions and 7 deletions
|
|
@ -185,20 +185,31 @@ _EXPLICIT_PLAN_HEADER = re.compile(
|
|||
# to perform + a numbered list. Catches stalls like
|
||||
# ``First, I'll do this:\n1. Search ...`` or ``Let me do this:\n1. Parse
|
||||
# the file ...`` where each list item is an action the model promised
|
||||
# to take without actually invoking a tool. The follow-up verb list
|
||||
# stays narrow to "do/proceed/build/run" style words so common
|
||||
# answer prose like "Let me explain", "Let me show", "Let me draft a
|
||||
# poem" is not misclassified as a stall.
|
||||
# to take without actually invoking a tool. The first-person intent
|
||||
# branch tolerates a broad set of work verbs (open/read/search/check/
|
||||
# review/inspect/etc.) because direct first-person announcements are
|
||||
# strongly plan-like; the "First, ..." / "Step N:" branch stays
|
||||
# narrow so algorithmic answers ("First, use binary search:") are
|
||||
# preserved.
|
||||
_DIRECT_NUMBERED_PLAN_FRAMING = re.compile(
|
||||
r"(?:"
|
||||
r"\b(?:i['’](?:ll|m going to|m gonna)|i am (?:going to|gonna)|"
|
||||
r"i will|i shall|let me|allow me|now i|next i|"
|
||||
r"first|step \d+:?)\b"
|
||||
r"i will|i shall|let me|allow me|now i|next i)\b"
|
||||
r"[^\r\n]{0,160}"
|
||||
r"\b(?:open|read|search|look (?:this |that |it |them )?up|browse|"
|
||||
r"google|find|check|verify|compare|review|inspect|examine|"
|
||||
r"do (?:this|these|the following|it)|proceed|start|begin|"
|
||||
r"create|build|implement|set up|add|calculate|compute|analy[sz]e|"
|
||||
r"parse|load|run|execute|test)\b"
|
||||
r"|"
|
||||
r"\b(?:first|step \d+:?)\b"
|
||||
r"[^\r\n]{0,160}"
|
||||
r"\b(?:do (?:this|these|the following|it)|"
|
||||
r"look (?:this |that |it |them )?up|"
|
||||
r"proceed|start|begin|"
|
||||
r"create|build|implement|set up|add|"
|
||||
r"calculate|compute|analy[sz]e|parse|load|run|execute|test)\b"
|
||||
r")"
|
||||
r"[\s\S]{0,500}?"
|
||||
r"(?:^|\r?\n)[ \t]*\d+\.",
|
||||
re.IGNORECASE,
|
||||
|
|
@ -5136,7 +5147,17 @@ class LlamaCppBackend:
|
|||
# case); otherwise reasoning stays hidden and
|
||||
# an artifact inside it must NOT suppress the
|
||||
# re-prompt.
|
||||
_visible = content_accum.strip()
|
||||
# Strip orphan tool-call XML before measuring
|
||||
# the visible answer. An ``<tool_call>...</tool_call>``
|
||||
# block that the route layer would scrub from the
|
||||
# final visible message must not satisfy the
|
||||
# artifact check.
|
||||
_visible_raw = content_accum.strip()
|
||||
_visible = (
|
||||
_strip_tool_markup(content_accum, final=True).strip()
|
||||
if _visible_raw
|
||||
else ""
|
||||
)
|
||||
_reasoning = reasoning_accum.strip()
|
||||
_stripped = _visible if _visible else _reasoning
|
||||
# Cheap gates first so long final answers never
|
||||
|
|
|
|||
|
|
@ -985,6 +985,40 @@ def test_no_reprompt_on_code_fence_containing_markup_literal():
|
|||
assert not _would_reprompt(content), content
|
||||
|
||||
|
||||
def test_reprompts_on_direct_first_person_read_check_open_plan():
|
||||
"""Direct first-person intent + open/read/check/review/inspect verbs
|
||||
+ numbered list is a tool stall. The broader verb set applies to
|
||||
first-person intent only; bare ``First, ...`` and ``Step N: ...``
|
||||
keep their narrower verb whitelist."""
|
||||
samples = [
|
||||
(
|
||||
"Let me read the uploaded file:\n"
|
||||
"1. Identify the columns.\n"
|
||||
"2. Return the total."
|
||||
),
|
||||
(
|
||||
"I will check the docs:\n"
|
||||
"1. Gather relevant sections.\n"
|
||||
"2. Answer."
|
||||
),
|
||||
(
|
||||
"First, I'll review the repository:\n"
|
||||
"1. Open the relevant file.\n"
|
||||
"2. Read the implementation.\n"
|
||||
"3. Suggest a fix."
|
||||
),
|
||||
(
|
||||
"Let me examine the log file:\n"
|
||||
"1. Open the log.\n"
|
||||
"2. Read the errors.\n"
|
||||
"3. Summarize."
|
||||
),
|
||||
]
|
||||
for content in samples:
|
||||
assert not _has_answer_artifact(content), content
|
||||
assert _would_reprompt(content), content
|
||||
|
||||
|
||||
def test_no_reprompt_on_html_with_inner_svg_or_self_closing_tag():
|
||||
"""Complete <html> answers that contain nested SVG / self-closing
|
||||
tags are still complete pages. The unbalanced-count cross-check is
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue