Studio: r13 fixes - freshness-gated lookup verbs, anchored CommonMark fences, open-fence cross-check
- _TOOL_ACTION_VERBS gates the lookup verbs (search / look up / browse / google / fetch / research / investigate / find / check / verify) on a freshness or web/internet/online target. Plain answer prose like \"binary search: 1. Search the left half\" or \"1. Find the bug\" stays a valid answer, while \"1. Search the web for X\" / \"1. Google the current chart\" / \"1. Research the latest docs\" still re-prompts. Strong unambiguous patterns (web search, query the web, call a tool, run python) remain bare. - _HAS_ANSWER_ARTIFACT anchors the fence opener and closer with (?<!\\`) / (?!\\`) lookarounds so a 4-backtick opener cannot backtrack to a 3-backtick fence and treat the surplus delimiter as info-string text. Same rule for tildes. - _has_answer_artifact now consults a small _has_unclosed_code_fence helper before the numbered-list fallback. A numbered list embedded INSIDE an open fence no longer masquerades as a final answer. - Existing plan-framing tests updated to use freshness-gated lookup phrasing so they continue to assert the intended invariants.
This commit is contained in:
Daniel Han
parent
693bd89a79
commit
e16f898c26
2 changed files with 148 additions and 37 deletions
|
|
@ -53,23 +53,27 @@ logger = get_logger(__name__)
|
|||
|
||||
# ── Pre-compiled patterns for plan-without-action re-prompt ──
|
||||
# Tool-action verbs used by _PLAN_LIST_FRAMING to distinguish plan-only
|
||||
# numbered lists ("1. search the docs", "1. fetch the data") from
|
||||
# answer numbered lists ("1. Apple", "1. Use BFS", "1. Write a poem").
|
||||
# Kept intentionally narrow: only verbs that strongly imply an actual
|
||||
# tool invocation. Broad verbs like ``use``, ``compare``, ``write``,
|
||||
# ``create``, ``make``, ``build``, ``think``, ``respond``, ``answer``,
|
||||
# ``analyse``, ``explore`` are deliberately excluded because real
|
||||
# answer lists use them ("1. Use BFS", "1. Compare versions",
|
||||
# "1. Write a poem"). ``find`` / ``check`` / ``verify`` are admitted
|
||||
# ONLY when paired with a freshness signal (current / latest /
|
||||
# today's / up-to-date / live / online / web), so "find the bug" or
|
||||
# "check the answer" still read as valid answer text.
|
||||
# numbered lists ("1. search the web for X", "1. query the internet")
|
||||
# from answer numbered lists ("1. Search the left half", "1. Apple",
|
||||
# "1. Write a poem"). Each lookup verb is gated on a freshness or
|
||||
# web/internet/online target so ordinary answer prose like
|
||||
# "binary search: 1. Search the left half" or "1. Find the bug" is
|
||||
# preserved. The strong, unambiguous patterns (``web search``,
|
||||
# ``query the web``, ``call a tool``, ``run python``) stay bare.
|
||||
_TOOL_LOOKUP_TARGET = (
|
||||
r"(?:web|internet|online(?: sources?)?|"
|
||||
r"current|latest|today['’]?s?|up[- ]to[- ]date|live)"
|
||||
)
|
||||
_TOOL_ACTION_VERBS = (
|
||||
r"search|look up|fetch|browse|web[ _-]?search|"
|
||||
r"web[ _-]?search|"
|
||||
r"(?:search|look up|browse|google) (?:for )?(?:the |a |an )?"
|
||||
rf"{_TOOL_LOOKUP_TARGET}|"
|
||||
r"(?:query|consult) (?:the |a |an )?"
|
||||
r"(?:web|internet|online(?: sources?)?)|"
|
||||
r"(?:find|check|verify) (?:for )?(?:the |a |an )?"
|
||||
r"(?:current|latest|today['’]?s?|up[- ]to[- ]date|live|online|web)|"
|
||||
r"fetch (?:the |a |an )?"
|
||||
rf"{_TOOL_LOOKUP_TARGET}|"
|
||||
r"(?:research|investigate|find|check|verify) (?:for )?(?:the |a |an )?"
|
||||
rf"{_TOOL_LOOKUP_TARGET}|"
|
||||
r"call (?:a |the )?tool|run (?:python|the code)|execute (?:python|the code)"
|
||||
)
|
||||
|
||||
|
|
@ -118,10 +122,11 @@ _HAS_ANSWER_ARTIFACT = re.compile(
|
|||
# closing fence must have at least as many delimiters, and the line
|
||||
# must end cleanly (only trailing whitespace before newline / EOS),
|
||||
# so spam like ``` ```not actually closed ``` does not count.
|
||||
r"(?P<bf>`{3,})[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=bf)`*[ \t]*(?:\r?\n|\Z)"
|
||||
r"(?<!`)(?P<bf>`{3,})(?!`)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=bf)(?!`)[ \t]*(?:\r?\n|\Z)"
|
||||
# Closed tilde code fence; same 3+ rule (several models emit ~~~ when
|
||||
# the body itself contains backticks).
|
||||
r"|(?P<tf>~{3,})[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=tf)~*[ \t]*(?:\r?\n|\Z)"
|
||||
# the body itself contains backticks). Anchored to the full run of
|
||||
# tildes on both sides so a 4-tilde open cannot match a 3-tilde close.
|
||||
r"|(?<!~)(?P<tf>~{3,})(?!~)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=tf)(?!~)[ \t]*(?:\r?\n|\Z)"
|
||||
# Complete HTML page; doctype prefix is optional.
|
||||
r"|(?:<!doctype\b[\s\S]{0,200}?)?<html\b[\s\S]{0,4000}?</html>"
|
||||
# Complete SVG document.
|
||||
|
|
@ -154,15 +159,48 @@ _PLAN_LIST_FRAMING = re.compile(
|
|||
)
|
||||
|
||||
|
||||
_FENCE_LINE_RE = re.compile(r"^[ \t]*(?P<fence>`{3,}|~{3,})(?P<trailing>[^\r\n]*)$")
|
||||
|
||||
|
||||
def _has_unclosed_code_fence(text: str) -> bool:
|
||||
"""True if ``text`` contains a code fence whose closer is missing.
|
||||
|
||||
A complete fence answer is already caught by _HAS_ANSWER_ARTIFACT.
|
||||
This helper exists so that an OPEN fence (model still streaming
|
||||
code, or stream cut short) does not let an embedded numbered list
|
||||
inside the fence body masquerade as a final answer.
|
||||
"""
|
||||
active_char: Optional[str] = None
|
||||
active_len = 0
|
||||
for line in text.splitlines():
|
||||
m = _FENCE_LINE_RE.match(line)
|
||||
if not m:
|
||||
continue
|
||||
fence = m.group("fence")
|
||||
trailing = m.group("trailing").strip()
|
||||
ch = fence[0]
|
||||
if active_char is None:
|
||||
active_char = ch
|
||||
active_len = len(fence)
|
||||
elif ch == active_char and len(fence) >= active_len and not trailing:
|
||||
active_char = None
|
||||
active_len = 0
|
||||
return active_char is not None
|
||||
|
||||
|
||||
def _has_answer_artifact(text: str) -> bool:
|
||||
"""True if ``text`` looks like a completed answer artifact.
|
||||
|
||||
Code fences, complete HTML, and complete SVG count directly. A
|
||||
numbered list counts only when there is no plan framing, so stalls
|
||||
like ``Here's my plan:\\n1. search\\n2. summarise`` still re-prompt.
|
||||
An unclosed fence disqualifies the numbered-list fallback so a list
|
||||
INSIDE incomplete code does not look like a final answer.
|
||||
"""
|
||||
if _HAS_ANSWER_ARTIFACT.search(text):
|
||||
return True
|
||||
if _has_unclosed_code_fence(text):
|
||||
return False
|
||||
if _NUMBERED_LIST_ARTIFACT.search(text):
|
||||
return _PLAN_LIST_FRAMING.search(text) is None
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -234,15 +234,16 @@ def test_numbered_list_without_plan_framing_is_artifact():
|
|||
|
||||
def test_numbered_list_with_plan_framing_is_NOT_artifact():
|
||||
"""A numbered list paired with explicit plan framing (intent phrase
|
||||
followed by a narrow tool-action verb such as ``search`` / ``fetch``
|
||||
/ ``browse``) must NOT count as a completed artifact. The list IS
|
||||
the plan, not the answer. Broad verbs like ``compare`` / ``use`` /
|
||||
``verify`` are intentionally NOT plan framing because real answer
|
||||
lists use them."""
|
||||
followed by a freshness-gated tool-action verb such as
|
||||
``search the web`` / ``fetch the latest`` / ``query the internet``)
|
||||
must NOT count as a completed artifact. The list IS the plan, not
|
||||
the answer. Bare ``search the docs`` / ``compare versions`` /
|
||||
``verify input`` are intentionally NOT plan framing because real
|
||||
answer lists use them."""
|
||||
samples = [
|
||||
"Here's my plan:\n1. Search the web\n2. then summarise.",
|
||||
"First, I'll do these:\n1. search for the song list\n2. cross-check the chart",
|
||||
"Let me look up the values: fetch the data first.",
|
||||
"Here's my plan:\n1. Search the web for the answer.\n2. then summarise.",
|
||||
"First, I'll do these:\n1. fetch the latest chart\n2. cross-check",
|
||||
"Let me look up the values: fetch the current data first.",
|
||||
]
|
||||
for s in samples:
|
||||
assert _PLAN_LIST_FRAMING.search(s), s
|
||||
|
|
@ -350,12 +351,12 @@ def test_reprompts_on_incomplete_html_intent():
|
|||
def test_plan_framing_requires_apostrophe_in_ill():
|
||||
"""The ``i['’]ll`` plan-framing alternative requires an apostrophe so
|
||||
the regex does not match the word "ill" (sick). Without this, a
|
||||
numbered list near "ill" plus an unrelated action verb would be
|
||||
misclassified as a plan and trigger a spurious re-prompt."""
|
||||
numbered list near "ill" plus a freshness-gated lookup verb would
|
||||
be misclassified as a plan and trigger a spurious re-prompt."""
|
||||
samples = [
|
||||
("She is ill. Here is the list:\n1. Apple\n2. Orange\n3. Banana", False),
|
||||
("I'll search the docs:\n1. step\n2. step", True),
|
||||
("I will search:\n1. step\n2. step", True),
|
||||
("I'll search the web for X:\n1. step\n2. step", True),
|
||||
("I will search the latest docs:\n1. step\n2. step", True),
|
||||
]
|
||||
for content, expected in samples:
|
||||
got = _would_reprompt(content)
|
||||
|
|
@ -367,15 +368,17 @@ def test_reprompts_on_all_intent_form_numbered_action_plans():
|
|||
``_INTENT_SIGNAL`` accepts so numbered action plans phrased with
|
||||
``Allow me``, ``I'm going to``, ``I'm gonna``, ``I am gonna``,
|
||||
``I shall``, ``Now I``, ``Next I`` also re-prompt instead of being
|
||||
silently classified as completed answers."""
|
||||
silently classified as completed answers. Each sample pairs the
|
||||
intent form with a freshness-gated lookup verb so the cross-check
|
||||
against _TOOL_ACTION_VERBS succeeds."""
|
||||
samples = [
|
||||
"Allow me to do this:\n1. search the docs\n2. fetch the result",
|
||||
"I'm going to do this:\n1. search the docs\n2. fetch the result",
|
||||
"I'm gonna do this:\n1. search the docs\n2. fetch the result",
|
||||
"I am gonna do this:\n1. search the docs\n2. fetch the result",
|
||||
"I shall do this:\n1. search the docs\n2. fetch the result",
|
||||
"Now I will do these:\n1. search\n2. summarise",
|
||||
"Next I will do these:\n1. fetch\n2. compare",
|
||||
"Allow me to do this:\n1. search the web for X\n2. fetch the latest result",
|
||||
"I'm going to do this:\n1. search the latest docs\n2. fetch the current result",
|
||||
"I'm gonna do this:\n1. search the web for X\n2. fetch the latest result",
|
||||
"I am gonna do this:\n1. search the latest docs\n2. fetch the current result",
|
||||
"I shall do this:\n1. search the web for X\n2. fetch the latest result",
|
||||
"Now I will do these:\n1. search the web\n2. summarise",
|
||||
"Next I will do these:\n1. fetch the latest chart\n2. compare",
|
||||
]
|
||||
for s in samples:
|
||||
assert _would_reprompt(s), s
|
||||
|
|
@ -683,6 +686,76 @@ def test_reasoning_only_visible_artifact_suppresses_reprompt():
|
|||
assert not would_reprompt
|
||||
|
||||
|
||||
def test_no_reprompt_on_binary_search_algorithm_answer():
|
||||
"""A final answer that uses ``search`` as an ordinary algorithm verb
|
||||
(binary search, linear search, depth-first search, etc.) must NOT
|
||||
re-prompt. The lookup gating on ``search`` requires a freshness or
|
||||
web/internet target, so ``Search the left half`` stays an answer."""
|
||||
samples = [
|
||||
(
|
||||
"First, use binary search:\n"
|
||||
"1. Search the left half.\n"
|
||||
"2. Search the right half."
|
||||
),
|
||||
(
|
||||
"First, here are the debugging steps:\n"
|
||||
"1. Search the project for the failing function.\n"
|
||||
"2. Check the stack trace.\n"
|
||||
"3. Verify your fix with tests."
|
||||
),
|
||||
]
|
||||
for content in samples:
|
||||
assert _has_answer_artifact(content), content
|
||||
assert not _would_reprompt(content), content
|
||||
|
||||
|
||||
def test_reprompts_on_numbered_plan_with_google_synonym():
|
||||
"""``google the current X`` reads as an external lookup and STILL
|
||||
re-prompts as a numbered tool plan."""
|
||||
samples = [
|
||||
"Here's my plan:\n1. Google the current Billboard chart.\n2. Summarise.",
|
||||
"First, I'll do this:\n1. Investigate the current exchange rate.\n2. Cite source.",
|
||||
"Here's my approach:\n1. Research the latest release notes.\n2. Summarise.",
|
||||
]
|
||||
for s in samples:
|
||||
assert _would_reprompt(s), s
|
||||
|
||||
|
||||
def test_artifact_regex_rejects_shorter_commonmark_closing_fence():
|
||||
"""Four-or-more delimiter opening fence cannot be closed by fewer
|
||||
delimiters. The opener cannot backtrack to three delimiters and
|
||||
consume the rest as info-string text."""
|
||||
samples = [
|
||||
"First, let me show.\n````python\nprint('hi')\n```",
|
||||
"First, let me show.\n~~~~python\nprint('hi')\n~~~",
|
||||
]
|
||||
for content in samples:
|
||||
assert not _has_answer_artifact(content), content
|
||||
assert _would_reprompt(content), content
|
||||
|
||||
|
||||
def test_open_fence_with_inner_numbered_list_still_reprompts():
|
||||
"""A response that opens a code fence and emits numbered lines INSIDE
|
||||
must NOT count those lines as a completed numbered-list answer."""
|
||||
samples = [
|
||||
(
|
||||
"First, let me write it.\n"
|
||||
"```text\n"
|
||||
"1. Install dependencies\n"
|
||||
"2. Run the app"
|
||||
),
|
||||
(
|
||||
"Let me draft a checklist.\n"
|
||||
"````markdown\n"
|
||||
"1. step one\n"
|
||||
"2. step two"
|
||||
),
|
||||
]
|
||||
for content in samples:
|
||||
assert not _has_answer_artifact(content), content
|
||||
assert _would_reprompt(content), content
|
||||
|
||||
|
||||
def test_hidden_reasoning_artifact_still_reprompts():
|
||||
"""When content tokens were emitted but content_accum is empty (a
|
||||
streaming oddity) and reasoning hides a complete artifact, the user
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue