Studio: r12 fixes - CommonMark 3+ fences, query/consult synonyms, full-window plan scan, visible-reasoning artifact
- _HAS_ANSWER_ARTIFACT now matches fences with three OR MORE backticks / tildes using a named-group backreference (CommonMark rule). Models routinely emit \`\`\`\` / \`\`\`\`\` when the body itself contains a triple fence. The previous regex only matched exactly three. - _TOOL_ACTION_VERBS adds \"query / consult the web / internet / online sources\" so numbered plan stalls phrased with these synonyms still re-prompt instead of being read as final answers. - _PLAN_LIST_FRAMING widens the intent-to-action scan from 80 chars to the full short candidate (caller already gates at _REPROMPT_MAX_CHARS = 2000). Realistic plans where item 1 is preamble and item 2 is the explicit tool action no longer slip through. - Re-prompt call site separates VISIBLE-content artifact check from hidden reasoning. When content_accum is empty AND has_content_tokens is False, reasoning_accum is the user-visible text and counts for the artifact check. Otherwise reasoning stays hidden and an artifact inside it must not suppress the re-prompt.
This commit is contained in:
Daniel Han
parent
562af754c8
commit
ef3ee3d8bd
2 changed files with 162 additions and 20 deletions
|
|
@ -66,6 +66,8 @@ logger = get_logger(__name__)
|
|||
# "check the answer" still read as valid answer text.
|
||||
_TOOL_ACTION_VERBS = (
|
||||
r"search|look up|fetch|browse|web[ _-]?search|"
|
||||
r"(?:query|consult) (?:the |a |an )?"
|
||||
r"(?:web|internet|online(?: sources?)?)|"
|
||||
r"(?:find|check|verify) (?:for )?(?:the |a |an )?"
|
||||
r"(?:current|latest|today['’]?s?|up[- ]to[- ]date|live|online|web)|"
|
||||
r"call (?:a |the )?tool|run (?:python|the code)|execute (?:python|the code)"
|
||||
|
|
@ -112,13 +114,14 @@ _MAX_REPROMPTS = 3
|
|||
# linear on adversarial input (CRLF spam, repeated `<html>` etc.).
|
||||
_HAS_ANSWER_ARTIFACT = re.compile(
|
||||
# Closed backtick code fence (any markdown info string, optional indent
|
||||
# on close). The closing fence must end the line: only optional
|
||||
# trailing whitespace before a newline or end-of-string, so spam
|
||||
# like ``` ```not actually closed ``` does not count.
|
||||
r"```[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*```[ \t]*(?:\r?\n|\Z)"
|
||||
# Closed tilde code fence (CommonMark also allows ~~~ fences; several
|
||||
# models emit them when the body itself contains backticks).
|
||||
r"|~~~[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*~~~[ \t]*(?:\r?\n|\Z)"
|
||||
# on close). CommonMark allows opening fences of 3+ backticks; the
|
||||
# closing fence must have at least as many delimiters, and the line
|
||||
# must end cleanly (only trailing whitespace before newline / EOS),
|
||||
# so spam like ``` ```not actually closed ``` does not count.
|
||||
r"(?P<bf>`{3,})[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=bf)`*[ \t]*(?:\r?\n|\Z)"
|
||||
# Closed tilde code fence; same 3+ rule (several models emit ~~~ when
|
||||
# the body itself contains backticks).
|
||||
r"|(?P<tf>~{3,})[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=tf)~*[ \t]*(?:\r?\n|\Z)"
|
||||
# Complete HTML page; doctype prefix is optional.
|
||||
r"|(?:<!doctype\b[\s\S]{0,200}?)?<html\b[\s\S]{0,4000}?</html>"
|
||||
# Complete SVG document.
|
||||
|
|
@ -133,17 +136,19 @@ _NUMBERED_LIST_ARTIFACT = re.compile(
|
|||
)
|
||||
|
||||
# Markers that a numbered list is a plan (still re-promptable), not a
|
||||
# final answer. Only fires when an intent phrase from _INTENT_SIGNAL is
|
||||
# already followed within 80 chars by a narrow tool-action verb. The
|
||||
# apostrophe in ``i['’]ll`` is required (no ``?``) so the regex does
|
||||
# not accidentally match the word "ill". Without a tool-action verb the
|
||||
# numbered list is treated as a completed answer artifact.
|
||||
# final answer. Fires when an intent phrase from _INTENT_SIGNAL is
|
||||
# followed anywhere in the short re-prompt candidate by a tool-action
|
||||
# verb. The apostrophe in ``i['’]ll`` is required (no ``?``) so the
|
||||
# regex does not accidentally match the word "ill". Without a tool-
|
||||
# action verb the numbered list is treated as a completed answer
|
||||
# artifact. The scan window is bounded at _REPROMPT_MAX_CHARS by the
|
||||
# caller, so the lazy ``[\s\S]{0,2000}?`` quantifier stays linear.
|
||||
_PLAN_LIST_FRAMING = re.compile(
|
||||
r"\b(?:here['’]?s (?:my |the |a )?(?:plan|approach)|"
|
||||
r"step \d+|first|"
|
||||
r"i['’](?:ll|m going to|m gonna)|i am (?:going to|gonna)|"
|
||||
r"i will|i shall|let me|allow me|now i|next i)\b"
|
||||
r"[\s\S]{0,80}"
|
||||
r"[\s\S]{0,2000}?"
|
||||
rf"\b(?:{_TOOL_ACTION_VERBS})\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
|
@ -4902,14 +4907,21 @@ class LlamaCppBackend:
|
|||
# like "4" or "Hello!" won't trigger this.
|
||||
# Use content if available, otherwise fall back
|
||||
# to reasoning text (reasoning-only stalls).
|
||||
# Artifact check uses VISIBLE content only:
|
||||
# a closed code fence inside hidden reasoning is
|
||||
# not a user-visible answer, so it must not
|
||||
# suppress the re-prompt.
|
||||
# Artifact check uses USER-VISIBLE text only.
|
||||
# Reasoning is only user-visible when there are
|
||||
# no content tokens (the branch above yields
|
||||
# reasoning_accum as plain content in that
|
||||
# case); otherwise reasoning stays hidden and
|
||||
# an artifact inside it must NOT suppress the
|
||||
# re-prompt.
|
||||
_visible = content_accum.strip()
|
||||
_stripped = _visible if _visible else reasoning_accum.strip()
|
||||
_visible_has_artifact = bool(_visible) and _has_answer_artifact(
|
||||
_visible
|
||||
_reasoning = reasoning_accum.strip()
|
||||
_stripped = _visible if _visible else _reasoning
|
||||
_artifact_text = _visible if _visible else (
|
||||
_reasoning if not has_content_tokens else ""
|
||||
)
|
||||
_visible_has_artifact = bool(_artifact_text) and _has_answer_artifact(
|
||||
_artifact_text
|
||||
)
|
||||
if (
|
||||
tools
|
||||
|
|
|
|||
|
|
@ -576,3 +576,133 @@ def test_no_reprompt_on_numbered_answer_with_bare_find_or_check():
|
|||
for s in samples:
|
||||
assert _has_answer_artifact(s), s
|
||||
assert not _would_reprompt(s), s
|
||||
|
||||
|
||||
# ── CommonMark fences with 4+ delimiters ──────────────────────────
|
||||
|
||||
|
||||
def test_artifact_regex_detects_four_or_more_backticks():
|
||||
"""CommonMark allows opening fences of 3+ backticks. Models use
|
||||
4+ delimiters when the body itself contains a triple fence."""
|
||||
samples = [
|
||||
"First, let me show.\n````python\nprint('``` inside')\n````",
|
||||
"Let me show.\n`````markdown\n```python\nprint(1)\n```\n`````",
|
||||
]
|
||||
for text in samples:
|
||||
assert _has_answer_artifact(text), text
|
||||
assert not _would_reprompt(text), text
|
||||
|
||||
|
||||
def test_artifact_regex_detects_four_or_more_tildes():
|
||||
"""Same 3+ delimiter rule for tilde fences."""
|
||||
text = "First, let me show.\n~~~~python\nprint('hi')\n~~~~"
|
||||
assert _has_answer_artifact(text)
|
||||
assert not _would_reprompt(text)
|
||||
|
||||
|
||||
# ── Query / consult online sources ────────────────────────────────
|
||||
|
||||
|
||||
def test_reprompts_on_numbered_plan_with_query_consult_synonyms():
|
||||
"""``query the web`` / ``consult online sources`` are tool-lookup
|
||||
synonyms and STILL re-prompt as numbered tool plans."""
|
||||
samples = [
|
||||
"Here's my plan:\n1. Query the web for today's USD/EUR rate.\n2. Summarize.",
|
||||
"Here's my plan:\n1. Consult online sources for the latest release.\n2. Answer.",
|
||||
"First, I'll do this:\n1. Query the internet for the current chart.\n2. Summarize.",
|
||||
]
|
||||
for s in samples:
|
||||
assert _would_reprompt(s), s
|
||||
|
||||
|
||||
# ── Delayed numbered tool action ──────────────────────────────────
|
||||
|
||||
|
||||
def test_reprompts_on_numbered_plan_when_action_after_long_first_item():
|
||||
"""Plans where the explicit tool action appears beyond the first 80
|
||||
chars (long preamble or long item 1) must STILL re-prompt. The
|
||||
framing scan needs to cover the whole short candidate, not just the
|
||||
nearest 80 chars."""
|
||||
samples = [
|
||||
(
|
||||
"Here's my plan:\n"
|
||||
"1. Review the question and identify exactly what current data is "
|
||||
"needed before using external sources.\n"
|
||||
"2. Search the web for today's USD/EUR rate.\n"
|
||||
"3. Answer with a citation."
|
||||
),
|
||||
(
|
||||
"Here's my plan:\n"
|
||||
"1. Clarify the requirements and identify the exact data source "
|
||||
"that contains the current numbers.\n"
|
||||
"2. Search the web for the current Billboard chart.\n"
|
||||
"3. Summarise the answer."
|
||||
),
|
||||
(
|
||||
"First, I'll explain the process before acting so the user can "
|
||||
"follow along safely and so I can avoid using stale information.\n"
|
||||
"1. Search the web for the current Billboard chart.\n"
|
||||
"2. Summarise the answer."
|
||||
),
|
||||
]
|
||||
for s in samples:
|
||||
assert _would_reprompt(s), s
|
||||
|
||||
|
||||
# ── Reasoning-only visible-output path ────────────────────────────
|
||||
|
||||
|
||||
def test_reasoning_only_visible_artifact_suppresses_reprompt():
|
||||
"""When content_accum is empty AND there are no content tokens, the
|
||||
backend yields reasoning_accum as plain content. In that case the
|
||||
reasoning text IS the user-visible answer and a complete artifact
|
||||
inside it should suppress the re-prompt."""
|
||||
from core.inference.llama_cpp import _REPROMPT_MAX_CHARS
|
||||
|
||||
content_accum = ""
|
||||
reasoning_accum = (
|
||||
"First, let me set up pygame.\n"
|
||||
"```python\n"
|
||||
"import pygame\n"
|
||||
"pygame.init()\n"
|
||||
"```"
|
||||
)
|
||||
has_content_tokens = False
|
||||
|
||||
visible = content_accum.strip()
|
||||
reasoning = reasoning_accum.strip()
|
||||
stripped = visible if visible else reasoning
|
||||
artifact_text = visible if visible else (reasoning if not has_content_tokens else "")
|
||||
would_reprompt = bool(
|
||||
0 < len(stripped) < _REPROMPT_MAX_CHARS
|
||||
and _INTENT_SIGNAL.search(stripped)
|
||||
and not (artifact_text and _has_answer_artifact(artifact_text))
|
||||
)
|
||||
assert not would_reprompt
|
||||
|
||||
|
||||
def test_hidden_reasoning_artifact_still_reprompts():
|
||||
"""When content tokens were emitted but content_accum is empty (a
|
||||
streaming oddity) and reasoning hides a complete artifact, the user
|
||||
sees nothing, so the re-prompt MUST still fire."""
|
||||
from core.inference.llama_cpp import _REPROMPT_MAX_CHARS
|
||||
|
||||
content_accum = ""
|
||||
reasoning_accum = (
|
||||
"First, let me draft it.\n"
|
||||
"```python\n"
|
||||
"print('hidden answer')\n"
|
||||
"```"
|
||||
)
|
||||
has_content_tokens = True # content existed but was stripped
|
||||
|
||||
visible = content_accum.strip()
|
||||
reasoning = reasoning_accum.strip()
|
||||
stripped = visible if visible else reasoning
|
||||
artifact_text = visible if visible else (reasoning if not has_content_tokens else "")
|
||||
would_reprompt = bool(
|
||||
0 < len(stripped) < _REPROMPT_MAX_CHARS
|
||||
and _INTENT_SIGNAL.search(stripped)
|
||||
and not (artifact_text and _has_answer_artifact(artifact_text))
|
||||
)
|
||||
assert would_reprompt
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue