From ef3ee3d8bd34e9a645a658d103ed9860277ee6af Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 15:35:27 +0000 Subject: [PATCH] Studio: r12 fixes - CommonMark 3+ fences, query/consult synonyms, full-window plan scan, visible-reasoning artifact - _HAS_ANSWER_ARTIFACT now matches fences with three OR MORE backticks / tildes using a named-group backreference (CommonMark rule). Models routinely emit \`\`\`\` / \`\`\`\`\` when the body itself contains a triple fence. The previous regex only matched exactly three. - _TOOL_ACTION_VERBS adds \"query / consult the web / internet / online sources\" so numbered plan stalls phrased with these synonyms still re-prompt instead of being read as final answers. - _PLAN_LIST_FRAMING widens the intent-to-action scan from 80 chars to the full short candidate (caller already gates at _REPROMPT_MAX_CHARS = 2000). Realistic plans where item 1 is preamble and item 2 is the explicit tool action no longer slip through. - Re-prompt call site separates VISIBLE-content artifact check from hidden reasoning. When content_accum is empty AND has_content_tokens is False, reasoning_accum is the user-visible text and counts for the artifact check. Otherwise reasoning stays hidden and an artifact inside it must not suppress the re-prompt. --- studio/backend/core/inference/llama_cpp.py | 52 ++++--- .../tests/test_llama_cpp_reprompt_guard.py | 130 ++++++++++++++++++ 2 files changed, 162 insertions(+), 20 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index d885734a9e..eef41608a8 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -66,6 +66,8 @@ logger = get_logger(__name__) # "check the answer" still read as valid answer text. _TOOL_ACTION_VERBS = ( r"search|look up|fetch|browse|web[ _-]?search|" + r"(?:query|consult) (?:the |a |an )?" + r"(?:web|internet|online(?: sources?)?)|" r"(?:find|check|verify) (?:for )?(?:the |a |an )?" r"(?:current|latest|today['’]?s?|up[- ]to[- ]date|live|online|web)|" r"call (?:a |the )?tool|run (?:python|the code)|execute (?:python|the code)" @@ -112,13 +114,14 @@ _MAX_REPROMPTS = 3 # linear on adversarial input (CRLF spam, repeated `` etc.). _HAS_ANSWER_ARTIFACT = re.compile( # Closed backtick code fence (any markdown info string, optional indent - # on close). The closing fence must end the line: only optional - # trailing whitespace before a newline or end-of-string, so spam - # like ``` ```not actually closed ``` does not count. - r"```[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*```[ \t]*(?:\r?\n|\Z)" - # Closed tilde code fence (CommonMark also allows ~~~ fences; several - # models emit them when the body itself contains backticks). - r"|~~~[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*~~~[ \t]*(?:\r?\n|\Z)" + # on close). CommonMark allows opening fences of 3+ backticks; the + # closing fence must have at least as many delimiters, and the line + # must end cleanly (only trailing whitespace before newline / EOS), + # so spam like ``` ```not actually closed ``` does not count. + r"(?P`{3,})[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=bf)`*[ \t]*(?:\r?\n|\Z)" + # Closed tilde code fence; same 3+ rule (several models emit ~~~ when + # the body itself contains backticks). + r"|(?P~{3,})[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=tf)~*[ \t]*(?:\r?\n|\Z)" # Complete HTML page; doctype prefix is optional. r"|(?:" # Complete SVG document. @@ -133,17 +136,19 @@ _NUMBERED_LIST_ARTIFACT = re.compile( ) # Markers that a numbered list is a plan (still re-promptable), not a -# final answer. Only fires when an intent phrase from _INTENT_SIGNAL is -# already followed within 80 chars by a narrow tool-action verb. The -# apostrophe in ``i['’]ll`` is required (no ``?``) so the regex does -# not accidentally match the word "ill". Without a tool-action verb the -# numbered list is treated as a completed answer artifact. +# final answer. Fires when an intent phrase from _INTENT_SIGNAL is +# followed anywhere in the short re-prompt candidate by a tool-action +# verb. The apostrophe in ``i['’]ll`` is required (no ``?``) so the +# regex does not accidentally match the word "ill". Without a tool- +# action verb the numbered list is treated as a completed answer +# artifact. The scan window is bounded at _REPROMPT_MAX_CHARS by the +# caller, so the lazy ``[\s\S]{0,2000}?`` quantifier stays linear. _PLAN_LIST_FRAMING = re.compile( r"\b(?:here['’]?s (?:my |the |a )?(?:plan|approach)|" r"step \d+|first|" r"i['’](?:ll|m going to|m gonna)|i am (?:going to|gonna)|" r"i will|i shall|let me|allow me|now i|next i)\b" - r"[\s\S]{0,80}" + r"[\s\S]{0,2000}?" rf"\b(?:{_TOOL_ACTION_VERBS})\b", re.IGNORECASE, ) @@ -4902,14 +4907,21 @@ class LlamaCppBackend: # like "4" or "Hello!" won't trigger this. # Use content if available, otherwise fall back # to reasoning text (reasoning-only stalls). - # Artifact check uses VISIBLE content only: - # a closed code fence inside hidden reasoning is - # not a user-visible answer, so it must not - # suppress the re-prompt. + # Artifact check uses USER-VISIBLE text only. + # Reasoning is only user-visible when there are + # no content tokens (the branch above yields + # reasoning_accum as plain content in that + # case); otherwise reasoning stays hidden and + # an artifact inside it must NOT suppress the + # re-prompt. _visible = content_accum.strip() - _stripped = _visible if _visible else reasoning_accum.strip() - _visible_has_artifact = bool(_visible) and _has_answer_artifact( - _visible + _reasoning = reasoning_accum.strip() + _stripped = _visible if _visible else _reasoning + _artifact_text = _visible if _visible else ( + _reasoning if not has_content_tokens else "" + ) + _visible_has_artifact = bool(_artifact_text) and _has_answer_artifact( + _artifact_text ) if ( tools diff --git a/studio/backend/tests/test_llama_cpp_reprompt_guard.py b/studio/backend/tests/test_llama_cpp_reprompt_guard.py index 59e10a2083..b81f0c80df 100644 --- a/studio/backend/tests/test_llama_cpp_reprompt_guard.py +++ b/studio/backend/tests/test_llama_cpp_reprompt_guard.py @@ -576,3 +576,133 @@ def test_no_reprompt_on_numbered_answer_with_bare_find_or_check(): for s in samples: assert _has_answer_artifact(s), s assert not _would_reprompt(s), s + + +# ── CommonMark fences with 4+ delimiters ────────────────────────── + + +def test_artifact_regex_detects_four_or_more_backticks(): + """CommonMark allows opening fences of 3+ backticks. Models use + 4+ delimiters when the body itself contains a triple fence.""" + samples = [ + "First, let me show.\n````python\nprint('``` inside')\n````", + "Let me show.\n`````markdown\n```python\nprint(1)\n```\n`````", + ] + for text in samples: + assert _has_answer_artifact(text), text + assert not _would_reprompt(text), text + + +def test_artifact_regex_detects_four_or_more_tildes(): + """Same 3+ delimiter rule for tilde fences.""" + text = "First, let me show.\n~~~~python\nprint('hi')\n~~~~" + assert _has_answer_artifact(text) + assert not _would_reprompt(text) + + +# ── Query / consult online sources ──────────────────────────────── + + +def test_reprompts_on_numbered_plan_with_query_consult_synonyms(): + """``query the web`` / ``consult online sources`` are tool-lookup + synonyms and STILL re-prompt as numbered tool plans.""" + samples = [ + "Here's my plan:\n1. Query the web for today's USD/EUR rate.\n2. Summarize.", + "Here's my plan:\n1. Consult online sources for the latest release.\n2. Answer.", + "First, I'll do this:\n1. Query the internet for the current chart.\n2. Summarize.", + ] + for s in samples: + assert _would_reprompt(s), s + + +# ── Delayed numbered tool action ────────────────────────────────── + + +def test_reprompts_on_numbered_plan_when_action_after_long_first_item(): + """Plans where the explicit tool action appears beyond the first 80 + chars (long preamble or long item 1) must STILL re-prompt. The + framing scan needs to cover the whole short candidate, not just the + nearest 80 chars.""" + samples = [ + ( + "Here's my plan:\n" + "1. Review the question and identify exactly what current data is " + "needed before using external sources.\n" + "2. Search the web for today's USD/EUR rate.\n" + "3. Answer with a citation." + ), + ( + "Here's my plan:\n" + "1. Clarify the requirements and identify the exact data source " + "that contains the current numbers.\n" + "2. Search the web for the current Billboard chart.\n" + "3. Summarise the answer." + ), + ( + "First, I'll explain the process before acting so the user can " + "follow along safely and so I can avoid using stale information.\n" + "1. Search the web for the current Billboard chart.\n" + "2. Summarise the answer." + ), + ] + for s in samples: + assert _would_reprompt(s), s + + +# ── Reasoning-only visible-output path ──────────────────────────── + + +def test_reasoning_only_visible_artifact_suppresses_reprompt(): + """When content_accum is empty AND there are no content tokens, the + backend yields reasoning_accum as plain content. In that case the + reasoning text IS the user-visible answer and a complete artifact + inside it should suppress the re-prompt.""" + from core.inference.llama_cpp import _REPROMPT_MAX_CHARS + + content_accum = "" + reasoning_accum = ( + "First, let me set up pygame.\n" + "```python\n" + "import pygame\n" + "pygame.init()\n" + "```" + ) + has_content_tokens = False + + visible = content_accum.strip() + reasoning = reasoning_accum.strip() + stripped = visible if visible else reasoning + artifact_text = visible if visible else (reasoning if not has_content_tokens else "") + would_reprompt = bool( + 0 < len(stripped) < _REPROMPT_MAX_CHARS + and _INTENT_SIGNAL.search(stripped) + and not (artifact_text and _has_answer_artifact(artifact_text)) + ) + assert not would_reprompt + + +def test_hidden_reasoning_artifact_still_reprompts(): + """When content tokens were emitted but content_accum is empty (a + streaming oddity) and reasoning hides a complete artifact, the user + sees nothing, so the re-prompt MUST still fire.""" + from core.inference.llama_cpp import _REPROMPT_MAX_CHARS + + content_accum = "" + reasoning_accum = ( + "First, let me draft it.\n" + "```python\n" + "print('hidden answer')\n" + "```" + ) + has_content_tokens = True # content existed but was stripped + + visible = content_accum.strip() + reasoning = reasoning_accum.strip() + stripped = visible if visible else reasoning + artifact_text = visible if visible else (reasoning if not has_content_tokens else "") + would_reprompt = bool( + 0 < len(stripped) < _REPROMPT_MAX_CHARS + and _INTENT_SIGNAL.search(stripped) + and not (artifact_text and _has_answer_artifact(artifact_text)) + ) + assert would_reprompt