From 4652a4b03c63fc49a42a0ec1038aaa2a68dd172e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 20:20:38 +0000 Subject: [PATCH] Studio: r14 fixes - longer CommonMark closing fence, explicit-plan header standalone, use-python tool wording - _HAS_ANSWER_ARTIFACT closing fence now accepts strictly more delimiters than the opener (CommonMark rule). The opener stays anchored on both sides so a 4-open / 3-close payload still does not match, but a legitimate 3-open / 4-close (and 3-tilde / 4-tilde) answer is now recognised as a completed artifact. - _EXPLICIT_PLAN_HEADER triggers the plan classification by itself when the response contains \"Here's my plan\" / \"Here's my approach\" / \"Here's the plan\". Numbered stalls like \"Here's my plan:\n1. Analyze\n 2. Draft\" re-prompt again without needing a freshness-gated verb. Plain \"Plan:\" / \"My weekly plan:\" stay valid answers because they lack the possessive first-person header. - _TOOL_ACTION_VERBS adds \"use python (tool) to ...\", \"use the python tool\", \"invoke the python tool\", and \"use the search tool\" so numbered plans that route through these phrasings still re-prompt. --- studio/backend/core/inference/llama_cpp.py | 29 +++++++-- .../tests/test_llama_cpp_reprompt_guard.py | 63 +++++++++++++++++++ 2 files changed, 86 insertions(+), 6 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 6278167491..a843336eaf 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -74,6 +74,8 @@ _TOOL_ACTION_VERBS = ( rf"{_TOOL_LOOKUP_TARGET}|" r"(?:research|investigate|find|check|verify) (?:for )?(?:the |a |an )?" rf"{_TOOL_LOOKUP_TARGET}|" + r"(?:use|invoke|call) (?:the )?(?:python|search) tool|" + r"use python(?: tool)? to|" r"call (?:a |the )?tool|run (?:python|the code)|execute (?:python|the code)" ) @@ -122,11 +124,11 @@ _HAS_ANSWER_ARTIFACT = re.compile( # closing fence must have at least as many delimiters, and the line # must end cleanly (only trailing whitespace before newline / EOS), # so spam like ``` ```not actually closed ``` does not count. - r"(?`{3,})(?!`)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=bf)(?!`)[ \t]*(?:\r?\n|\Z)" + r"(?`{3,})(?!`)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=bf)`*[ \t]*(?:\r?\n|\Z)" # Closed tilde code fence; same 3+ rule (several models emit ~~~ when - # the body itself contains backticks). Anchored to the full run of - # tildes on both sides so a 4-tilde open cannot match a 3-tilde close. - r"|(?~{3,})(?!~)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=tf)(?!~)[ \t]*(?:\r?\n|\Z)" + # the body itself contains backticks). Opener anchored to the full + # run of tildes; closer accepts >= opener length per CommonMark. + r"|(?~{3,})(?!~)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=tf)~*[ \t]*(?:\r?\n|\Z)" # Complete HTML page; doctype prefix is optional. r"|(?:" # Complete SVG document. @@ -158,6 +160,16 @@ _PLAN_LIST_FRAMING = re.compile( re.IGNORECASE, ) +# "Here's my plan" / "Here's my approach" are strong stand-alone plan +# signals: a possessive, first-person framing where the model is +# announcing what it WILL do. Treat the following numbered list as a +# plan regardless of the specific verbs each item uses, so stalls like +# ``Here's my plan: 1. Analyze 2. Draft`` still re-prompt. +_EXPLICIT_PLAN_HEADER = re.compile( + r"\bhere['’]?s (?:my |the |a )?(?:plan|approach)\b", + re.IGNORECASE, +) + _FENCE_LINE_RE = re.compile(r"^[ \t]*(?P`{3,}|~{3,})(?P[^\r\n]*)$") @@ -194,14 +206,19 @@ def _has_answer_artifact(text: str) -> bool: Code fences, complete HTML, and complete SVG count directly. A numbered list counts only when there is no plan framing, so stalls like ``Here's my plan:\\n1. search\\n2. summarise`` still re-prompt. - An unclosed fence disqualifies the numbered-list fallback so a list - INSIDE incomplete code does not look like a final answer. + An explicit ``Here's my plan`` / ``Here's my approach`` header is + also enough to flag the list as a plan, even when no narrow tool- + action verb appears in the items. An unclosed fence disqualifies + the numbered-list fallback so a list INSIDE incomplete code does + not look like a final answer. """ if _HAS_ANSWER_ARTIFACT.search(text): return True if _has_unclosed_code_fence(text): return False if _NUMBERED_LIST_ARTIFACT.search(text): + if _EXPLICIT_PLAN_HEADER.search(text): + return False return _PLAN_LIST_FRAMING.search(text) is None return False diff --git a/studio/backend/tests/test_llama_cpp_reprompt_guard.py b/studio/backend/tests/test_llama_cpp_reprompt_guard.py index 6ebff5778c..d712e1bbac 100644 --- a/studio/backend/tests/test_llama_cpp_reprompt_guard.py +++ b/studio/backend/tests/test_llama_cpp_reprompt_guard.py @@ -749,6 +749,69 @@ def test_artifact_regex_rejects_shorter_commonmark_closing_fence(): assert _would_reprompt(content), content +def test_artifact_regex_accepts_longer_commonmark_closing_fence(): + """CommonMark allows the closing fence to have MORE delimiters than + the opener. A 3-backtick opener with a 4-backtick close, or a + 3-tilde opener with a 4-tilde close, is still a complete artifact.""" + samples = [ + "First, let me show.\n```python\nprint('hi')\n````", + "First, let me show.\n````python\nprint('``` inside')\n`````", + "First, let me show.\n~~~python\nprint('hi')\n~~~~", + ] + for content in samples: + assert _has_answer_artifact(content), content + assert not _would_reprompt(content), content + + +def test_reprompts_on_explicit_plan_header_numbered_list(): + """``Here's my plan`` / ``Here's my approach`` is a strong stand-alone + plan signal. The following numbered list is the plan itself, not a + final answer, even when no narrow tool-action verb appears.""" + samples = [ + "Here's my plan:\n1. Analyze the request.\n2. Draft the answer.", + "Here's my plan:\n1. Create the Python file.\n2. Add the game loop.\n3. Test.", + "Here's my approach:\n1. Outline.\n2. Write.\n3. Review.", + "Here's the plan:\n1. Define the variables.\n2. Return the result.", + ] + for s in samples: + assert _would_reprompt(s), s + + +def test_reprompts_on_numbered_plan_with_python_tool_wording(): + """``use python (tool) to ...`` / ``use the python tool`` / ``use the + search tool`` in a numbered plan still re-prompts.""" + samples = [ + "Here's my plan:\n1. Use Python to calculate the answer.\n2. Return.", + "First, I'll do this:\n1. Use the python tool to parse the file.\n2. Summarize.", + "Here's my plan:\n1. Use the search tool.\n2. Summarize.", + ] + for s in samples: + assert _would_reprompt(s), s + + +def test_no_reprompt_on_lesson_plan_answer_without_explicit_header(): + """A final answer with a ``Plan:`` heading (no ``Here's my`` + possessive) and no tool framing must STILL count as an answer. + Common cases: lesson plan, workout plan, meal plan.""" + samples = [ + ( + "Plan:\n" + "1. Warm up for 5 minutes.\n" + "2. Run for 20 minutes.\n" + "3. Cool down with stretching." + ), + ( + "My weekly plan:\n" + "1. Monday: rest.\n" + "2. Tuesday: jog.\n" + "3. Wednesday: swim." + ), + ] + for content in samples: + assert _has_answer_artifact(content), content + assert not _would_reprompt(content), content + + def test_open_fence_with_inner_numbered_list_still_reprompts(): """A response that opens a code fence and emits numbered lines INSIDE must NOT count those lines as a completed numbered-list answer."""