From 9fa736bef192d195a5ea420ff3df64f74ac05f3c Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 20:33:15 +0000 Subject: [PATCH] Studio: r15 fixes - direct-intent numbered plan stalls, unclosed-fence short-circuit - _DIRECT_NUMBERED_PLAN_FRAMING matches first-person intent ("I'll", "Let me", etc.) plus a narrow follow-up verb ("do this", "do these", "create", "build", "set up", "calculate", "parse", "run", etc.) followed by a numbered list. This catches stalls like "First, I'll do this:\n1. Search for X." or "Let me do this:\n1. Parse the JSON.\n2. Calculate the average." where the model announces actions but never invokes a tool. The verb whitelist stays narrow so "Let me explain" / "Let me show" / "Let me draft a poem" answers are NOT misclassified. - _has_answer_artifact() now checks for an unclosed code fence BEFORE consulting _HAS_ANSWER_ARTIFACT. A response with one complete fence followed by a second, still-open fence (mid-stream multi-file answers) no longer suppresses the re-prompt; the unclosed second fence wins. --- studio/backend/core/inference/llama_cpp.py | 39 ++++++++-- .../tests/test_llama_cpp_reprompt_guard.py | 74 +++++++++++++++++++ 2 files changed, 106 insertions(+), 7 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index a843336eaf..64abdae139 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -170,6 +170,27 @@ _EXPLICIT_PLAN_HEADER = re.compile( re.IGNORECASE, ) +# Direct first-person intent + a tool/work verb that the model is about +# to perform + a numbered list. Catches stalls like +# ``First, I'll do this:\n1. Search ...`` or ``Let me do this:\n1. Parse +# the file ...`` where each list item is an action the model promised +# to take without actually invoking a tool. The follow-up verb list +# stays narrow to "do/proceed/build/run" style words so common +# answer prose like "Let me explain", "Let me show", "Let me draft a +# poem" is not misclassified as a stall. +_DIRECT_NUMBERED_PLAN_FRAMING = re.compile( + r"\b(?:i['’](?:ll|m going to|m gonna)|i am (?:going to|gonna)|" + r"i will|i shall|let me|allow me|now i|next i)\b" + r"[^\r\n]{0,160}" + r"\b(?:do (?:this|these|the following|it)|" + r"proceed|start|begin|" + r"create|build|implement|set up|add|" + r"calculate|compute|analy[sz]e|parse|load|run|execute|test)\b" + r"[\s\S]{0,500}?" + r"(?:^|\r?\n)[ \t]*\d+\.", + re.IGNORECASE, +) + _FENCE_LINE_RE = re.compile(r"^[ \t]*(?P`{3,}|~{3,})(?P[^\r\n]*)$") @@ -206,19 +227,23 @@ def _has_answer_artifact(text: str) -> bool: Code fences, complete HTML, and complete SVG count directly. A numbered list counts only when there is no plan framing, so stalls like ``Here's my plan:\\n1. search\\n2. summarise`` still re-prompt. - An explicit ``Here's my plan`` / ``Here's my approach`` header is - also enough to flag the list as a plan, even when no narrow tool- - action verb appears in the items. An unclosed fence disqualifies - the numbered-list fallback so a list INSIDE incomplete code does - not look like a final answer. + An explicit ``Here's my plan`` / ``Here's my approach`` header, or a + direct first-person ``I'll do this:\\n1. ...`` framing with a + work/tool verb before the list, also flags the list as a plan even + when no narrow tool-action verb appears in the items. An unclosed + fence anywhere in the response (including after an earlier complete + fence) disqualifies the answer-artifact path so half-finished code + does not look like a final answer. """ - if _HAS_ANSWER_ARTIFACT.search(text): - return True if _has_unclosed_code_fence(text): return False + if _HAS_ANSWER_ARTIFACT.search(text): + return True if _NUMBERED_LIST_ARTIFACT.search(text): if _EXPLICIT_PLAN_HEADER.search(text): return False + if _DIRECT_NUMBERED_PLAN_FRAMING.search(text): + return False return _PLAN_LIST_FRAMING.search(text) is None return False diff --git a/studio/backend/tests/test_llama_cpp_reprompt_guard.py b/studio/backend/tests/test_llama_cpp_reprompt_guard.py index d712e1bbac..e54b8de05d 100644 --- a/studio/backend/tests/test_llama_cpp_reprompt_guard.py +++ b/studio/backend/tests/test_llama_cpp_reprompt_guard.py @@ -812,6 +812,80 @@ def test_no_reprompt_on_lesson_plan_answer_without_explicit_header(): assert not _would_reprompt(content), content +def test_reprompts_on_direct_intent_numbered_local_action_plan(): + """Direct first-person intent (``I'll do this``, ``Let me do this``, + etc.) followed by a numbered list of work/tool actions is a plan + stall, not a final answer. Even when no narrow lookup verb appears + in the items, the model is announcing actions it has not yet + taken.""" + samples = [ + ( + "First, I'll do this:\n" + "1. Load the uploaded CSV.\n" + "2. Compute the total revenue.\n" + "3. Return the answer." + ), + ( + "Let me do this:\n" + "1. Parse the pasted JSON.\n" + "2. Calculate the average.\n" + "3. Explain the result." + ), + ( + "First, I'll create a Python game:\n" + "1. Set up pygame.\n" + "2. Add the game loop." + ), + ( + "First, I'll do these:\n" + "1. Create the Python file.\n" + "2. Add the game loop.\n" + "3. Test it." + ), + ] + for content in samples: + assert _would_reprompt(content), content + + +def test_no_reprompt_on_let_me_explain_numbered_answer(): + """``Let me explain`` / ``Let me show`` followed by a numbered + answer must NOT be misclassified as a plan stall. The verb after + the intent phrase is not in the work/tool whitelist.""" + samples = [ + ( + "Let me explain in steps:\n" + "1. Apples are red.\n" + "2. Bananas are yellow.\n" + "3. Cherries are red." + ), + ( + "Let me show the matches:\n" + "1. Maroon 5 - Animals.\n" + "2. Hozier - Take Me to Church." + ), + ] + for content in samples: + assert _has_answer_artifact(content), content + assert not _would_reprompt(content), content + + +def test_reprompts_when_later_fence_is_open_after_closed_fence(): + """A response with a complete code fence followed by a SECOND, + unclosed fence is still mid-stream and must re-prompt. The + `_has_unclosed_code_fence` cross-check must short-circuit even + after `_HAS_ANSWER_ARTIFACT` finds the first complete fence.""" + content = ( + "First, let me provide two files:\n" + "```python\n" + "print('main')\n" + "```\n" + "```python\n" + "print('utils')" + ) + assert not _has_answer_artifact(content) + assert _would_reprompt(content) + + def test_open_fence_with_inner_numbered_list_still_reprompts(): """A response that opens a code fence and emits numbered lines INSIDE must NOT count those lines as a completed numbered-list answer."""