diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 6278167491..a843336eaf 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -74,6 +74,8 @@ _TOOL_ACTION_VERBS = ( rf"{_TOOL_LOOKUP_TARGET}|" r"(?:research|investigate|find|check|verify) (?:for )?(?:the |a |an )?" rf"{_TOOL_LOOKUP_TARGET}|" + r"(?:use|invoke|call) (?:the )?(?:python|search) tool|" + r"use python(?: tool)? to|" r"call (?:a |the )?tool|run (?:python|the code)|execute (?:python|the code)" ) @@ -122,11 +124,11 @@ _HAS_ANSWER_ARTIFACT = re.compile( # closing fence must have at least as many delimiters, and the line # must end cleanly (only trailing whitespace before newline / EOS), # so spam like ``` ```not actually closed ``` does not count. - r"(?`{3,})(?!`)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=bf)(?!`)[ \t]*(?:\r?\n|\Z)" + r"(?`{3,})(?!`)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=bf)`*[ \t]*(?:\r?\n|\Z)" # Closed tilde code fence; same 3+ rule (several models emit ~~~ when - # the body itself contains backticks). Anchored to the full run of - # tildes on both sides so a 4-tilde open cannot match a 3-tilde close. - r"|(?~{3,})(?!~)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=tf)(?!~)[ \t]*(?:\r?\n|\Z)" + # the body itself contains backticks). Opener anchored to the full + # run of tildes; closer accepts >= opener length per CommonMark. + r"|(?~{3,})(?!~)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=tf)~*[ \t]*(?:\r?\n|\Z)" # Complete HTML page; doctype prefix is optional. r"|(?:" # Complete SVG document. @@ -158,6 +160,16 @@ _PLAN_LIST_FRAMING = re.compile( re.IGNORECASE, ) +# "Here's my plan" / "Here's my approach" are strong stand-alone plan +# signals: a possessive, first-person framing where the model is +# announcing what it WILL do. Treat the following numbered list as a +# plan regardless of the specific verbs each item uses, so stalls like +# ``Here's my plan: 1. Analyze 2. Draft`` still re-prompt. +_EXPLICIT_PLAN_HEADER = re.compile( + r"\bhere['’]?s (?:my |the |a )?(?:plan|approach)\b", + re.IGNORECASE, +) + _FENCE_LINE_RE = re.compile(r"^[ \t]*(?P`{3,}|~{3,})(?P[^\r\n]*)$") @@ -194,14 +206,19 @@ def _has_answer_artifact(text: str) -> bool: Code fences, complete HTML, and complete SVG count directly. A numbered list counts only when there is no plan framing, so stalls like ``Here's my plan:\\n1. search\\n2. summarise`` still re-prompt. - An unclosed fence disqualifies the numbered-list fallback so a list - INSIDE incomplete code does not look like a final answer. + An explicit ``Here's my plan`` / ``Here's my approach`` header is + also enough to flag the list as a plan, even when no narrow tool- + action verb appears in the items. An unclosed fence disqualifies + the numbered-list fallback so a list INSIDE incomplete code does + not look like a final answer. """ if _HAS_ANSWER_ARTIFACT.search(text): return True if _has_unclosed_code_fence(text): return False if _NUMBERED_LIST_ARTIFACT.search(text): + if _EXPLICIT_PLAN_HEADER.search(text): + return False return _PLAN_LIST_FRAMING.search(text) is None return False diff --git a/studio/backend/tests/test_llama_cpp_reprompt_guard.py b/studio/backend/tests/test_llama_cpp_reprompt_guard.py index 6ebff5778c..d712e1bbac 100644 --- a/studio/backend/tests/test_llama_cpp_reprompt_guard.py +++ b/studio/backend/tests/test_llama_cpp_reprompt_guard.py @@ -749,6 +749,69 @@ def test_artifact_regex_rejects_shorter_commonmark_closing_fence(): assert _would_reprompt(content), content +def test_artifact_regex_accepts_longer_commonmark_closing_fence(): + """CommonMark allows the closing fence to have MORE delimiters than + the opener. A 3-backtick opener with a 4-backtick close, or a + 3-tilde opener with a 4-tilde close, is still a complete artifact.""" + samples = [ + "First, let me show.\n```python\nprint('hi')\n````", + "First, let me show.\n````python\nprint('``` inside')\n`````", + "First, let me show.\n~~~python\nprint('hi')\n~~~~", + ] + for content in samples: + assert _has_answer_artifact(content), content + assert not _would_reprompt(content), content + + +def test_reprompts_on_explicit_plan_header_numbered_list(): + """``Here's my plan`` / ``Here's my approach`` is a strong stand-alone + plan signal. The following numbered list is the plan itself, not a + final answer, even when no narrow tool-action verb appears.""" + samples = [ + "Here's my plan:\n1. Analyze the request.\n2. Draft the answer.", + "Here's my plan:\n1. Create the Python file.\n2. Add the game loop.\n3. Test.", + "Here's my approach:\n1. Outline.\n2. Write.\n3. Review.", + "Here's the plan:\n1. Define the variables.\n2. Return the result.", + ] + for s in samples: + assert _would_reprompt(s), s + + +def test_reprompts_on_numbered_plan_with_python_tool_wording(): + """``use python (tool) to ...`` / ``use the python tool`` / ``use the + search tool`` in a numbered plan still re-prompts.""" + samples = [ + "Here's my plan:\n1. Use Python to calculate the answer.\n2. Return.", + "First, I'll do this:\n1. Use the python tool to parse the file.\n2. Summarize.", + "Here's my plan:\n1. Use the search tool.\n2. Summarize.", + ] + for s in samples: + assert _would_reprompt(s), s + + +def test_no_reprompt_on_lesson_plan_answer_without_explicit_header(): + """A final answer with a ``Plan:`` heading (no ``Here's my`` + possessive) and no tool framing must STILL count as an answer. + Common cases: lesson plan, workout plan, meal plan.""" + samples = [ + ( + "Plan:\n" + "1. Warm up for 5 minutes.\n" + "2. Run for 20 minutes.\n" + "3. Cool down with stretching." + ), + ( + "My weekly plan:\n" + "1. Monday: rest.\n" + "2. Tuesday: jog.\n" + "3. Wednesday: swim." + ), + ] + for content in samples: + assert _has_answer_artifact(content), content + assert not _would_reprompt(content), content + + def test_open_fence_with_inner_numbered_list_still_reprompts(): """A response that opens a code fence and emits numbered lines INSIDE must NOT count those lines as a completed numbered-list answer."""