diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index c2652a683b..75aeaf3859 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -198,7 +198,11 @@ _DIRECT_NUMBERED_PLAN_FRAMING = re.compile( r"[^\r\n]{0,160}" r"\b(?:open|read|search|look (?:this |that |it |them )?up|browse|" r"google|find|check|verify|compare|review|inspect|examine|" - r"do (?:this|these|the following|it)|proceed|start|begin|" + r"do (?:this|these|the following|it)|" + r"take (?:these|the following) steps|" + r"follow (?:these|the following) steps|" + r"perform (?:these|the following) actions|" + r"proceed|start|begin|" r"create|build|implement|set up|add|calculate|compute|analy[sz]e|" r"parse|load|run|execute|test)\b" r"|" @@ -224,13 +228,12 @@ _FENCE_RUN_RE = re.compile( def _has_unclosed_code_fence(text: str) -> bool: """True if ``text`` contains a code fence whose closer is missing. - A complete fence answer is already caught by _HAS_ANSWER_ARTIFACT. - This helper exists so that an OPEN fence (model still streaming - code, or stream cut short) does not let an embedded numbered list - inside the fence body masquerade as a final answer. The scan is - done per line and uses ``search`` (not ``match``) so an inline - opener such as ``First, let me write it. \\`\\`\\`python`` is also - tracked. + Each line is scanned so inline openers like ``First. \\`\\`\\`python`` + are tracked. Prose mentions such as ``Use \\`\\`\\` to start a + fence.`` are filtered out by requiring the trailing characters + after the fence run to look like a CommonMark info string: empty, + or starting with a non-space character (so prose with a leading + space disqualifies the run). """ active_char: Optional[str] = None active_len = 0 @@ -239,8 +242,13 @@ def _has_unclosed_code_fence(text: str) -> bool: if not m: continue fence = m.group("backticks") or m.group("tildes") - trailing = line[m.end() :].strip() + raw_trailing = line[m.end():] + trailing = raw_trailing.strip() ch = fence[0] + # Prose mention guard: a real fence line never has a space + # immediately after the delimiters followed by sentence text. + if raw_trailing and raw_trailing[0] == " " and trailing: + continue if active_char is None: active_char = ch active_len = len(fence) diff --git a/studio/backend/tests/test_llama_cpp_reprompt_guard.py b/studio/backend/tests/test_llama_cpp_reprompt_guard.py index b0a64bf0bc..1b408da27f 100644 --- a/studio/backend/tests/test_llama_cpp_reprompt_guard.py +++ b/studio/backend/tests/test_llama_cpp_reprompt_guard.py @@ -985,6 +985,43 @@ def test_no_reprompt_on_code_fence_containing_markup_literal(): assert not _would_reprompt(content), content +def test_reprompts_on_take_or_follow_steps_numbered_plan(): + """``I'll take these steps:`` / ``I will follow these steps:`` + + numbered list of work items is a plan stall.""" + samples = [ + ( + "I'll take these steps:\n" + "1. Open the URL.\n" + "2. Read the page.\n" + "3. Summarize the answer." + ), + ( + "I will follow these steps:\n" + "1. Open the current docs.\n" + "2. Read the relevant section.\n" + "3. Answer." + ), + ] + for content in samples: + assert not _has_answer_artifact(content), content + assert _would_reprompt(content), content + + +def test_no_reprompt_on_prose_mention_of_triple_backticks_after_code(): + """Closed code fence followed by prose that describes triple- + backtick syntax (with leading space after the ticks) must NOT be + treated as an unclosed fence.""" + content = ( + "Here is the snippet:\n" + "```python\n" + "print(1)\n" + "```\n" + "Use ``` to start a markdown code fence in your reply." + ) + assert _has_answer_artifact(content) + assert not _would_reprompt(content) + + def test_reprompts_on_direct_first_person_read_check_open_plan(): """Direct first-person intent + open/read/check/review/inspect verbs + numbered list is a tool stall. The broader verb set applies to