diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 64abdae139..ab4a078d92 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -180,9 +180,11 @@ _EXPLICIT_PLAN_HEADER = re.compile( # poem" is not misclassified as a stall. _DIRECT_NUMBERED_PLAN_FRAMING = re.compile( r"\b(?:i['’](?:ll|m going to|m gonna)|i am (?:going to|gonna)|" - r"i will|i shall|let me|allow me|now i|next i)\b" + r"i will|i shall|let me|allow me|now i|next i|" + r"first|step \d+:?)\b" r"[^\r\n]{0,160}" r"\b(?:do (?:this|these|the following|it)|" + r"look (?:this |that |it |them )?up|" r"proceed|start|begin|" r"create|build|implement|set up|add|" r"calculate|compute|analy[sz]e|parse|load|run|execute|test)\b" @@ -192,7 +194,9 @@ _DIRECT_NUMBERED_PLAN_FRAMING = re.compile( ) -_FENCE_LINE_RE = re.compile(r"^[ \t]*(?P`{3,}|~{3,})(?P[^\r\n]*)$") +_FENCE_RUN_RE = re.compile( + r"(?`{3,})(?!`)|(?~{3,})(?!~)" +) def _has_unclosed_code_fence(text: str) -> bool: @@ -201,16 +205,19 @@ def _has_unclosed_code_fence(text: str) -> bool: A complete fence answer is already caught by _HAS_ANSWER_ARTIFACT. This helper exists so that an OPEN fence (model still streaming code, or stream cut short) does not let an embedded numbered list - inside the fence body masquerade as a final answer. + inside the fence body masquerade as a final answer. The scan is + done per line and uses ``search`` (not ``match``) so an inline + opener such as ``First, let me write it. \\`\\`\\`python`` is also + tracked. """ active_char: Optional[str] = None active_len = 0 for line in text.splitlines(): - m = _FENCE_LINE_RE.match(line) + m = _FENCE_RUN_RE.search(line) if not m: continue - fence = m.group("fence") - trailing = m.group("trailing").strip() + fence = m.group("backticks") or m.group("tildes") + trailing = line[m.end():].strip() ch = fence[0] if active_char is None: active_char = ch diff --git a/studio/backend/tests/test_llama_cpp_reprompt_guard.py b/studio/backend/tests/test_llama_cpp_reprompt_guard.py index e54b8de05d..94a368a7ea 100644 --- a/studio/backend/tests/test_llama_cpp_reprompt_guard.py +++ b/studio/backend/tests/test_llama_cpp_reprompt_guard.py @@ -869,6 +869,69 @@ def test_no_reprompt_on_let_me_explain_numbered_answer(): assert not _would_reprompt(content), content +def test_same_line_open_fence_with_numbered_body_still_reprompts(): + """An OPEN code fence on the same line as preceding prose ("First, + let me write it. ``\\u00e0``text\\n...") still gates the numbered-list + fallback. The unclosed-fence helper now uses ``search`` so inline + openers are tracked, not just openers at column 0.""" + content = ( + "First, let me write it. ```text\n" + "1. Install dependencies\n" + "2. Run the app" + ) + assert not _has_answer_artifact(content) + assert _would_reprompt(content) + + +def test_reprompts_on_first_step_numbered_compute_plan(): + """Bare ``First, [verb]`` / ``Step N: [verb]`` followed by a numbered + list is a plan stall when the verb implies compute / tool work + (analyze, parse, calculate, create, etc.). Distinct from + ``First, use binary search:`` (verb ``use`` not in whitelist).""" + samples = [ + ( + "First, analyze the uploaded CSV:\n" + "1. Load the rows.\n" + "2. Compute the average revenue." + ), + ( + "First, parse the pasted JSON:\n" + "1. Load the object.\n" + "2. Calculate the total." + ), + ( + "First, create the Python game:\n" + "1. Set up pygame.\n" + "2. Add the game loop." + ), + ( + "Step 1: analyze the uploaded CSV:\n" + "1. Load rows.\n" + "2. Compute the total." + ), + ( + "I'll look that up:\n" + "1. Search the docs.\n" + "2. Summarize the result." + ), + ] + for content in samples: + assert _would_reprompt(content), content + + +def test_no_reprompt_on_first_use_binary_search_answer(): + """``First, use binary search:`` is an ordinary algorithm answer. + ``use`` is not in the direct-numbered-plan verb whitelist so the + following list stays an answer.""" + content = ( + "First, use binary search:\n" + "1. Search the left half.\n" + "2. Search the right half." + ) + assert _has_answer_artifact(content) + assert not _would_reprompt(content) + + def test_reprompts_when_later_fence_is_open_after_closed_fence(): """A response with a complete code fence followed by a SECOND, unclosed fence is still mid-stream and must re-prompt. The