From a6f6022bd0fd418225b231bda38c82b4bbe22c43 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 14:56:04 +0000 Subject: [PATCH] Studio: require nearby action verb for Plan: / "Here is the plan" intents Reviewer round 8 surfaced a real false positive in the previous commit: a final answer naturally titled "Plan:" / "My plan:" / "Approach:" with numbered content items now slipped through _INTENT_SIGNAL and got wiped by the synthetic STOP turn. Examples: Plan: 1. Warm-up: Students review fractions. 2. Group practice. 3. Assessment. My plan: 1. Breakfast: oatmeal and fruit. 2. Lunch: rice bowl. 3. Dinner: lentil soup. Here is the plan you asked for. It is two pages long. Add a lookahead requiring one of the conservative re-prompt action verbs (search / fetch / verify / look up / call / compare / think / respond / etc.) to appear within 120 chars after the "Plan:" / "Approach:" / "Here is the plan" / "Here are my steps" marker. Plan stalls whose items are tool actions ("Plan:\n1. search the docs\n2. summarise the result") still match and re-prompt; prose plans whose items are content do not. Also mirror "first" in _PLAN_LIST_FRAMING so numbered action plans that start with "First" stay disqualified even after the helper enters the numbered-list branch. Factor the action-verb set out as _REPROMPT_ACTION_VERBS so both regexes share one source of truth. Six new regression samples: three lesson / meal / weather plans that must NOT wipe, three action-plan headers that must re-prompt, three prose "Here is the plan" answers that must not wipe. --- studio/backend/core/inference/llama_cpp.py | 51 ++++++++++++------- .../tests/test_llama_cpp_reprompt_guard.py | 50 ++++++++++++++++-- 2 files changed, 79 insertions(+), 22 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 2650168efd..5c44c2062b 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -52,6 +52,20 @@ logger = get_logger(__name__) # ── Pre-compiled patterns for plan-without-action re-prompt ── +# Re-prompt-action verbs. Used both as a nearby-verb lookahead for the +# new ``Plan:`` / ``Here is the plan`` intents (so prose final answers +# such as ``Plan:\n1. Warm-up\n2. Group practice`` or "Here is the plan +# you asked for" do not wipe) and as the plan-list disqualifier verb +# set in _PLAN_LIST_FRAMING below. Conservative on purpose: ambiguous +# verbs like ``write``, ``create``, ``make``, ``build``, ``do``, +# ``handle`` are deliberately excluded because real answer lists use +# them ("1. Write a poem", "1. Create directory"). +_REPROMPT_ACTION_VERBS = ( + r"search|look up|call|use|fetch|browse|run|execute|" + r"check|find|open|verify|compare|summari[sz]e|think|respond|" + r"answer|analy[sz]e|explore|outline|gather|query|reason" +) + # Forward-looking intent signals that indicate the model is # describing what it *will* do rather than giving a final answer. _INTENT_SIGNAL = re.compile( @@ -62,18 +76,22 @@ _INTENT_SIGNAL = re.compile( # appear frequently in direct answers / explanations. r"\b(i['\u2019](ll|m going to|m gonna)|i am (going to|gonna)|i will|i shall|let me|allow me)\b" r"|" - # Step/plan framing: "First ...", "Step 1:", "Here's my plan", - # "Here is the plan", "Here are my steps". - r"\b(?:first\b|step \d+:?|" - r"here['\u2019]?s (?:my |the |a )?(?:plan|approach)|" - r"here (?:is|are) (?:my |the |a )?(?:plan|approach|steps))" + # Step/plan framing: "First ...", "Step 1:", "Here's my plan". + r"\b(?:first\b|step \d+:?|here['\u2019]?s (?:my |the |a )?(?:plan|approach))" + r"|" + # "Here is the plan" / "Here are my steps" framings. Require an + # action verb within 120 chars so prose answers like "Here is the + # plan you asked for" do not match. + r"\bhere (?:is|are) (?:my |the |a )?(?:plan|approach|steps)\b" + rf"(?=[\s\S]{{0,120}}\b(?:{_REPROMPT_ACTION_VERBS})\b)" r"|" # Bare "Plan:" / "Approach:" (optionally preceded by a determiner # like "My" / "The" / "Our") anchored to start of line AND followed - # by a newline. Inline forms like "Your current Plan: Pro includes - # local chats" or "The plan: $10/month" must NOT trip the re-prompt - # path; only header-style framings ("Plan:\n1. ...") count. + # by a newline AND followed within 120 chars by an action verb so + # final answers shaped like "Plan:\n1. Warm-up\n2. Group practice" + # or "My plan:\n1. Breakfast\n2. Lunch" do NOT wipe. r"(?:^|\r?\n)[ \t]*(?:(?:my|the|our|a|this|that)\s+)?(?:plan|approach):[ \t]*(?=\r?\n)" + rf"(?=[\s\S]{{0,120}}\b(?:{_REPROMPT_ACTION_VERBS})\b)" r"|" # "Now I" / "Next I" patterns r"\b(?:now i|next i)\b" @@ -124,22 +142,19 @@ _NUMBERED_LIST_ARTIFACT = re.compile( # final answer. The intent alternatives mirror _INTENT_SIGNAL above so # every recognised intent phrase can disqualify a numbered list. The # apostrophe in ``i['’]ll`` is required (no ``?``) so the regex does not -# accidentally match the word "ill". The verb set is intentionally -# conservative: ambiguous verbs like "write", "create", "make", "build" -# are omitted because real answer lists use them ("1. Write a poem", -# "1. Create directory"). ``plan:`` / ``approach:`` is anchored to the -# start of a line so "lesson plan:" / "meal plan:" do not trip the guard. +# accidentally match the word "ill". Both branches require an action +# verb nearby so plan-style answer headers ("Plan:\n1. Warm-up\n2. +# Group practice") are NOT treated as plans and stay artifacts. _PLAN_LIST_FRAMING = re.compile( r"\b(?:here['’]?s (?:my |the |a )?(?:plan|approach)|" r"here (?:is|are) (?:my |the |a )?(?:plan|approach|steps)|" - r"step \d+|" + r"step \d+|first|" r"i['’](?:ll|m going to|m gonna)|i am (?:going to|gonna)|" r"i will|i shall|let me|allow me|now i|next i)\b" r"[\s\S]{0,80}" - r"\b(?:search|look up|call|use|fetch|browse|run|execute|" - r"check|find|open|verify|compare|summari[sz]e|think|respond|" - r"answer|analy[sz]e|explore|outline|gather|query|reason)\b" - r"|(?:^|\r?\n)[ \t]*(?:(?:my|the|our|a|this|that)\s+)?(?:plan|approach):[ \t]*(?=\r?\n)", + rf"\b(?:{_REPROMPT_ACTION_VERBS})\b" + r"|(?:^|\r?\n)[ \t]*(?:(?:my|the|our|a|this|that)\s+)?(?:plan|approach):" + rf"[\s\S]{{0,120}}\b(?:{_REPROMPT_ACTION_VERBS})\b", re.IGNORECASE, ) diff --git a/studio/backend/tests/test_llama_cpp_reprompt_guard.py b/studio/backend/tests/test_llama_cpp_reprompt_guard.py index 8923a8294c..7f531c1115 100644 --- a/studio/backend/tests/test_llama_cpp_reprompt_guard.py +++ b/studio/backend/tests/test_llama_cpp_reprompt_guard.py @@ -396,6 +396,46 @@ def test_reprompts_on_all_intent_form_numbered_action_plans(): assert _would_reprompt(s), s +def test_no_reprompt_on_plan_titled_final_answer_without_actions(): + """A final answer naturally titled ``Plan:`` / ``My plan:`` / + ``Approach:`` whose numbered items are content (not action verbs) + must NOT wipe. The action-verb lookahead on the Plan: intent + branch is what filters lesson plans, meal plans, dinner plans, + and similar from being misclassified as tool-action stalls.""" + samples = [ + "Plan:\n1. Warm-up: Students review fractions.\n2. Group practice.\n3. Assessment.", + "My plan:\n1. Breakfast: oatmeal and fruit.\n2. Lunch: rice bowl.\n3. Dinner: lentil soup.", + "The plan:\n1. Bring umbrellas.\n2. Pack snacks.\n3. Drive carefully.", + ] + for s in samples: + assert not _would_reprompt(s), s + + +def test_reprompts_on_plan_titled_action_stall(): + """A ``Plan:`` / ``Approach:`` header whose items DO contain action + verbs (search / fetch / verify / ...) still re-prompts.""" + samples = [ + "Plan:\n1. search the docs\n2. summarise the result", + "My plan:\n1. fetch the data\n2. verify the rows", + "The approach:\n1. look up the value\n2. compare versions", + ] + for s in samples: + assert _would_reprompt(s), s + + +def test_no_reprompt_on_here_is_the_plan_prose_answer(): + """``Here is the plan you asked for. ...`` and similar prose + answers without action verbs must NOT wipe. The action-verb + lookahead on the ``Here is the plan`` intent branch filters them.""" + samples = [ + "Here is the plan you asked for. It is two pages long and covers Q4 goals.", + "Here are my steps in plain English. Step one is patience.", + "Here is a plan for the dinner party. Welcome, eat, dance.", + ] + for s in samples: + assert not _would_reprompt(s), s + + def test_plan_colon_intent_is_line_anchored(): """``Plan:`` / ``Approach:`` only counts as an intent marker when it is at the start of a line. Without this anchor, normal direct @@ -420,14 +460,16 @@ def test_plan_colon_intent_is_line_anchored(): assert not _INTENT_SIGNAL.search(s), s assert not _would_reprompt(s), s # "Plan:" / "Approach:" with optional generic determiner at the start - # of a line, FOLLOWED BY A NEWLINE, IS an intent signal. The newline - # requirement is what filters inline product/answer text such as - # "The plan: Pro is $10/month" out of the intent path. + # of a line, followed by a newline AND followed by an action verb + # within 120 chars, IS an intent signal. The newline requirement + # filters inline product/answer text such as "The plan: Pro is + # $10/month"; the action-verb requirement filters real prose answers + # such as "Plan:\n1. Warm-up\n2. Group practice". plan_starts = [ "Plan:\n1. search\n2. summarise", "Approach:\n1. fetch\n2. compare", " Plan:\n1. think\n2. respond", # leading indent OK - "Lorem ipsum\nPlan:\n1. step\n2. step", # plan: on a later line + "Lorem ipsum\nPlan:\n1. search\n2. fetch", # plan: on a later line "My plan:\n1. search\n2. summarise", "The plan:\n1. look up\n2. compare", "Our approach:\n1. fetch\n2. verify",