From 64ae2ac4c583e2ccf9d92a193736889f2345f20e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 14:36:04 +0000 Subject: [PATCH] Studio: sync re-prompt guard intent forms and tighten Plan: anchor Two more gaps surfaced by another reviewer sweep on the previous commit: 1. _PLAN_LIST_FRAMING was missing several intent forms that _INTENT_SIGNAL accepts, so numbered tool-action plans phrased with "Allow me", "I'm going to", "I'm gonna", "I am gonna", or "I shall" were silently classified as completed answers and skipped the tool-call re-prompt. Mirror the full intent set from _INTENT_SIGNAL so the two regexes stay in lock-step. 2. Bare \b(?:plan|approach): in _INTENT_SIGNAL / _PLAN_LIST_FRAMING matched any in-text occurrence of "plan:" / "approach:", including "lesson plan:" / "meal plan:" / "migration plan:". A direct answer like "Here is a lesson plan:\n1. Warm-up\n2. Group practice" would trip _INTENT_SIGNAL and risk wiping the response. Anchor the colon marker to start of line and only allow generic determiners (my, the, our, a, this, that) between the line start and the keyword. 3. Add "Here is the plan" / "Here are my steps" to both _INTENT_SIGNAL and _PLAN_LIST_FRAMING so non-apostrophe phrasings of the same framing pattern are caught. Added regression tests covering every intent form against a numbered action plan, and a line-anchor test that distinguishes generic plan framings ("My plan:", "The approach:") from content noun phrases ("lesson plan:", "meal plan:"). --- studio/backend/core/inference/llama_cpp.py | 30 +++++++---- .../tests/test_llama_cpp_reprompt_guard.py | 52 +++++++++++++++++++ 2 files changed, 73 insertions(+), 9 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 0968bf9b6d..1207395ed6 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -62,9 +62,17 @@ _INTENT_SIGNAL = re.compile( # appear frequently in direct answers / explanations. r"\b(i['\u2019](ll|m going to|m gonna)|i am (going to|gonna)|i will|i shall|let me|allow me)\b" r"|" - # Step/plan framing: "First ...", "Step 1:", "Here's my plan", bare - # "Plan:" / "Approach:" as the first line of a structured reply. - r"\b(?:first\b|step \d+:?|here['\u2019]?s (?:my |the |a )?(?:plan|approach)|(?:plan|approach):)" + # Step/plan framing: "First ...", "Step 1:", "Here's my plan", + # "Here is the plan", "Here are my steps". + r"\b(?:first\b|step \d+:?|" + r"here['\u2019]?s (?:my |the |a )?(?:plan|approach)|" + r"here (?:is|are) (?:my |the |a )?(?:plan|approach|steps))" + r"|" + # Bare "Plan:" / "Approach:" (optionally preceded by a determiner + # like "My" / "The" / "Our") anchored to start of line so direct + # answers like "Here is a lesson plan:" or "meal plan:" do not trip + # the re-prompt path. + r"(?:^|\r?\n)[ \t]*(?:(?:my|the|our|a|this|that)\s+)?(?:plan|approach):" r"|" # "Now I" / "Next I" patterns r"\b(?:now i|next i)\b" @@ -112,21 +120,25 @@ _NUMBERED_LIST_ARTIFACT = re.compile( ) # Markers that a numbered list is a plan (still re-promptable), not a -# final answer. Explicit "plan:" / "approach:" / "Here's my plan", OR -# intent phrasing followed shortly by a plan / tool-action verb. The +# final answer. The intent alternatives mirror _INTENT_SIGNAL above so +# every recognised intent phrase can disqualify a numbered list. The # apostrophe in ``i['’]ll`` is required (no ``?``) so the regex does not # accidentally match the word "ill". The verb set is intentionally # conservative: ambiguous verbs like "write", "create", "make", "build" # are omitted because real answer lists use them ("1. Write a poem", -# "1. Create directory"). +# "1. Create directory"). ``plan:`` / ``approach:`` is anchored to the +# start of a line so "lesson plan:" / "meal plan:" do not trip the guard. _PLAN_LIST_FRAMING = re.compile( - r"\b(?:here['’]?s (?:my |the |a )?(?:plan|approach)|step \d+|" - r"i['’]ll|i will|i am going to|let me|now i|next i)\b" + r"\b(?:here['’]?s (?:my |the |a )?(?:plan|approach)|" + r"here (?:is|are) (?:my |the |a )?(?:plan|approach|steps)|" + r"step \d+|" + r"i['’](?:ll|m going to|m gonna)|i am (?:going to|gonna)|" + r"i will|i shall|let me|allow me|now i|next i)\b" r"[\s\S]{0,80}" r"\b(?:search|look up|call|use|fetch|browse|run|execute|" r"check|find|open|verify|compare|summari[sz]e|think|respond|" r"answer|analy[sz]e|explore|outline|gather|query|reason)\b" - r"|\b(?:plan|approach):", + r"|(?:^|\r?\n)[ \t]*(?:(?:my|the|our|a|this|that)\s+)?(?:plan|approach):", re.IGNORECASE, ) diff --git a/studio/backend/tests/test_llama_cpp_reprompt_guard.py b/studio/backend/tests/test_llama_cpp_reprompt_guard.py index 22f5d11185..955dc423eb 100644 --- a/studio/backend/tests/test_llama_cpp_reprompt_guard.py +++ b/studio/backend/tests/test_llama_cpp_reprompt_guard.py @@ -376,6 +376,58 @@ def test_plan_framing_requires_apostrophe_in_ill(): assert got == expected, f"{content!r} expected reprompt={expected} got {got}" +def test_reprompts_on_all_intent_form_numbered_action_plans(): + """``_PLAN_LIST_FRAMING`` must mirror every intent form that + ``_INTENT_SIGNAL`` accepts so numbered action plans phrased with + ``Allow me``, ``I'm going to``, ``I'm gonna``, ``I am gonna``, + ``I shall``, ``Now I``, ``Next I`` also re-prompt instead of being + silently classified as completed answers.""" + samples = [ + "Allow me to do this:\n1. search the docs\n2. fetch the result", + "I'm going to do this:\n1. search the docs\n2. fetch the result", + "I'm gonna do this:\n1. search the docs\n2. fetch the result", + "I am gonna do this:\n1. search the docs\n2. fetch the result", + "I shall do this:\n1. search the docs\n2. fetch the result", + "Now I will do these:\n1. search\n2. summarise", + "Next I will do these:\n1. fetch\n2. compare", + ] + for s in samples: + assert _would_reprompt(s), s + + +def test_plan_colon_intent_is_line_anchored(): + """``Plan:`` / ``Approach:`` only counts as an intent marker when it + is at the start of a line. Without this anchor, normal direct + answers containing phrases like ``lesson plan:``, ``meal plan:``, + ``migration plan:``, or ``My approach:`` would trigger the + re-prompt path and risk wiping a valid response.""" + # These mid-line "plan:" / "approach:" mentions are NOT intent signals. + # The qualifier before "plan" is a content noun ("lesson", "meal", + # "migration") rather than a generic determiner ("my", "the", ...). + direct_answers = [ + "Here is a lesson plan:\n1. Warm-up\n2. Group practice\n3. Assessment", + "I prepared a meal plan: rice, beans, eggs.", + "Quick approach: top-down then bottom-up.", + ] + for s in direct_answers: + assert not _INTENT_SIGNAL.search(s), s + assert not _would_reprompt(s), s + # "Plan:" / "Approach:" with optional generic determiner at the start + # of a line IS an intent signal. + plan_starts = [ + "Plan:\n1. search\n2. summarise", + "Approach:\n1. fetch\n2. compare", + " Plan:\n1. think\n2. respond", # leading indent OK + "Lorem ipsum\nPlan:\n1. step\n2. step", # plan: on a later line + "My plan:\n1. search\n2. summarise", + "The plan:\n1. look up\n2. compare", + "Our approach:\n1. fetch\n2. verify", + ] + for s in plan_starts: + assert _INTENT_SIGNAL.search(s), s + assert _would_reprompt(s), s + + # ── Cross-platform line endings ────────────────────────────────────