From 3dc26e7acf9dc7e876dd312ed7b3d380fd624486 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 14:41:44 +0000 Subject: [PATCH] Studio: require newline after Plan: / Approach: header so inline product text does not re-prompt After narrowing the colon marker to lines starting with a generic determiner ("My plan:" / "The approach:" / ...), inline product or pricing answers like "Your current Plan: Pro includes local chats", "The plan: Basic is free, Pro is $10/month", or "My plan: use dynamic programming" still slipped into the re-prompt path and could wipe a valid answer. Add a lookahead requiring a newline (with optional trailing horizontal whitespace) after the colon, so only header-style framings like "Plan:\n1. search\n2. summarise" or "My approach:\n1. fetch" count. Inline "Plan: " is now treated as ordinary prose. Add eight regression samples (lesson plan, meal plan, marketing plan, pricing plan, recommended approach, migration plan, dynamic-programming plan, currently active plan) all of which previously re-prompted under the unanchored matcher and now correctly do not. --- studio/backend/core/inference/llama_cpp.py | 11 +++++----- .../tests/test_llama_cpp_reprompt_guard.py | 21 +++++++++++++------ 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 1207395ed6..2650168efd 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -69,10 +69,11 @@ _INTENT_SIGNAL = re.compile( r"here (?:is|are) (?:my |the |a )?(?:plan|approach|steps))" r"|" # Bare "Plan:" / "Approach:" (optionally preceded by a determiner - # like "My" / "The" / "Our") anchored to start of line so direct - # answers like "Here is a lesson plan:" or "meal plan:" do not trip - # the re-prompt path. - r"(?:^|\r?\n)[ \t]*(?:(?:my|the|our|a|this|that)\s+)?(?:plan|approach):" + # like "My" / "The" / "Our") anchored to start of line AND followed + # by a newline. Inline forms like "Your current Plan: Pro includes + # local chats" or "The plan: $10/month" must NOT trip the re-prompt + # path; only header-style framings ("Plan:\n1. ...") count. + r"(?:^|\r?\n)[ \t]*(?:(?:my|the|our|a|this|that)\s+)?(?:plan|approach):[ \t]*(?=\r?\n)" r"|" # "Now I" / "Next I" patterns r"\b(?:now i|next i)\b" @@ -138,7 +139,7 @@ _PLAN_LIST_FRAMING = re.compile( r"\b(?:search|look up|call|use|fetch|browse|run|execute|" r"check|find|open|verify|compare|summari[sz]e|think|respond|" r"answer|analy[sz]e|explore|outline|gather|query|reason)\b" - r"|(?:^|\r?\n)[ \t]*(?:(?:my|the|our|a|this|that)\s+)?(?:plan|approach):", + r"|(?:^|\r?\n)[ \t]*(?:(?:my|the|our|a|this|that)\s+)?(?:plan|approach):[ \t]*(?=\r?\n)", re.IGNORECASE, ) diff --git a/studio/backend/tests/test_llama_cpp_reprompt_guard.py b/studio/backend/tests/test_llama_cpp_reprompt_guard.py index 955dc423eb..8923a8294c 100644 --- a/studio/backend/tests/test_llama_cpp_reprompt_guard.py +++ b/studio/backend/tests/test_llama_cpp_reprompt_guard.py @@ -335,13 +335,14 @@ def test_reprompts_on_incomplete_html_intent(): def test_reprompts_on_plan_colon_intent(): - """Bare ``Plan:`` / ``Approach:`` at the start of a structured reply - is now an intent signal so the plan stall re-prompts. Pre-fix the - response slipped past ``_INTENT_SIGNAL`` entirely.""" + """Bare ``Plan:`` / ``Approach:`` followed by a newline at the start + of a structured reply is now an intent signal so the plan stall + re-prompts. Inline ``Plan: `` (no newline) is NOT an intent + signal because that shape is common in marketing / product answers + such as ``Plan: Pro is $10/month``.""" samples = [ "Plan:\n1. search the docs\n2. summarise", "Approach:\n1. fetch the data\n2. compare", - "Plan: search the docs then summarise", ] for s in samples: assert _INTENT_SIGNAL.search(s), s @@ -403,17 +404,25 @@ def test_plan_colon_intent_is_line_anchored(): re-prompt path and risk wiping a valid response.""" # These mid-line "plan:" / "approach:" mentions are NOT intent signals. # The qualifier before "plan" is a content noun ("lesson", "meal", - # "migration") rather than a generic determiner ("my", "the", ...). + # "migration") rather than a generic determiner, OR the colon is + # followed by inline content instead of a newline-anchored header. direct_answers = [ "Here is a lesson plan:\n1. Warm-up\n2. Group practice\n3. Assessment", "I prepared a meal plan: rice, beans, eggs.", "Quick approach: top-down then bottom-up.", + "Your current Plan: Pro includes local chats.", + "The plan: Basic is free, Pro is $10/month, Enterprise is custom.", + "My plan: use dynamic programming with memoisation.", + "Recommended approach: use the Python SDK for uploads.", + "The migration plan: backup, run, verify all in one window.", ] for s in direct_answers: assert not _INTENT_SIGNAL.search(s), s assert not _would_reprompt(s), s # "Plan:" / "Approach:" with optional generic determiner at the start - # of a line IS an intent signal. + # of a line, FOLLOWED BY A NEWLINE, IS an intent signal. The newline + # requirement is what filters inline product/answer text such as + # "The plan: Pro is $10/month" out of the intent path. plan_starts = [ "Plan:\n1. search\n2. summarise", "Approach:\n1. fetch\n2. compare",