diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 0968bf9b6d..1207395ed6 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -62,9 +62,17 @@ _INTENT_SIGNAL = re.compile( # appear frequently in direct answers / explanations. r"\b(i['\u2019](ll|m going to|m gonna)|i am (going to|gonna)|i will|i shall|let me|allow me)\b" r"|" - # Step/plan framing: "First ...", "Step 1:", "Here's my plan", bare - # "Plan:" / "Approach:" as the first line of a structured reply. - r"\b(?:first\b|step \d+:?|here['\u2019]?s (?:my |the |a )?(?:plan|approach)|(?:plan|approach):)" + # Step/plan framing: "First ...", "Step 1:", "Here's my plan", + # "Here is the plan", "Here are my steps". + r"\b(?:first\b|step \d+:?|" + r"here['\u2019]?s (?:my |the |a )?(?:plan|approach)|" + r"here (?:is|are) (?:my |the |a )?(?:plan|approach|steps))" + r"|" + # Bare "Plan:" / "Approach:" (optionally preceded by a determiner + # like "My" / "The" / "Our") anchored to start of line so direct + # answers like "Here is a lesson plan:" or "meal plan:" do not trip + # the re-prompt path. + r"(?:^|\r?\n)[ \t]*(?:(?:my|the|our|a|this|that)\s+)?(?:plan|approach):" r"|" # "Now I" / "Next I" patterns r"\b(?:now i|next i)\b" @@ -112,21 +120,25 @@ _NUMBERED_LIST_ARTIFACT = re.compile( ) # Markers that a numbered list is a plan (still re-promptable), not a -# final answer. Explicit "plan:" / "approach:" / "Here's my plan", OR -# intent phrasing followed shortly by a plan / tool-action verb. The +# final answer. The intent alternatives mirror _INTENT_SIGNAL above so +# every recognised intent phrase can disqualify a numbered list. The # apostrophe in ``i['’]ll`` is required (no ``?``) so the regex does not # accidentally match the word "ill". The verb set is intentionally # conservative: ambiguous verbs like "write", "create", "make", "build" # are omitted because real answer lists use them ("1. Write a poem", -# "1. Create directory"). +# "1. Create directory"). ``plan:`` / ``approach:`` is anchored to the +# start of a line so "lesson plan:" / "meal plan:" do not trip the guard. _PLAN_LIST_FRAMING = re.compile( - r"\b(?:here['’]?s (?:my |the |a )?(?:plan|approach)|step \d+|" - r"i['’]ll|i will|i am going to|let me|now i|next i)\b" + r"\b(?:here['’]?s (?:my |the |a )?(?:plan|approach)|" + r"here (?:is|are) (?:my |the |a )?(?:plan|approach|steps)|" + r"step \d+|" + r"i['’](?:ll|m going to|m gonna)|i am (?:going to|gonna)|" + r"i will|i shall|let me|allow me|now i|next i)\b" r"[\s\S]{0,80}" r"\b(?:search|look up|call|use|fetch|browse|run|execute|" r"check|find|open|verify|compare|summari[sz]e|think|respond|" r"answer|analy[sz]e|explore|outline|gather|query|reason)\b" - r"|\b(?:plan|approach):", + r"|(?:^|\r?\n)[ \t]*(?:(?:my|the|our|a|this|that)\s+)?(?:plan|approach):", re.IGNORECASE, ) diff --git a/studio/backend/tests/test_llama_cpp_reprompt_guard.py b/studio/backend/tests/test_llama_cpp_reprompt_guard.py index 22f5d11185..955dc423eb 100644 --- a/studio/backend/tests/test_llama_cpp_reprompt_guard.py +++ b/studio/backend/tests/test_llama_cpp_reprompt_guard.py @@ -376,6 +376,58 @@ def test_plan_framing_requires_apostrophe_in_ill(): assert got == expected, f"{content!r} expected reprompt={expected} got {got}" +def test_reprompts_on_all_intent_form_numbered_action_plans(): + """``_PLAN_LIST_FRAMING`` must mirror every intent form that + ``_INTENT_SIGNAL`` accepts so numbered action plans phrased with + ``Allow me``, ``I'm going to``, ``I'm gonna``, ``I am gonna``, + ``I shall``, ``Now I``, ``Next I`` also re-prompt instead of being + silently classified as completed answers.""" + samples = [ + "Allow me to do this:\n1. search the docs\n2. fetch the result", + "I'm going to do this:\n1. search the docs\n2. fetch the result", + "I'm gonna do this:\n1. search the docs\n2. fetch the result", + "I am gonna do this:\n1. search the docs\n2. fetch the result", + "I shall do this:\n1. search the docs\n2. fetch the result", + "Now I will do these:\n1. search\n2. summarise", + "Next I will do these:\n1. fetch\n2. compare", + ] + for s in samples: + assert _would_reprompt(s), s + + +def test_plan_colon_intent_is_line_anchored(): + """``Plan:`` / ``Approach:`` only counts as an intent marker when it + is at the start of a line. Without this anchor, normal direct + answers containing phrases like ``lesson plan:``, ``meal plan:``, + ``migration plan:``, or ``My approach:`` would trigger the + re-prompt path and risk wiping a valid response.""" + # These mid-line "plan:" / "approach:" mentions are NOT intent signals. + # The qualifier before "plan" is a content noun ("lesson", "meal", + # "migration") rather than a generic determiner ("my", "the", ...). + direct_answers = [ + "Here is a lesson plan:\n1. Warm-up\n2. Group practice\n3. Assessment", + "I prepared a meal plan: rice, beans, eggs.", + "Quick approach: top-down then bottom-up.", + ] + for s in direct_answers: + assert not _INTENT_SIGNAL.search(s), s + assert not _would_reprompt(s), s + # "Plan:" / "Approach:" with optional generic determiner at the start + # of a line IS an intent signal. + plan_starts = [ + "Plan:\n1. search\n2. summarise", + "Approach:\n1. fetch\n2. compare", + " Plan:\n1. think\n2. respond", # leading indent OK + "Lorem ipsum\nPlan:\n1. step\n2. step", # plan: on a later line + "My plan:\n1. search\n2. summarise", + "The plan:\n1. look up\n2. compare", + "Our approach:\n1. fetch\n2. verify", + ] + for s in plan_starts: + assert _INTENT_SIGNAL.search(s), s + assert _would_reprompt(s), s + + # ── Cross-platform line endings ────────────────────────────────────