Studio: sync re-prompt guard intent forms and tighten Plan: anchor
Two more gaps surfaced by another reviewer sweep on the previous commit:
1. _PLAN_LIST_FRAMING was missing several intent forms that
_INTENT_SIGNAL accepts, so numbered tool-action plans phrased with
"Allow me", "I'm going to", "I'm gonna", "I am gonna", or "I shall"
were silently classified as completed answers and skipped the
tool-call re-prompt. Mirror the full intent set from _INTENT_SIGNAL
so the two regexes stay in lock-step.
2. Bare \b(?:plan|approach): in _INTENT_SIGNAL / _PLAN_LIST_FRAMING
matched any in-text occurrence of "plan:" / "approach:", including
"lesson plan:" / "meal plan:" / "migration plan:". A direct answer
like "Here is a lesson plan:\n1. Warm-up\n2. Group practice" would
trip _INTENT_SIGNAL and risk wiping the response. Anchor the colon
marker to start of line and only allow generic determiners (my, the,
our, a, this, that) between the line start and the keyword.
3. Add "Here is the plan" / "Here are my steps" to both _INTENT_SIGNAL
and _PLAN_LIST_FRAMING so non-apostrophe phrasings of the same
framing pattern are caught.
Added regression tests covering every intent form against a numbered
action plan, and a line-anchor test that distinguishes generic plan
framings ("My plan:", "The approach:") from content noun phrases
("lesson plan:", "meal plan:").
This commit is contained in:
parent
a2ab9895b1
commit
64ae2ac4c5
2 changed files with 73 additions and 9 deletions
|
|
@ -62,9 +62,17 @@ _INTENT_SIGNAL = re.compile(
|
|||
# appear frequently in direct answers / explanations.
|
||||
r"\b(i['\u2019](ll|m going to|m gonna)|i am (going to|gonna)|i will|i shall|let me|allow me)\b"
|
||||
r"|"
|
||||
# Step/plan framing: "First ...", "Step 1:", "Here's my plan", bare
|
||||
# "Plan:" / "Approach:" as the first line of a structured reply.
|
||||
r"\b(?:first\b|step \d+:?|here['\u2019]?s (?:my |the |a )?(?:plan|approach)|(?:plan|approach):)"
|
||||
# Step/plan framing: "First ...", "Step 1:", "Here's my plan",
|
||||
# "Here is the plan", "Here are my steps".
|
||||
r"\b(?:first\b|step \d+:?|"
|
||||
r"here['\u2019]?s (?:my |the |a )?(?:plan|approach)|"
|
||||
r"here (?:is|are) (?:my |the |a )?(?:plan|approach|steps))"
|
||||
r"|"
|
||||
# Bare "Plan:" / "Approach:" (optionally preceded by a determiner
|
||||
# like "My" / "The" / "Our") anchored to start of line so direct
|
||||
# answers like "Here is a lesson plan:" or "meal plan:" do not trip
|
||||
# the re-prompt path.
|
||||
r"(?:^|\r?\n)[ \t]*(?:(?:my|the|our|a|this|that)\s+)?(?:plan|approach):"
|
||||
r"|"
|
||||
# "Now I" / "Next I" patterns
|
||||
r"\b(?:now i|next i)\b"
|
||||
|
|
@ -112,21 +120,25 @@ _NUMBERED_LIST_ARTIFACT = re.compile(
|
|||
)
|
||||
|
||||
# Markers that a numbered list is a plan (still re-promptable), not a
|
||||
# final answer. Explicit "plan:" / "approach:" / "Here's my plan", OR
|
||||
# intent phrasing followed shortly by a plan / tool-action verb. The
|
||||
# final answer. The intent alternatives mirror _INTENT_SIGNAL above so
|
||||
# every recognised intent phrase can disqualify a numbered list. The
|
||||
# apostrophe in ``i['’]ll`` is required (no ``?``) so the regex does not
|
||||
# accidentally match the word "ill". The verb set is intentionally
|
||||
# conservative: ambiguous verbs like "write", "create", "make", "build"
|
||||
# are omitted because real answer lists use them ("1. Write a poem",
|
||||
# "1. Create directory").
|
||||
# "1. Create directory"). ``plan:`` / ``approach:`` is anchored to the
|
||||
# start of a line so "lesson plan:" / "meal plan:" do not trip the guard.
|
||||
_PLAN_LIST_FRAMING = re.compile(
|
||||
r"\b(?:here['’]?s (?:my |the |a )?(?:plan|approach)|step \d+|"
|
||||
r"i['’]ll|i will|i am going to|let me|now i|next i)\b"
|
||||
r"\b(?:here['’]?s (?:my |the |a )?(?:plan|approach)|"
|
||||
r"here (?:is|are) (?:my |the |a )?(?:plan|approach|steps)|"
|
||||
r"step \d+|"
|
||||
r"i['’](?:ll|m going to|m gonna)|i am (?:going to|gonna)|"
|
||||
r"i will|i shall|let me|allow me|now i|next i)\b"
|
||||
r"[\s\S]{0,80}"
|
||||
r"\b(?:search|look up|call|use|fetch|browse|run|execute|"
|
||||
r"check|find|open|verify|compare|summari[sz]e|think|respond|"
|
||||
r"answer|analy[sz]e|explore|outline|gather|query|reason)\b"
|
||||
r"|\b(?:plan|approach):",
|
||||
r"|(?:^|\r?\n)[ \t]*(?:(?:my|the|our|a|this|that)\s+)?(?:plan|approach):",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -376,6 +376,58 @@ def test_plan_framing_requires_apostrophe_in_ill():
|
|||
assert got == expected, f"{content!r} expected reprompt={expected} got {got}"
|
||||
|
||||
|
||||
def test_reprompts_on_all_intent_form_numbered_action_plans():
|
||||
"""``_PLAN_LIST_FRAMING`` must mirror every intent form that
|
||||
``_INTENT_SIGNAL`` accepts so numbered action plans phrased with
|
||||
``Allow me``, ``I'm going to``, ``I'm gonna``, ``I am gonna``,
|
||||
``I shall``, ``Now I``, ``Next I`` also re-prompt instead of being
|
||||
silently classified as completed answers."""
|
||||
samples = [
|
||||
"Allow me to do this:\n1. search the docs\n2. fetch the result",
|
||||
"I'm going to do this:\n1. search the docs\n2. fetch the result",
|
||||
"I'm gonna do this:\n1. search the docs\n2. fetch the result",
|
||||
"I am gonna do this:\n1. search the docs\n2. fetch the result",
|
||||
"I shall do this:\n1. search the docs\n2. fetch the result",
|
||||
"Now I will do these:\n1. search\n2. summarise",
|
||||
"Next I will do these:\n1. fetch\n2. compare",
|
||||
]
|
||||
for s in samples:
|
||||
assert _would_reprompt(s), s
|
||||
|
||||
|
||||
def test_plan_colon_intent_is_line_anchored():
|
||||
"""``Plan:`` / ``Approach:`` only counts as an intent marker when it
|
||||
is at the start of a line. Without this anchor, normal direct
|
||||
answers containing phrases like ``lesson plan:``, ``meal plan:``,
|
||||
``migration plan:``, or ``My approach:`` would trigger the
|
||||
re-prompt path and risk wiping a valid response."""
|
||||
# These mid-line "plan:" / "approach:" mentions are NOT intent signals.
|
||||
# The qualifier before "plan" is a content noun ("lesson", "meal",
|
||||
# "migration") rather than a generic determiner ("my", "the", ...).
|
||||
direct_answers = [
|
||||
"Here is a lesson plan:\n1. Warm-up\n2. Group practice\n3. Assessment",
|
||||
"I prepared a meal plan: rice, beans, eggs.",
|
||||
"Quick approach: top-down then bottom-up.",
|
||||
]
|
||||
for s in direct_answers:
|
||||
assert not _INTENT_SIGNAL.search(s), s
|
||||
assert not _would_reprompt(s), s
|
||||
# "Plan:" / "Approach:" with optional generic determiner at the start
|
||||
# of a line IS an intent signal.
|
||||
plan_starts = [
|
||||
"Plan:\n1. search\n2. summarise",
|
||||
"Approach:\n1. fetch\n2. compare",
|
||||
" Plan:\n1. think\n2. respond", # leading indent OK
|
||||
"Lorem ipsum\nPlan:\n1. step\n2. step", # plan: on a later line
|
||||
"My plan:\n1. search\n2. summarise",
|
||||
"The plan:\n1. look up\n2. compare",
|
||||
"Our approach:\n1. fetch\n2. verify",
|
||||
]
|
||||
for s in plan_starts:
|
||||
assert _INTENT_SIGNAL.search(s), s
|
||||
assert _would_reprompt(s), s
|
||||
|
||||
|
||||
# ── Cross-platform line endings ────────────────────────────────────
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue