Studio: r24 fixes - revert bare i need to, add complete-these-steps verbs
- _INTENT_SIGNAL, _DIRECT_NUMBERED_PLAN_FRAMING, _BARE_INTENT_NUMBERED_PLAN,
and _STRONG_INTENT_BEFORE_LIST all drop bare "i need to" from their
intent vocabulary. The phrase is too common in ordinary clarification
prose ("I need to know your operating system") and quoted answer text
("I need to leave early"), so adding it as a re-prompt trigger
produced too many false positives. Genuine "I need to X..." plans
are still caught when paired with "I'll", "Let me", or "first" /
"step N" framing elsewhere in the candidate.
- _DIRECT_NUMBERED_PLAN_FRAMING adds "complete these steps" /
"complete the following steps" to the verb whitelist, parallel
to the existing "take/follow these steps" / "perform these actions".
This commit is contained in:
parent
b6477eddb1
commit
6cf6797fb3
2 changed files with 32 additions and 19 deletions
|
|
@ -88,7 +88,7 @@ _INTENT_SIGNAL = re.compile(
|
|||
# Handles both straight and curly apostrophes.
|
||||
# Excludes "I can", "I should", "I want to", "let's" which
|
||||
# appear frequently in direct answers / explanations.
|
||||
r"\b(i['\u2019](ll|m going to|m gonna)|i am (going to|gonna)|i will|i shall|i need to|let me|allow me)\b"
|
||||
r"\b(i['\u2019](ll|m going to|m gonna)|i am (going to|gonna)|i will|i shall|let me|allow me)\b"
|
||||
r"|"
|
||||
# Step/plan framing: "First ...", "Step 1:", "Here's my plan".
|
||||
r"\b(?:first\b|step \d+:?|here['\u2019]?s (?:my |the |a )?(?:plan|approach))"
|
||||
|
|
@ -194,7 +194,7 @@ _EXPLICIT_PLAN_HEADER = re.compile(
|
|||
_DIRECT_NUMBERED_PLAN_FRAMING = re.compile(
|
||||
r"(?:"
|
||||
r"\b(?:i['’](?:ll|m going to|m gonna)|i am (?:going to|gonna)|"
|
||||
r"i will|i shall|i need to|let me|allow me|now i|next i)\b"
|
||||
r"i will|i shall|let me|allow me|now i|next i)\b"
|
||||
r"[^\r\n]{0,160}"
|
||||
r"\b(?:open|read|search|look (?:this |that |it |them )?up|browse|"
|
||||
r"google|find|check|verify|compare|review|inspect|examine|"
|
||||
|
|
@ -202,6 +202,7 @@ _DIRECT_NUMBERED_PLAN_FRAMING = re.compile(
|
|||
r"do (?:this|these|the following|it)|"
|
||||
r"take (?:these|the following) steps|"
|
||||
r"follow (?:these|the following) steps|"
|
||||
r"complete (?:these|the following) steps|"
|
||||
r"perform (?:these|the following) actions|"
|
||||
r"proceed|start|begin|"
|
||||
r"create|build|implement|set up|add|calculate|compute|analy[sz]e|"
|
||||
|
|
@ -327,7 +328,7 @@ _NUMBERED_ACTION_ITEM = re.compile(
|
|||
# numbered-list cross-check.
|
||||
_STRONG_INTENT_BEFORE_LIST = re.compile(
|
||||
r"\b(?:i['’](?:ll|m going to|m gonna)|i am (?:going to|gonna)|"
|
||||
r"i will|i shall|i need to|let me|allow me|now i|next i)\b",
|
||||
r"i will|i shall|let me|allow me|now i|next i)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
|
@ -340,7 +341,7 @@ _STRONG_INTENT_BEFORE_LIST = re.compile(
|
|||
# signal that this is a tool stall.
|
||||
_BARE_INTENT_NUMBERED_PLAN = re.compile(
|
||||
r"\b(?:i['’](?:ll|m going to|m gonna)|i am (?:going to|gonna)|"
|
||||
r"i will|i shall|i need to|let me|allow me|now i|next i)\s*:[ \t]*"
|
||||
r"i will|i shall|let me|allow me|now i|next i)\s*:[ \t]*"
|
||||
r"(?:\r?\n)[ \t]*\d+\.[ \t]+"
|
||||
r"(?:open|read|search|look up|check|verify|create|build|add|set up|"
|
||||
r"load|inspect|parse|calculate|compute|analy[sz]e|extract|run|execute|"
|
||||
|
|
|
|||
|
|
@ -985,21 +985,28 @@ def test_no_reprompt_on_code_fence_containing_markup_literal():
|
|||
assert not _would_reprompt(content), content
|
||||
|
||||
|
||||
def test_reprompts_on_i_need_to_numbered_plan():
|
||||
"""``First, I need to:`` / ``I need to ...`` numbered plans are
|
||||
tool stalls and still re-prompt."""
|
||||
def test_reprompts_on_i_will_gather_identify_numbered_plan():
|
||||
"""First-person + gather/identify verbs in list items is a tool
|
||||
stall when the intent appears directly before the list."""
|
||||
content = (
|
||||
"I'll:\n"
|
||||
"1. Gather the relevant files.\n"
|
||||
"2. Identify the issue."
|
||||
)
|
||||
assert not _has_answer_artifact(content), content
|
||||
assert _would_reprompt(content), content
|
||||
|
||||
|
||||
def test_no_reprompt_on_bare_i_need_to_clarification():
|
||||
"""Bare ``I need to`` clarification or prose answers must NOT
|
||||
trigger the re-prompt. The phrase is too common in plain answers."""
|
||||
samples = [
|
||||
("First, I need to:\n" "1. Read the uploaded file.\n" "2. Summarize it."),
|
||||
(
|
||||
"I need to fetch the latest data:\n"
|
||||
"1. Query the price.\n"
|
||||
"2. Format the answer."
|
||||
),
|
||||
("I'll:\n" "1. Gather the relevant files.\n" "2. Identify the issue."),
|
||||
"I need to know your operating system before giving the install command.",
|
||||
"I need to be clear: the answer is Paris.",
|
||||
'The sentence is: "I need to leave early today."',
|
||||
]
|
||||
for content in samples:
|
||||
assert not _has_answer_artifact(content), content
|
||||
assert _would_reprompt(content), content
|
||||
assert not _would_reprompt(content), content
|
||||
|
||||
|
||||
def test_reprompts_on_visit_or_access_numbered_plan():
|
||||
|
|
@ -1037,9 +1044,9 @@ def test_no_reprompt_on_inline_backtick_python_prose_after_code():
|
|||
assert not _would_reprompt(content)
|
||||
|
||||
|
||||
def test_reprompts_on_take_or_follow_steps_numbered_plan():
|
||||
"""``I'll take these steps:`` / ``I will follow these steps:`` +
|
||||
numbered list of work items is a plan stall."""
|
||||
def test_reprompts_on_take_follow_complete_steps_numbered_plan():
|
||||
"""``I'll take/follow/complete these steps:`` + numbered list of
|
||||
work items is a plan stall."""
|
||||
samples = [
|
||||
(
|
||||
"I'll take these steps:\n"
|
||||
|
|
@ -1053,6 +1060,11 @@ def test_reprompts_on_take_or_follow_steps_numbered_plan():
|
|||
"2. Read the relevant section.\n"
|
||||
"3. Answer."
|
||||
),
|
||||
(
|
||||
"Let me complete these steps:\n"
|
||||
"1. Read the uploaded CSV.\n"
|
||||
"2. Check the totals."
|
||||
),
|
||||
]
|
||||
for content in samples:
|
||||
assert not _has_answer_artifact(content), content
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue