diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index d86b0d9b80..9552e7af4c 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -88,7 +88,7 @@ _INTENT_SIGNAL = re.compile( # Handles both straight and curly apostrophes. # Excludes "I can", "I should", "I want to", "let's" which # appear frequently in direct answers / explanations. - r"\b(i['\u2019](ll|m going to|m gonna)|i am (going to|gonna)|i will|i shall|i need to|let me|allow me)\b" + r"\b(i['\u2019](ll|m going to|m gonna)|i am (going to|gonna)|i will|i shall|let me|allow me)\b" r"|" # Step/plan framing: "First ...", "Step 1:", "Here's my plan". r"\b(?:first\b|step \d+:?|here['\u2019]?s (?:my |the |a )?(?:plan|approach))" @@ -194,7 +194,7 @@ _EXPLICIT_PLAN_HEADER = re.compile( _DIRECT_NUMBERED_PLAN_FRAMING = re.compile( r"(?:" r"\b(?:i['’](?:ll|m going to|m gonna)|i am (?:going to|gonna)|" - r"i will|i shall|i need to|let me|allow me|now i|next i)\b" + r"i will|i shall|let me|allow me|now i|next i)\b" r"[^\r\n]{0,160}" r"\b(?:open|read|search|look (?:this |that |it |them )?up|browse|" r"google|find|check|verify|compare|review|inspect|examine|" @@ -202,6 +202,7 @@ _DIRECT_NUMBERED_PLAN_FRAMING = re.compile( r"do (?:this|these|the following|it)|" r"take (?:these|the following) steps|" r"follow (?:these|the following) steps|" + r"complete (?:these|the following) steps|" r"perform (?:these|the following) actions|" r"proceed|start|begin|" r"create|build|implement|set up|add|calculate|compute|analy[sz]e|" @@ -327,7 +328,7 @@ _NUMBERED_ACTION_ITEM = re.compile( # numbered-list cross-check. _STRONG_INTENT_BEFORE_LIST = re.compile( r"\b(?:i['’](?:ll|m going to|m gonna)|i am (?:going to|gonna)|" - r"i will|i shall|i need to|let me|allow me|now i|next i)\b", + r"i will|i shall|let me|allow me|now i|next i)\b", re.IGNORECASE, ) @@ -340,7 +341,7 @@ _STRONG_INTENT_BEFORE_LIST = re.compile( # signal that this is a tool stall. _BARE_INTENT_NUMBERED_PLAN = re.compile( r"\b(?:i['’](?:ll|m going to|m gonna)|i am (?:going to|gonna)|" - r"i will|i shall|i need to|let me|allow me|now i|next i)\s*:[ \t]*" + r"i will|i shall|let me|allow me|now i|next i)\s*:[ \t]*" r"(?:\r?\n)[ \t]*\d+\.[ \t]+" r"(?:open|read|search|look up|check|verify|create|build|add|set up|" r"load|inspect|parse|calculate|compute|analy[sz]e|extract|run|execute|" diff --git a/studio/backend/tests/test_llama_cpp_reprompt_guard.py b/studio/backend/tests/test_llama_cpp_reprompt_guard.py index 96e2b5ad45..73e88637e2 100644 --- a/studio/backend/tests/test_llama_cpp_reprompt_guard.py +++ b/studio/backend/tests/test_llama_cpp_reprompt_guard.py @@ -985,21 +985,28 @@ def test_no_reprompt_on_code_fence_containing_markup_literal(): assert not _would_reprompt(content), content -def test_reprompts_on_i_need_to_numbered_plan(): - """``First, I need to:`` / ``I need to ...`` numbered plans are - tool stalls and still re-prompt.""" +def test_reprompts_on_i_will_gather_identify_numbered_plan(): + """First-person + gather/identify verbs in list items is a tool + stall when the intent appears directly before the list.""" + content = ( + "I'll:\n" + "1. Gather the relevant files.\n" + "2. Identify the issue." + ) + assert not _has_answer_artifact(content), content + assert _would_reprompt(content), content + + +def test_no_reprompt_on_bare_i_need_to_clarification(): + """Bare ``I need to`` clarification or prose answers must NOT + trigger the re-prompt. The phrase is too common in plain answers.""" samples = [ - ("First, I need to:\n" "1. Read the uploaded file.\n" "2. Summarize it."), - ( - "I need to fetch the latest data:\n" - "1. Query the price.\n" - "2. Format the answer." - ), - ("I'll:\n" "1. Gather the relevant files.\n" "2. Identify the issue."), + "I need to know your operating system before giving the install command.", + "I need to be clear: the answer is Paris.", + 'The sentence is: "I need to leave early today."', ] for content in samples: - assert not _has_answer_artifact(content), content - assert _would_reprompt(content), content + assert not _would_reprompt(content), content def test_reprompts_on_visit_or_access_numbered_plan(): @@ -1037,9 +1044,9 @@ def test_no_reprompt_on_inline_backtick_python_prose_after_code(): assert not _would_reprompt(content) -def test_reprompts_on_take_or_follow_steps_numbered_plan(): - """``I'll take these steps:`` / ``I will follow these steps:`` + - numbered list of work items is a plan stall.""" +def test_reprompts_on_take_follow_complete_steps_numbered_plan(): + """``I'll take/follow/complete these steps:`` + numbered list of + work items is a plan stall.""" samples = [ ( "I'll take these steps:\n" @@ -1053,6 +1060,11 @@ def test_reprompts_on_take_or_follow_steps_numbered_plan(): "2. Read the relevant section.\n" "3. Answer." ), + ( + "Let me complete these steps:\n" + "1. Read the uploaded CSV.\n" + "2. Check the totals." + ), ] for content in samples: assert not _has_answer_artifact(content), content