Studio: r14 fixes - longer CommonMark closing fence, explicit-plan header standalone, use-python tool wording
- _HAS_ANSWER_ARTIFACT closing fence now accepts strictly more delimiters than the opener (CommonMark rule). The opener stays anchored on both sides so a 4-open / 3-close payload still does not match, but a legitimate 3-open / 4-close (and 3-tilde / 4-tilde) answer is now recognised as a completed artifact. - _EXPLICIT_PLAN_HEADER triggers the plan classification by itself when the response contains \"Here's my plan\" / \"Here's my approach\" / \"Here's the plan\". Numbered stalls like \"Here's my plan:\n1. Analyze\n 2. Draft\" re-prompt again without needing a freshness-gated verb. Plain \"Plan:\" / \"My weekly plan:\" stay valid answers because they lack the possessive first-person header. - _TOOL_ACTION_VERBS adds \"use python (tool) to ...\", \"use the python tool\", \"invoke the python tool\", and \"use the search tool\" so numbered plans that route through these phrasings still re-prompt.
This commit is contained in:
parent
b7c7427eb8
commit
4652a4b03c
2 changed files with 86 additions and 6 deletions
|
|
@ -74,6 +74,8 @@ _TOOL_ACTION_VERBS = (
|
|||
rf"{_TOOL_LOOKUP_TARGET}|"
|
||||
r"(?:research|investigate|find|check|verify) (?:for )?(?:the |a |an )?"
|
||||
rf"{_TOOL_LOOKUP_TARGET}|"
|
||||
r"(?:use|invoke|call) (?:the )?(?:python|search) tool|"
|
||||
r"use python(?: tool)? to|"
|
||||
r"call (?:a |the )?tool|run (?:python|the code)|execute (?:python|the code)"
|
||||
)
|
||||
|
||||
|
|
@ -122,11 +124,11 @@ _HAS_ANSWER_ARTIFACT = re.compile(
|
|||
# closing fence must have at least as many delimiters, and the line
|
||||
# must end cleanly (only trailing whitespace before newline / EOS),
|
||||
# so spam like ``` ```not actually closed ``` does not count.
|
||||
r"(?<!`)(?P<bf>`{3,})(?!`)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=bf)(?!`)[ \t]*(?:\r?\n|\Z)"
|
||||
r"(?<!`)(?P<bf>`{3,})(?!`)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=bf)`*[ \t]*(?:\r?\n|\Z)"
|
||||
# Closed tilde code fence; same 3+ rule (several models emit ~~~ when
|
||||
# the body itself contains backticks). Anchored to the full run of
|
||||
# tildes on both sides so a 4-tilde open cannot match a 3-tilde close.
|
||||
r"|(?<!~)(?P<tf>~{3,})(?!~)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=tf)(?!~)[ \t]*(?:\r?\n|\Z)"
|
||||
# the body itself contains backticks). Opener anchored to the full
|
||||
# run of tildes; closer accepts >= opener length per CommonMark.
|
||||
r"|(?<!~)(?P<tf>~{3,})(?!~)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=tf)~*[ \t]*(?:\r?\n|\Z)"
|
||||
# Complete HTML page; doctype prefix is optional.
|
||||
r"|(?:<!doctype\b[\s\S]{0,200}?)?<html\b[\s\S]{0,4000}?</html>"
|
||||
# Complete SVG document.
|
||||
|
|
@ -158,6 +160,16 @@ _PLAN_LIST_FRAMING = re.compile(
|
|||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# "Here's my plan" / "Here's my approach" are strong stand-alone plan
|
||||
# signals: a possessive, first-person framing where the model is
|
||||
# announcing what it WILL do. Treat the following numbered list as a
|
||||
# plan regardless of the specific verbs each item uses, so stalls like
|
||||
# ``Here's my plan: 1. Analyze 2. Draft`` still re-prompt.
|
||||
_EXPLICIT_PLAN_HEADER = re.compile(
|
||||
r"\bhere['’]?s (?:my |the |a )?(?:plan|approach)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
_FENCE_LINE_RE = re.compile(r"^[ \t]*(?P<fence>`{3,}|~{3,})(?P<trailing>[^\r\n]*)$")
|
||||
|
||||
|
|
@ -194,14 +206,19 @@ def _has_answer_artifact(text: str) -> bool:
|
|||
Code fences, complete HTML, and complete SVG count directly. A
|
||||
numbered list counts only when there is no plan framing, so stalls
|
||||
like ``Here's my plan:\\n1. search\\n2. summarise`` still re-prompt.
|
||||
An unclosed fence disqualifies the numbered-list fallback so a list
|
||||
INSIDE incomplete code does not look like a final answer.
|
||||
An explicit ``Here's my plan`` / ``Here's my approach`` header is
|
||||
also enough to flag the list as a plan, even when no narrow tool-
|
||||
action verb appears in the items. An unclosed fence disqualifies
|
||||
the numbered-list fallback so a list INSIDE incomplete code does
|
||||
not look like a final answer.
|
||||
"""
|
||||
if _HAS_ANSWER_ARTIFACT.search(text):
|
||||
return True
|
||||
if _has_unclosed_code_fence(text):
|
||||
return False
|
||||
if _NUMBERED_LIST_ARTIFACT.search(text):
|
||||
if _EXPLICIT_PLAN_HEADER.search(text):
|
||||
return False
|
||||
return _PLAN_LIST_FRAMING.search(text) is None
|
||||
return False
|
||||
|
||||
|
|
|
|||
|
|
@ -749,6 +749,69 @@ def test_artifact_regex_rejects_shorter_commonmark_closing_fence():
|
|||
assert _would_reprompt(content), content
|
||||
|
||||
|
||||
def test_artifact_regex_accepts_longer_commonmark_closing_fence():
|
||||
"""CommonMark allows the closing fence to have MORE delimiters than
|
||||
the opener. A 3-backtick opener with a 4-backtick close, or a
|
||||
3-tilde opener with a 4-tilde close, is still a complete artifact."""
|
||||
samples = [
|
||||
"First, let me show.\n```python\nprint('hi')\n````",
|
||||
"First, let me show.\n````python\nprint('``` inside')\n`````",
|
||||
"First, let me show.\n~~~python\nprint('hi')\n~~~~",
|
||||
]
|
||||
for content in samples:
|
||||
assert _has_answer_artifact(content), content
|
||||
assert not _would_reprompt(content), content
|
||||
|
||||
|
||||
def test_reprompts_on_explicit_plan_header_numbered_list():
|
||||
"""``Here's my plan`` / ``Here's my approach`` is a strong stand-alone
|
||||
plan signal. The following numbered list is the plan itself, not a
|
||||
final answer, even when no narrow tool-action verb appears."""
|
||||
samples = [
|
||||
"Here's my plan:\n1. Analyze the request.\n2. Draft the answer.",
|
||||
"Here's my plan:\n1. Create the Python file.\n2. Add the game loop.\n3. Test.",
|
||||
"Here's my approach:\n1. Outline.\n2. Write.\n3. Review.",
|
||||
"Here's the plan:\n1. Define the variables.\n2. Return the result.",
|
||||
]
|
||||
for s in samples:
|
||||
assert _would_reprompt(s), s
|
||||
|
||||
|
||||
def test_reprompts_on_numbered_plan_with_python_tool_wording():
|
||||
"""``use python (tool) to ...`` / ``use the python tool`` / ``use the
|
||||
search tool`` in a numbered plan still re-prompts."""
|
||||
samples = [
|
||||
"Here's my plan:\n1. Use Python to calculate the answer.\n2. Return.",
|
||||
"First, I'll do this:\n1. Use the python tool to parse the file.\n2. Summarize.",
|
||||
"Here's my plan:\n1. Use the search tool.\n2. Summarize.",
|
||||
]
|
||||
for s in samples:
|
||||
assert _would_reprompt(s), s
|
||||
|
||||
|
||||
def test_no_reprompt_on_lesson_plan_answer_without_explicit_header():
|
||||
"""A final answer with a ``Plan:`` heading (no ``Here's my``
|
||||
possessive) and no tool framing must STILL count as an answer.
|
||||
Common cases: lesson plan, workout plan, meal plan."""
|
||||
samples = [
|
||||
(
|
||||
"Plan:\n"
|
||||
"1. Warm up for 5 minutes.\n"
|
||||
"2. Run for 20 minutes.\n"
|
||||
"3. Cool down with stretching."
|
||||
),
|
||||
(
|
||||
"My weekly plan:\n"
|
||||
"1. Monday: rest.\n"
|
||||
"2. Tuesday: jog.\n"
|
||||
"3. Wednesday: swim."
|
||||
),
|
||||
]
|
||||
for content in samples:
|
||||
assert _has_answer_artifact(content), content
|
||||
assert not _would_reprompt(content), content
|
||||
|
||||
|
||||
def test_open_fence_with_inner_numbered_list_still_reprompts():
|
||||
"""A response that opens a code fence and emits numbered lines INSIDE
|
||||
must NOT count those lines as a completed numbered-list answer."""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue