Studio: r14 fixes - longer CommonMark closing fence, explicit-plan header standalone, use-python tool wording

- _HAS_ANSWER_ARTIFACT closing fence now accepts strictly more delimiters
  than the opener (CommonMark rule). The opener stays anchored on both
  sides so a 4-open / 3-close payload still does not match, but a
  legitimate 3-open / 4-close (and 3-tilde / 4-tilde) answer is now
  recognised as a completed artifact.

- _EXPLICIT_PLAN_HEADER triggers the plan classification by itself when
  the response contains \"Here's my plan\" / \"Here's my approach\" /
  \"Here's the plan\". Numbered stalls like \"Here's my plan:\n1. Analyze\n
  2. Draft\" re-prompt again without needing a freshness-gated verb.
  Plain \"Plan:\" / \"My weekly plan:\" stay valid answers because they
  lack the possessive first-person header.

- _TOOL_ACTION_VERBS adds \"use python (tool) to ...\", \"use the python
  tool\", \"invoke the python tool\", and \"use the search tool\" so
  numbered plans that route through these phrasings still re-prompt.
This commit is contained in:
Daniel Han 2026-05-24 20:20:38 +00:00
commit 4652a4b03c
2 changed files with 86 additions and 6 deletions

View file

@ -74,6 +74,8 @@ _TOOL_ACTION_VERBS = (
rf"{_TOOL_LOOKUP_TARGET}|"
r"(?:research|investigate|find|check|verify) (?:for )?(?:the |a |an )?"
rf"{_TOOL_LOOKUP_TARGET}|"
r"(?:use|invoke|call) (?:the )?(?:python|search) tool|"
r"use python(?: tool)? to|"
r"call (?:a |the )?tool|run (?:python|the code)|execute (?:python|the code)"
)
@ -122,11 +124,11 @@ _HAS_ANSWER_ARTIFACT = re.compile(
# closing fence must have at least as many delimiters, and the line
# must end cleanly (only trailing whitespace before newline / EOS),
# so spam like ``` ```not actually closed ``` does not count.
r"(?<!`)(?P<bf>`{3,})(?!`)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=bf)(?!`)[ \t]*(?:\r?\n|\Z)"
r"(?<!`)(?P<bf>`{3,})(?!`)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=bf)`*[ \t]*(?:\r?\n|\Z)"
# Closed tilde code fence; same 3+ rule (several models emit ~~~ when
# the body itself contains backticks). Anchored to the full run of
# tildes on both sides so a 4-tilde open cannot match a 3-tilde close.
r"|(?<!~)(?P<tf>~{3,})(?!~)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=tf)(?!~)[ \t]*(?:\r?\n|\Z)"
# the body itself contains backticks). Opener anchored to the full
# run of tildes; closer accepts >= opener length per CommonMark.
r"|(?<!~)(?P<tf>~{3,})(?!~)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=tf)~*[ \t]*(?:\r?\n|\Z)"
# Complete HTML page; doctype prefix is optional.
r"|(?:<!doctype\b[\s\S]{0,200}?)?<html\b[\s\S]{0,4000}?</html>"
# Complete SVG document.
@ -158,6 +160,16 @@ _PLAN_LIST_FRAMING = re.compile(
re.IGNORECASE,
)
# "Here's my plan" / "Here's my approach" are strong stand-alone plan
# signals: a possessive, first-person framing where the model is
# announcing what it WILL do. Treat the following numbered list as a
# plan regardless of the specific verbs each item uses, so stalls like
# ``Here's my plan: 1. Analyze 2. Draft`` still re-prompt.
_EXPLICIT_PLAN_HEADER = re.compile(
r"\bhere[']?s (?:my |the |a )?(?:plan|approach)\b",
re.IGNORECASE,
)
_FENCE_LINE_RE = re.compile(r"^[ \t]*(?P<fence>`{3,}|~{3,})(?P<trailing>[^\r\n]*)$")
@ -194,14 +206,19 @@ def _has_answer_artifact(text: str) -> bool:
Code fences, complete HTML, and complete SVG count directly. A
numbered list counts only when there is no plan framing, so stalls
like ``Here's my plan:\\n1. search\\n2. summarise`` still re-prompt.
An unclosed fence disqualifies the numbered-list fallback so a list
INSIDE incomplete code does not look like a final answer.
An explicit ``Here's my plan`` / ``Here's my approach`` header is
also enough to flag the list as a plan, even when no narrow tool-
action verb appears in the items. An unclosed fence disqualifies
the numbered-list fallback so a list INSIDE incomplete code does
not look like a final answer.
"""
if _HAS_ANSWER_ARTIFACT.search(text):
return True
if _has_unclosed_code_fence(text):
return False
if _NUMBERED_LIST_ARTIFACT.search(text):
if _EXPLICIT_PLAN_HEADER.search(text):
return False
return _PLAN_LIST_FRAMING.search(text) is None
return False

View file

@ -749,6 +749,69 @@ def test_artifact_regex_rejects_shorter_commonmark_closing_fence():
assert _would_reprompt(content), content
def test_artifact_regex_accepts_longer_commonmark_closing_fence():
"""CommonMark allows the closing fence to have MORE delimiters than
the opener. A 3-backtick opener with a 4-backtick close, or a
3-tilde opener with a 4-tilde close, is still a complete artifact."""
samples = [
"First, let me show.\n```python\nprint('hi')\n````",
"First, let me show.\n````python\nprint('``` inside')\n`````",
"First, let me show.\n~~~python\nprint('hi')\n~~~~",
]
for content in samples:
assert _has_answer_artifact(content), content
assert not _would_reprompt(content), content
def test_reprompts_on_explicit_plan_header_numbered_list():
"""``Here's my plan`` / ``Here's my approach`` is a strong stand-alone
plan signal. The following numbered list is the plan itself, not a
final answer, even when no narrow tool-action verb appears."""
samples = [
"Here's my plan:\n1. Analyze the request.\n2. Draft the answer.",
"Here's my plan:\n1. Create the Python file.\n2. Add the game loop.\n3. Test.",
"Here's my approach:\n1. Outline.\n2. Write.\n3. Review.",
"Here's the plan:\n1. Define the variables.\n2. Return the result.",
]
for s in samples:
assert _would_reprompt(s), s
def test_reprompts_on_numbered_plan_with_python_tool_wording():
"""``use python (tool) to ...`` / ``use the python tool`` / ``use the
search tool`` in a numbered plan still re-prompts."""
samples = [
"Here's my plan:\n1. Use Python to calculate the answer.\n2. Return.",
"First, I'll do this:\n1. Use the python tool to parse the file.\n2. Summarize.",
"Here's my plan:\n1. Use the search tool.\n2. Summarize.",
]
for s in samples:
assert _would_reprompt(s), s
def test_no_reprompt_on_lesson_plan_answer_without_explicit_header():
"""A final answer with a ``Plan:`` heading (no ``Here's my``
possessive) and no tool framing must STILL count as an answer.
Common cases: lesson plan, workout plan, meal plan."""
samples = [
(
"Plan:\n"
"1. Warm up for 5 minutes.\n"
"2. Run for 20 minutes.\n"
"3. Cool down with stretching."
),
(
"My weekly plan:\n"
"1. Monday: rest.\n"
"2. Tuesday: jog.\n"
"3. Wednesday: swim."
),
]
for content in samples:
assert _has_answer_artifact(content), content
assert not _would_reprompt(content), content
def test_open_fence_with_inner_numbered_list_still_reprompts():
"""A response that opens a code fence and emits numbered lines INSIDE
must NOT count those lines as a completed numbered-list answer."""