Studio: r16 fixes - inline fence tracking, broaden direct-intent plan with first/step prefixes

- _has_unclosed_code_fence() now scans every line with re.search and a
  shared FENCE_RUN regex, so an inline opening fence such as
  "First, let me write it. \`\`\`python" is tracked alongside the
  column-0 openers. A numbered list emitted INSIDE an inline-open
  fence no longer reads as a final answer.

- _DIRECT_NUMBERED_PLAN_FRAMING adds "first" and "step N(:?)" to its
  intent prefixes and "look up" to its verb whitelist. Plans like
  "First, analyze the uploaded CSV:\n1. Load rows\n2. Compute total"
  or "I'll look that up:\n1. Search the docs" now re-prompt instead
  of being mis-classified as final answers. The verb whitelist still
  excludes bare search/find/check/verify so "First, use binary
  search:\n1. Search the left half" stays an answer.
This commit is contained in:
Daniel Han 2026-05-24 20:47:34 +00:00
commit 7b0ed8333a
2 changed files with 76 additions and 6 deletions

View file

@ -180,9 +180,11 @@ _EXPLICIT_PLAN_HEADER = re.compile(
# poem" is not misclassified as a stall.
_DIRECT_NUMBERED_PLAN_FRAMING = re.compile(
r"\b(?:i['](?:ll|m going to|m gonna)|i am (?:going to|gonna)|"
r"i will|i shall|let me|allow me|now i|next i)\b"
r"i will|i shall|let me|allow me|now i|next i|"
r"first|step \d+:?)\b"
r"[^\r\n]{0,160}"
r"\b(?:do (?:this|these|the following|it)|"
r"look (?:this |that |it |them )?up|"
r"proceed|start|begin|"
r"create|build|implement|set up|add|"
r"calculate|compute|analy[sz]e|parse|load|run|execute|test)\b"
@ -192,7 +194,9 @@ _DIRECT_NUMBERED_PLAN_FRAMING = re.compile(
)
_FENCE_LINE_RE = re.compile(r"^[ \t]*(?P<fence>`{3,}|~{3,})(?P<trailing>[^\r\n]*)$")
_FENCE_RUN_RE = re.compile(
r"(?<!`)(?P<backticks>`{3,})(?!`)|(?<!~)(?P<tildes>~{3,})(?!~)"
)
def _has_unclosed_code_fence(text: str) -> bool:
@ -201,16 +205,19 @@ def _has_unclosed_code_fence(text: str) -> bool:
A complete fence answer is already caught by _HAS_ANSWER_ARTIFACT.
This helper exists so that an OPEN fence (model still streaming
code, or stream cut short) does not let an embedded numbered list
inside the fence body masquerade as a final answer.
inside the fence body masquerade as a final answer. The scan is
done per line and uses ``search`` (not ``match``) so an inline
opener such as ``First, let me write it. \\`\\`\\`python`` is also
tracked.
"""
active_char: Optional[str] = None
active_len = 0
for line in text.splitlines():
m = _FENCE_LINE_RE.match(line)
m = _FENCE_RUN_RE.search(line)
if not m:
continue
fence = m.group("fence")
trailing = m.group("trailing").strip()
fence = m.group("backticks") or m.group("tildes")
trailing = line[m.end():].strip()
ch = fence[0]
if active_char is None:
active_char = ch

View file

@ -869,6 +869,69 @@ def test_no_reprompt_on_let_me_explain_numbered_answer():
assert not _would_reprompt(content), content
def test_same_line_open_fence_with_numbered_body_still_reprompts():
"""An OPEN code fence on the same line as preceding prose ("First,
let me write it. ``\\u00e0``text\\n...") still gates the numbered-list
fallback. The unclosed-fence helper now uses ``search`` so inline
openers are tracked, not just openers at column 0."""
content = (
"First, let me write it. ```text\n"
"1. Install dependencies\n"
"2. Run the app"
)
assert not _has_answer_artifact(content)
assert _would_reprompt(content)
def test_reprompts_on_first_step_numbered_compute_plan():
"""Bare ``First, [verb]`` / ``Step N: [verb]`` followed by a numbered
list is a plan stall when the verb implies compute / tool work
(analyze, parse, calculate, create, etc.). Distinct from
``First, use binary search:`` (verb ``use`` not in whitelist)."""
samples = [
(
"First, analyze the uploaded CSV:\n"
"1. Load the rows.\n"
"2. Compute the average revenue."
),
(
"First, parse the pasted JSON:\n"
"1. Load the object.\n"
"2. Calculate the total."
),
(
"First, create the Python game:\n"
"1. Set up pygame.\n"
"2. Add the game loop."
),
(
"Step 1: analyze the uploaded CSV:\n"
"1. Load rows.\n"
"2. Compute the total."
),
(
"I'll look that up:\n"
"1. Search the docs.\n"
"2. Summarize the result."
),
]
for content in samples:
assert _would_reprompt(content), content
def test_no_reprompt_on_first_use_binary_search_answer():
"""``First, use binary search:`` is an ordinary algorithm answer.
``use`` is not in the direct-numbered-plan verb whitelist so the
following list stays an answer."""
content = (
"First, use binary search:\n"
"1. Search the left half.\n"
"2. Search the right half."
)
assert _has_answer_artifact(content)
assert not _would_reprompt(content)
def test_reprompts_when_later_fence_is_open_after_closed_fence():
"""A response with a complete code fence followed by a SECOND,
unclosed fence is still mid-stream and must re-prompt. The