Studio: r16 fixes - inline fence tracking, broaden direct-intent plan with first/step prefixes
- _has_unclosed_code_fence() now scans every line with re.search and a shared FENCE_RUN regex, so an inline opening fence such as "First, let me write it. \`\`\`python" is tracked alongside the column-0 openers. A numbered list emitted INSIDE an inline-open fence no longer reads as a final answer. - _DIRECT_NUMBERED_PLAN_FRAMING adds "first" and "step N(:?)" to its intent prefixes and "look up" to its verb whitelist. Plans like "First, analyze the uploaded CSV:\n1. Load rows\n2. Compute total" or "I'll look that up:\n1. Search the docs" now re-prompt instead of being mis-classified as final answers. The verb whitelist still excludes bare search/find/check/verify so "First, use binary search:\n1. Search the left half" stays an answer.
This commit is contained in:
parent
9fa736bef1
commit
7b0ed8333a
2 changed files with 76 additions and 6 deletions
|
|
@ -180,9 +180,11 @@ _EXPLICIT_PLAN_HEADER = re.compile(
|
|||
# poem" is not misclassified as a stall.
|
||||
_DIRECT_NUMBERED_PLAN_FRAMING = re.compile(
|
||||
r"\b(?:i['’](?:ll|m going to|m gonna)|i am (?:going to|gonna)|"
|
||||
r"i will|i shall|let me|allow me|now i|next i)\b"
|
||||
r"i will|i shall|let me|allow me|now i|next i|"
|
||||
r"first|step \d+:?)\b"
|
||||
r"[^\r\n]{0,160}"
|
||||
r"\b(?:do (?:this|these|the following|it)|"
|
||||
r"look (?:this |that |it |them )?up|"
|
||||
r"proceed|start|begin|"
|
||||
r"create|build|implement|set up|add|"
|
||||
r"calculate|compute|analy[sz]e|parse|load|run|execute|test)\b"
|
||||
|
|
@ -192,7 +194,9 @@ _DIRECT_NUMBERED_PLAN_FRAMING = re.compile(
|
|||
)
|
||||
|
||||
|
||||
_FENCE_LINE_RE = re.compile(r"^[ \t]*(?P<fence>`{3,}|~{3,})(?P<trailing>[^\r\n]*)$")
|
||||
_FENCE_RUN_RE = re.compile(
|
||||
r"(?<!`)(?P<backticks>`{3,})(?!`)|(?<!~)(?P<tildes>~{3,})(?!~)"
|
||||
)
|
||||
|
||||
|
||||
def _has_unclosed_code_fence(text: str) -> bool:
|
||||
|
|
@ -201,16 +205,19 @@ def _has_unclosed_code_fence(text: str) -> bool:
|
|||
A complete fence answer is already caught by _HAS_ANSWER_ARTIFACT.
|
||||
This helper exists so that an OPEN fence (model still streaming
|
||||
code, or stream cut short) does not let an embedded numbered list
|
||||
inside the fence body masquerade as a final answer.
|
||||
inside the fence body masquerade as a final answer. The scan is
|
||||
done per line and uses ``search`` (not ``match``) so an inline
|
||||
opener such as ``First, let me write it. \\`\\`\\`python`` is also
|
||||
tracked.
|
||||
"""
|
||||
active_char: Optional[str] = None
|
||||
active_len = 0
|
||||
for line in text.splitlines():
|
||||
m = _FENCE_LINE_RE.match(line)
|
||||
m = _FENCE_RUN_RE.search(line)
|
||||
if not m:
|
||||
continue
|
||||
fence = m.group("fence")
|
||||
trailing = m.group("trailing").strip()
|
||||
fence = m.group("backticks") or m.group("tildes")
|
||||
trailing = line[m.end():].strip()
|
||||
ch = fence[0]
|
||||
if active_char is None:
|
||||
active_char = ch
|
||||
|
|
|
|||
|
|
@ -869,6 +869,69 @@ def test_no_reprompt_on_let_me_explain_numbered_answer():
|
|||
assert not _would_reprompt(content), content
|
||||
|
||||
|
||||
def test_same_line_open_fence_with_numbered_body_still_reprompts():
|
||||
"""An OPEN code fence on the same line as preceding prose ("First,
|
||||
let me write it. ``\\u00e0``text\\n...") still gates the numbered-list
|
||||
fallback. The unclosed-fence helper now uses ``search`` so inline
|
||||
openers are tracked, not just openers at column 0."""
|
||||
content = (
|
||||
"First, let me write it. ```text\n"
|
||||
"1. Install dependencies\n"
|
||||
"2. Run the app"
|
||||
)
|
||||
assert not _has_answer_artifact(content)
|
||||
assert _would_reprompt(content)
|
||||
|
||||
|
||||
def test_reprompts_on_first_step_numbered_compute_plan():
|
||||
"""Bare ``First, [verb]`` / ``Step N: [verb]`` followed by a numbered
|
||||
list is a plan stall when the verb implies compute / tool work
|
||||
(analyze, parse, calculate, create, etc.). Distinct from
|
||||
``First, use binary search:`` (verb ``use`` not in whitelist)."""
|
||||
samples = [
|
||||
(
|
||||
"First, analyze the uploaded CSV:\n"
|
||||
"1. Load the rows.\n"
|
||||
"2. Compute the average revenue."
|
||||
),
|
||||
(
|
||||
"First, parse the pasted JSON:\n"
|
||||
"1. Load the object.\n"
|
||||
"2. Calculate the total."
|
||||
),
|
||||
(
|
||||
"First, create the Python game:\n"
|
||||
"1. Set up pygame.\n"
|
||||
"2. Add the game loop."
|
||||
),
|
||||
(
|
||||
"Step 1: analyze the uploaded CSV:\n"
|
||||
"1. Load rows.\n"
|
||||
"2. Compute the total."
|
||||
),
|
||||
(
|
||||
"I'll look that up:\n"
|
||||
"1. Search the docs.\n"
|
||||
"2. Summarize the result."
|
||||
),
|
||||
]
|
||||
for content in samples:
|
||||
assert _would_reprompt(content), content
|
||||
|
||||
|
||||
def test_no_reprompt_on_first_use_binary_search_answer():
|
||||
"""``First, use binary search:`` is an ordinary algorithm answer.
|
||||
``use`` is not in the direct-numbered-plan verb whitelist so the
|
||||
following list stays an answer."""
|
||||
content = (
|
||||
"First, use binary search:\n"
|
||||
"1. Search the left half.\n"
|
||||
"2. Search the right half."
|
||||
)
|
||||
assert _has_answer_artifact(content)
|
||||
assert not _would_reprompt(content)
|
||||
|
||||
|
||||
def test_reprompts_when_later_fence_is_open_after_closed_fence():
|
||||
"""A response with a complete code fence followed by a SECOND,
|
||||
unclosed fence is still mid-stream and must re-prompt. The
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue