Studio: r22 fixes - prose backtick guard, take/follow steps verbs
- _has_unclosed_code_fence() ignores a fence run when the trailing text on the same line starts with a space (typical English prose like "Use \`\`\` to start a markdown fence."). Real fence openers either end the line right after the delimiters or carry an info string with no leading space (\`\`\`python, \`\`\`bash-session). - _DIRECT_NUMBERED_PLAN_FRAMING accepts "take these steps", "follow these steps", and "perform these actions" as first-person intent verbs. Plans like "I'll take these steps:\n1. Open URL\n 2. Read" still re-prompt instead of being read as final answers.
This commit is contained in:
parent
394cdf48ea
commit
32b64aef97
2 changed files with 54 additions and 9 deletions
|
|
@ -198,7 +198,11 @@ _DIRECT_NUMBERED_PLAN_FRAMING = re.compile(
|
|||
r"[^\r\n]{0,160}"
|
||||
r"\b(?:open|read|search|look (?:this |that |it |them )?up|browse|"
|
||||
r"google|find|check|verify|compare|review|inspect|examine|"
|
||||
r"do (?:this|these|the following|it)|proceed|start|begin|"
|
||||
r"do (?:this|these|the following|it)|"
|
||||
r"take (?:these|the following) steps|"
|
||||
r"follow (?:these|the following) steps|"
|
||||
r"perform (?:these|the following) actions|"
|
||||
r"proceed|start|begin|"
|
||||
r"create|build|implement|set up|add|calculate|compute|analy[sz]e|"
|
||||
r"parse|load|run|execute|test)\b"
|
||||
r"|"
|
||||
|
|
@ -224,13 +228,12 @@ _FENCE_RUN_RE = re.compile(
|
|||
def _has_unclosed_code_fence(text: str) -> bool:
|
||||
"""True if ``text`` contains a code fence whose closer is missing.
|
||||
|
||||
A complete fence answer is already caught by _HAS_ANSWER_ARTIFACT.
|
||||
This helper exists so that an OPEN fence (model still streaming
|
||||
code, or stream cut short) does not let an embedded numbered list
|
||||
inside the fence body masquerade as a final answer. The scan is
|
||||
done per line and uses ``search`` (not ``match``) so an inline
|
||||
opener such as ``First, let me write it. \\`\\`\\`python`` is also
|
||||
tracked.
|
||||
Each line is scanned so inline openers like ``First. \\`\\`\\`python``
|
||||
are tracked. Prose mentions such as ``Use \\`\\`\\` to start a
|
||||
fence.`` are filtered out by requiring the trailing characters
|
||||
after the fence run to look like a CommonMark info string: empty,
|
||||
or starting with a non-space character (so prose with a leading
|
||||
space disqualifies the run).
|
||||
"""
|
||||
active_char: Optional[str] = None
|
||||
active_len = 0
|
||||
|
|
@ -239,8 +242,13 @@ def _has_unclosed_code_fence(text: str) -> bool:
|
|||
if not m:
|
||||
continue
|
||||
fence = m.group("backticks") or m.group("tildes")
|
||||
trailing = line[m.end() :].strip()
|
||||
raw_trailing = line[m.end():]
|
||||
trailing = raw_trailing.strip()
|
||||
ch = fence[0]
|
||||
# Prose mention guard: a real fence line never has a space
|
||||
# immediately after the delimiters followed by sentence text.
|
||||
if raw_trailing and raw_trailing[0] == " " and trailing:
|
||||
continue
|
||||
if active_char is None:
|
||||
active_char = ch
|
||||
active_len = len(fence)
|
||||
|
|
|
|||
|
|
@ -985,6 +985,43 @@ def test_no_reprompt_on_code_fence_containing_markup_literal():
|
|||
assert not _would_reprompt(content), content
|
||||
|
||||
|
||||
def test_reprompts_on_take_or_follow_steps_numbered_plan():
|
||||
"""``I'll take these steps:`` / ``I will follow these steps:`` +
|
||||
numbered list of work items is a plan stall."""
|
||||
samples = [
|
||||
(
|
||||
"I'll take these steps:\n"
|
||||
"1. Open the URL.\n"
|
||||
"2. Read the page.\n"
|
||||
"3. Summarize the answer."
|
||||
),
|
||||
(
|
||||
"I will follow these steps:\n"
|
||||
"1. Open the current docs.\n"
|
||||
"2. Read the relevant section.\n"
|
||||
"3. Answer."
|
||||
),
|
||||
]
|
||||
for content in samples:
|
||||
assert not _has_answer_artifact(content), content
|
||||
assert _would_reprompt(content), content
|
||||
|
||||
|
||||
def test_no_reprompt_on_prose_mention_of_triple_backticks_after_code():
|
||||
"""Closed code fence followed by prose that describes triple-
|
||||
backtick syntax (with leading space after the ticks) must NOT be
|
||||
treated as an unclosed fence."""
|
||||
content = (
|
||||
"Here is the snippet:\n"
|
||||
"```python\n"
|
||||
"print(1)\n"
|
||||
"```\n"
|
||||
"Use ``` to start a markdown code fence in your reply."
|
||||
)
|
||||
assert _has_answer_artifact(content)
|
||||
assert not _would_reprompt(content)
|
||||
|
||||
|
||||
def test_reprompts_on_direct_first_person_read_check_open_plan():
|
||||
"""Direct first-person intent + open/read/check/review/inspect verbs
|
||||
+ numbered list is a tool stall. The broader verb set applies to
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue