Studio: r19 fixes - cross-strip closed artifacts, iterate skeleton matches, bare-intent colon plan
- _has_answer_artifact() now strips closed code fences before checking for unclosed markup, and strips closed markup before checking for unclosed code fences. A Python / JS snippet containing literal "<html>" / "<svg>" strings no longer trips the unclosed-markup cross-check, and complete HTML containing a JS string with literal backticks no longer trips the unclosed-fence cross-check. - _looks_like_real_artifact() iterates every artifact match. An empty <html></html> / <svg></svg> skeleton followed by a real complete page no longer hides the real artifact. - _is_empty_markup_skeleton() strips an optional <!doctype ...> prefix before testing the empty-skeleton pattern, so "<!doctype html><html></html>" plan-only mentions also re-prompt. - _BARE_INTENT_NUMBERED_PLAN catches the tight "I'll:\n1. Open ..." / "Let me:\n1. Parse ..." shape where bare first-person intent + colon + newline is immediately followed by numbered action items. No work verb is required between the intent and the list.
This commit is contained in:
Daniel Han
parent
7c08ade3a4
commit
2ea2f3519a
2 changed files with 143 additions and 10 deletions
|
|
@ -119,6 +119,16 @@ _MAX_REPROMPTS = 3
|
|||
# `<html>` or `<!doctype>` do not bypass the re-prompt.
|
||||
# * All `[\s\S]{...}?` runs are length-bounded so the search stays
|
||||
# linear on adversarial input (CRLF spam, repeated `<html>` etc.).
|
||||
_CLOSED_CODE_FENCE = re.compile(
|
||||
r"(?<!`)(?P<bf>`{3,})(?!`)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=bf)`*[ \t]*(?:\r?\n|\Z)"
|
||||
r"|(?<!~)(?P<tf>~{3,})(?!~)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=tf)~*[ \t]*(?:\r?\n|\Z)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_CLOSED_MARKUP_ARTIFACT = re.compile(
|
||||
r"(?:<!doctype\b[\s\S]{0,200}?)?<html\b[\s\S]{0,4000}?</html>"
|
||||
r"|<svg\b[\s\S]{0,4000}?</svg>",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_HAS_ANSWER_ARTIFACT = re.compile(
|
||||
# Closed backtick code fence (any markdown info string, optional indent
|
||||
# on close). CommonMark allows opening fences of 3+ backticks; the
|
||||
|
|
@ -254,6 +264,18 @@ _EMPTY_MARKUP_SKELETON = re.compile(
|
|||
r"<(html|svg)\b[^>]*>\s*</\1>",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_DOCTYPE_PREFIX = re.compile(
|
||||
r"^<!doctype\b[\s\S]{0,200}?>",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _is_empty_markup_skeleton(matched: str) -> bool:
|
||||
"""True if ``matched`` is just an empty <html></html> / <svg></svg>
|
||||
(optionally with a `<!doctype>` prefix and surrounding whitespace).
|
||||
These read as plan-only mentions, not substantive answers."""
|
||||
candidate = _DOCTYPE_PREFIX.sub("", matched.strip(), count=1).strip()
|
||||
return _EMPTY_MARKUP_SKELETON.fullmatch(candidate) is not None
|
||||
|
||||
|
||||
# A numbered list whose item lines start with a strong work / tool
|
||||
|
|
@ -283,16 +305,34 @@ _STRONG_INTENT_BEFORE_LIST = re.compile(
|
|||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Bare first-person intent immediately followed by ``:`` and a numbered
|
||||
# list whose first item begins with a work verb. Catches
|
||||
# ``I'll:\n1. Open the URL`` and ``Let me:\n1. Parse the JSON`` where
|
||||
# no work verb appears between the intent phrase and the list. The
|
||||
# verb set here is broader than _LOCAL_ACTION_VERBS because the
|
||||
# tight ``intent + : + newline + numbered`` shape is itself the strong
|
||||
# signal that this is a tool stall.
|
||||
_BARE_INTENT_NUMBERED_PLAN = re.compile(
|
||||
r"\b(?:i['’](?:ll|m going to|m gonna)|i am (?:going to|gonna)|"
|
||||
r"i will|i shall|let me|allow me|now i|next i)\s*:[ \t]*"
|
||||
r"(?:\r?\n)[ \t]*\d+\.[ \t]+"
|
||||
r"(?:open|read|search|look up|check|verify|create|build|add|set up|"
|
||||
r"load|inspect|parse|calculate|compute|analy[sz]e|extract|run|execute|"
|
||||
r"fetch|download|query|summari[sz]e|implement|generate|draft|write)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _looks_like_real_artifact(text: str) -> bool:
|
||||
"""Match _HAS_ANSWER_ARTIFACT but reject empty markup skeletons."""
|
||||
m = _HAS_ANSWER_ARTIFACT.search(text)
|
||||
if not m:
|
||||
return False
|
||||
matched = m.group(0).strip()
|
||||
if _EMPTY_MARKUP_SKELETON.fullmatch(matched):
|
||||
return False
|
||||
return True
|
||||
"""Match _HAS_ANSWER_ARTIFACT but reject empty markup skeletons.
|
||||
|
||||
Iterates every artifact match in ``text`` so an empty <html></html>
|
||||
skeleton followed by a real complete page still classifies as a
|
||||
real artifact (the second match wins)."""
|
||||
for m in _HAS_ANSWER_ARTIFACT.finditer(text):
|
||||
if not _is_empty_markup_skeleton(m.group(0)):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _has_answer_artifact(text: str) -> bool:
|
||||
|
|
@ -310,9 +350,16 @@ def _has_answer_artifact(text: str) -> bool:
|
|||
answer, even when an earlier complete artifact is also present.
|
||||
Empty `<html></html>` / `<svg></svg>` skeletons do not count.
|
||||
"""
|
||||
if _has_unclosed_code_fence(text):
|
||||
# Cross-strip closed artifacts before the unclosed-state checks so
|
||||
# delimiter-like content INSIDE a complete code fence (e.g.
|
||||
# `html = '<html>'` literal in a Python snippet) or INSIDE complete
|
||||
# HTML (e.g. a JS string containing backticks) does not falsely
|
||||
# disqualify the artifact path.
|
||||
text_without_closed_fences = _CLOSED_CODE_FENCE.sub("", text)
|
||||
text_without_closed_markup = _CLOSED_MARKUP_ARTIFACT.sub("", text)
|
||||
if _has_unclosed_code_fence(text_without_closed_markup):
|
||||
return False
|
||||
if _has_unclosed_markup_block(text):
|
||||
if _has_unclosed_markup_block(text_without_closed_fences):
|
||||
return False
|
||||
if _looks_like_real_artifact(text):
|
||||
return True
|
||||
|
|
@ -321,6 +368,8 @@ def _has_answer_artifact(text: str) -> bool:
|
|||
return False
|
||||
if _DIRECT_NUMBERED_PLAN_FRAMING.search(text):
|
||||
return False
|
||||
if _BARE_INTENT_NUMBERED_PLAN.search(text):
|
||||
return False
|
||||
# First-person pronoun intent + numbered list where the items
|
||||
# themselves start with a strong work verb ("First, I'll:\n
|
||||
# 1. Load...\n2. Run...") is a plan stall, even when no work
|
||||
|
|
|
|||
|
|
@ -956,6 +956,90 @@ def test_reprompts_on_empty_html_or_svg_skeleton_mention():
|
|||
assert _would_reprompt(content), content
|
||||
|
||||
|
||||
def test_no_reprompt_on_code_fence_containing_markup_literal():
|
||||
"""A closed code fence whose body contains literal ``<html>``,
|
||||
``<svg>``, ``<body>`` strings is still a complete code answer.
|
||||
The unclosed-markup cross-check operates on text with closed
|
||||
fences stripped out so code literals do not falsely trip it."""
|
||||
samples = [
|
||||
(
|
||||
"First, let me write the scraper.\n"
|
||||
"```python\n"
|
||||
"html = '<html><body>'\n"
|
||||
"svg = \"<svg width='100'>\"\n"
|
||||
"print(html, svg)\n"
|
||||
"```"
|
||||
),
|
||||
(
|
||||
"First, let me write the parser.\n"
|
||||
"```javascript\n"
|
||||
"const open = '<html>';\n"
|
||||
"const fragment = '<svg width=\"10\">';\n"
|
||||
"console.log(open, fragment);\n"
|
||||
"```"
|
||||
),
|
||||
]
|
||||
for content in samples:
|
||||
assert _has_answer_artifact(content), content
|
||||
assert not _would_reprompt(content), content
|
||||
|
||||
|
||||
def test_no_reprompt_on_html_containing_backtick_literal():
|
||||
"""A complete <html> answer whose body contains a JS string with
|
||||
literal backticks is still a complete page. The unclosed-fence
|
||||
cross-check operates on text with closed markup stripped out."""
|
||||
content = (
|
||||
"First, here is the page.\n"
|
||||
"<html><body><script>const fence = '```';</script></body></html>"
|
||||
)
|
||||
assert _has_answer_artifact(content)
|
||||
assert not _would_reprompt(content)
|
||||
|
||||
|
||||
def test_empty_markup_before_real_artifact_still_counts_real_artifact():
|
||||
"""An empty <html></html> / <svg></svg> skeleton that PRECEDES a
|
||||
real complete artifact must not hide it. _looks_like_real_artifact
|
||||
iterates every match."""
|
||||
samples = [
|
||||
(
|
||||
"First, the minimal skeleton is <html></html>. "
|
||||
"Here is the full page: <html><body><h1>Hello</h1></body></html>"
|
||||
),
|
||||
(
|
||||
"First, the icon skeleton is <svg></svg>. "
|
||||
"Here is the full SVG: "
|
||||
"<svg width='10'><circle cx='5' cy='5' r='4'/></svg>"
|
||||
),
|
||||
]
|
||||
for content in samples:
|
||||
assert _has_answer_artifact(content), content
|
||||
assert not _would_reprompt(content), content
|
||||
|
||||
|
||||
def test_doctype_empty_html_skeleton_still_reprompts():
|
||||
"""``<!doctype html><html></html>`` is an empty skeleton even with
|
||||
a doctype prefix; the artifact check must reject it."""
|
||||
content = (
|
||||
"First, I'll create a <!doctype html><html></html> skeleton, then add CSS."
|
||||
)
|
||||
assert not _has_answer_artifact(content)
|
||||
assert _would_reprompt(content)
|
||||
|
||||
|
||||
def test_reprompts_on_bare_intent_colon_numbered_plan():
|
||||
"""Bare ``I'll:`` / ``Let me:`` immediately followed by a numbered
|
||||
list of work verbs is a tool stall regardless of whether the verbs
|
||||
sit before or in the list."""
|
||||
samples = [
|
||||
"I'll:\n1. Open the URL.\n2. Read the page.\n3. Summarize the answer.",
|
||||
"First, I'll:\n1. Create the Python file.\n2. Build the game loop.\n3. Test it.",
|
||||
"Let me:\n1. Parse the JSON.\n2. Calculate the average.",
|
||||
]
|
||||
for content in samples:
|
||||
assert not _has_answer_artifact(content), content
|
||||
assert _would_reprompt(content), content
|
||||
|
||||
|
||||
def test_reprompts_on_intent_plus_numbered_action_items():
|
||||
"""``First, I'll:\\n1. Load CSV\\n2. Compute total`` is a plan stall:
|
||||
the work verbs are in the list ITEMS even though no work verb
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue