ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Studio: r20 fixes - skip markup-count when real artifact exists

Browse source 
When a real complete artifact is already in the response, prose
mentions of bare <html> / <svg> tags in explanatory text are common
(for example "Use the <html> tag for the root"). The unbalanced-
open/close count would falsely classify the response as mid-stream
and wipe the valid answer. The artifact-counting cross-check now
only runs when NO real artifact has been emitted yet; once a real
artifact exists, mid-stream second markup is rare enough that the
count-based detector is not worth the false-positive cost.

This also unblocks complete <html> answers that nest <svg> children
or contain JS string literals like "<svg width=10>", since those
unmatched markup tokens were being flagged as unclosed.
This commit is contained in:
Daniel Han 2026-05-24 21:34:52 +00:00
commit 6ea922a58d
2 changed files with 47 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

12
studio/backend/core/inference/llama_cpp.py
View file

@ -357,11 +357,19 @@ def _has_answer_artifact(text: str) -> bool:
# disqualify the artifact path.
text_without_closed_fences = _CLOSED_CODE_FENCE.sub("", text)
text_without_closed_markup = _CLOSED_MARKUP_ARTIFACT.sub("", text)
text_without_both = _CLOSED_MARKUP_ARTIFACT.sub("", text_without_closed_fences)
if _has_unclosed_code_fence(text_without_closed_markup):
return False
if _has_unclosed_markup_block(text_without_closed_fences):
# When NO complete artifact has been emitted yet, count-based markup
# detection is reliable for spotting mid-stream output. Once a real
# artifact already exists, prose mentions of bare ``<html>`` /
# ``<svg>`` tags in explanations are common (and would falsely
# unbalance the open/close count), so we rely on the closed-artifact
# path instead and skip the count check.
real_artifact = _looks_like_real_artifact(text)
if not real_artifact and _has_unclosed_markup_block(text_without_both):
return False
if _looks_like_real_artifact(text):
if real_artifact:
return True
if _NUMBERED_LIST_ARTIFACT.search(text):
if _EXPLICIT_PLAN_HEADER.search(text):

45
studio/backend/tests/test_llama_cpp_reprompt_guard.py
View file

@ -931,17 +931,18 @@ def test_reprompts_on_incomplete_html_with_inner_numbered_list():
assert _would_reprompt(content), content
def test_reprompts_when_complete_html_is_followed_by_open_html():
"""A response with one closed <html> followed by a second <html>
that is still open must re-prompt. The unbalanced-tag count makes
the artifact path fail even though an earlier artifact exists."""
def test_complete_html_with_trailing_prose_tag_still_counts():
"""A complete <html> answer followed by prose that mentions <html>
or <svg> tags (explanatory text) stays a complete artifact. The
unbalanced-tag count is skipped once a real artifact exists so
common explanatory prose does not falsely wipe valid answers."""
samples = [
"Here is the first page:\n<html><body>1</body></html>\nNow the next:\n<html><body>",
"First page done:\n<svg width='10'><circle/></svg>\nNow:\n<svg width='10'>",
"Here is the page:\n<html><body>1</body></html>\nUse the <html> tag for the root.",
"Here is the SVG: <svg width='10'><circle/></svg> Place it inside an <html> page.",
]
for content in samples:
assert not _has_answer_artifact(content), content
assert _would_reprompt(content), content
assert _has_answer_artifact(content), content
assert not _would_reprompt(content), content
def test_reprompts_on_empty_html_or_svg_skeleton_mention():
@ -984,6 +985,34 @@ def test_no_reprompt_on_code_fence_containing_markup_literal():
assert not _would_reprompt(content), content
def test_no_reprompt_on_html_with_inner_svg_or_self_closing_tag():
"""Complete <html> answers that contain nested SVG / self-closing
tags are still complete pages. The unbalanced-count cross-check is
skipped when a real artifact already exists."""
samples = [
"<html><body><svg width='10'/></body></html>",
"<html><body>" + "<script>const s = '<svg width=10>';</script>" + "</body></html>",
]
for content in samples:
assert _has_answer_artifact(content), content
assert not _would_reprompt(content), content
def test_no_reprompt_on_complete_artifact_with_prose_tag_mention():
"""Complete code/markup artifacts followed by ordinary prose that
mentions ``<html>`` or ``<svg>`` tags are not mid-stream output."""
samples = [
"<html><body>hi</body></html>\nUse the <html> tag as the root.",
(
"First, here is the SVG: <svg width='10'><circle/></svg>\n"
"Put it inside an <html> page if needed."
),
]
for content in samples:
assert _has_answer_artifact(content), content
assert not _would_reprompt(content), content
def test_no_reprompt_on_html_containing_backtick_literal():
"""A complete <html> answer whose body contains a JS string with
literal backticks is still a complete page. The unclosed-fence