From 6ea922a58d80725b5676522a2c533163312dbf1f Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 21:34:52 +0000 Subject: [PATCH] Studio: r20 fixes - skip markup-count when real artifact exists When a real complete artifact is already in the response, prose mentions of bare / tags in explanatory text are common (for example "Use the tag for the root"). The unbalanced- open/close count would falsely classify the response as mid-stream and wipe the valid answer. The artifact-counting cross-check now only runs when NO real artifact has been emitted yet; once a real artifact exists, mid-stream second markup is rare enough that the count-based detector is not worth the false-positive cost. This also unblocks complete answers that nest children or contain JS string literals like "", since those unmatched markup tokens were being flagged as unclosed. --- studio/backend/core/inference/llama_cpp.py | 12 ++++- .../tests/test_llama_cpp_reprompt_guard.py | 45 +++++++++++++++---- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 63f26a51e0..589ecd2f52 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -357,11 +357,19 @@ def _has_answer_artifact(text: str) -> bool: # disqualify the artifact path. text_without_closed_fences = _CLOSED_CODE_FENCE.sub("", text) text_without_closed_markup = _CLOSED_MARKUP_ARTIFACT.sub("", text) + text_without_both = _CLOSED_MARKUP_ARTIFACT.sub("", text_without_closed_fences) if _has_unclosed_code_fence(text_without_closed_markup): return False - if _has_unclosed_markup_block(text_without_closed_fences): + # When NO complete artifact has been emitted yet, count-based markup + # detection is reliable for spotting mid-stream output. Once a real + # artifact already exists, prose mentions of bare ```` / + # ```` tags in explanations are common (and would falsely + # unbalance the open/close count), so we rely on the closed-artifact + # path instead and skip the count check. + real_artifact = _looks_like_real_artifact(text) + if not real_artifact and _has_unclosed_markup_block(text_without_both): return False - if _looks_like_real_artifact(text): + if real_artifact: return True if _NUMBERED_LIST_ARTIFACT.search(text): if _EXPLICIT_PLAN_HEADER.search(text): diff --git a/studio/backend/tests/test_llama_cpp_reprompt_guard.py b/studio/backend/tests/test_llama_cpp_reprompt_guard.py index 683e433e64..aa51669955 100644 --- a/studio/backend/tests/test_llama_cpp_reprompt_guard.py +++ b/studio/backend/tests/test_llama_cpp_reprompt_guard.py @@ -931,17 +931,18 @@ def test_reprompts_on_incomplete_html_with_inner_numbered_list(): assert _would_reprompt(content), content -def test_reprompts_when_complete_html_is_followed_by_open_html(): - """A response with one closed followed by a second - that is still open must re-prompt. The unbalanced-tag count makes - the artifact path fail even though an earlier artifact exists.""" +def test_complete_html_with_trailing_prose_tag_still_counts(): + """A complete answer followed by prose that mentions + or tags (explanatory text) stays a complete artifact. The + unbalanced-tag count is skipped once a real artifact exists so + common explanatory prose does not falsely wipe valid answers.""" samples = [ - "Here is the first page:\n1\nNow the next:\n", - "First page done:\n\nNow:\n", + "Here is the page:\n1\nUse the tag for the root.", + "Here is the SVG: Place it inside an page.", ] for content in samples: - assert not _has_answer_artifact(content), content - assert _would_reprompt(content), content + assert _has_answer_artifact(content), content + assert not _would_reprompt(content), content def test_reprompts_on_empty_html_or_svg_skeleton_mention(): @@ -984,6 +985,34 @@ def test_no_reprompt_on_code_fence_containing_markup_literal(): assert not _would_reprompt(content), content +def test_no_reprompt_on_html_with_inner_svg_or_self_closing_tag(): + """Complete answers that contain nested SVG / self-closing + tags are still complete pages. The unbalanced-count cross-check is + skipped when a real artifact already exists.""" + samples = [ + "", + "" + "" + "", + ] + for content in samples: + assert _has_answer_artifact(content), content + assert not _would_reprompt(content), content + + +def test_no_reprompt_on_complete_artifact_with_prose_tag_mention(): + """Complete code/markup artifacts followed by ordinary prose that + mentions ```` or ```` tags are not mid-stream output.""" + samples = [ + "hi\nUse the tag as the root.", + ( + "First, here is the SVG: \n" + "Put it inside an page if needed." + ), + ] + for content in samples: + assert _has_answer_artifact(content), content + assert not _would_reprompt(content), content + + def test_no_reprompt_on_html_containing_backtick_literal(): """A complete answer whose body contains a JS string with literal backticks is still a complete page. The unclosed-fence