diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 63f26a51e0..589ecd2f52 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -357,11 +357,19 @@ def _has_answer_artifact(text: str) -> bool: # disqualify the artifact path. text_without_closed_fences = _CLOSED_CODE_FENCE.sub("", text) text_without_closed_markup = _CLOSED_MARKUP_ARTIFACT.sub("", text) + text_without_both = _CLOSED_MARKUP_ARTIFACT.sub("", text_without_closed_fences) if _has_unclosed_code_fence(text_without_closed_markup): return False - if _has_unclosed_markup_block(text_without_closed_fences): + # When NO complete artifact has been emitted yet, count-based markup + # detection is reliable for spotting mid-stream output. Once a real + # artifact already exists, prose mentions of bare ```` / + # ```` tags in explanations are common (and would falsely + # unbalance the open/close count), so we rely on the closed-artifact + # path instead and skip the count check. + real_artifact = _looks_like_real_artifact(text) + if not real_artifact and _has_unclosed_markup_block(text_without_both): return False - if _looks_like_real_artifact(text): + if real_artifact: return True if _NUMBERED_LIST_ARTIFACT.search(text): if _EXPLICIT_PLAN_HEADER.search(text): diff --git a/studio/backend/tests/test_llama_cpp_reprompt_guard.py b/studio/backend/tests/test_llama_cpp_reprompt_guard.py index 683e433e64..aa51669955 100644 --- a/studio/backend/tests/test_llama_cpp_reprompt_guard.py +++ b/studio/backend/tests/test_llama_cpp_reprompt_guard.py @@ -931,17 +931,18 @@ def test_reprompts_on_incomplete_html_with_inner_numbered_list(): assert _would_reprompt(content), content -def test_reprompts_when_complete_html_is_followed_by_open_html(): - """A response with one closed followed by a second - that is still open must re-prompt. The unbalanced-tag count makes - the artifact path fail even though an earlier artifact exists.""" +def test_complete_html_with_trailing_prose_tag_still_counts(): + """A complete answer followed by prose that mentions + or tags (explanatory text) stays a complete artifact. The + unbalanced-tag count is skipped once a real artifact exists so + common explanatory prose does not falsely wipe valid answers.""" samples = [ - "Here is the first page:\n1\nNow the next:\n", - "First page done:\n\nNow:\n", + "Here is the page:\n1\nUse the tag for the root.", + "Here is the SVG: Place it inside an page.", ] for content in samples: - assert not _has_answer_artifact(content), content - assert _would_reprompt(content), content + assert _has_answer_artifact(content), content + assert not _would_reprompt(content), content def test_reprompts_on_empty_html_or_svg_skeleton_mention(): @@ -984,6 +985,34 @@ def test_no_reprompt_on_code_fence_containing_markup_literal(): assert not _would_reprompt(content), content +def test_no_reprompt_on_html_with_inner_svg_or_self_closing_tag(): + """Complete answers that contain nested SVG / self-closing + tags are still complete pages. The unbalanced-count cross-check is + skipped when a real artifact already exists.""" + samples = [ + "", + "" + "" + "", + ] + for content in samples: + assert _has_answer_artifact(content), content + assert not _would_reprompt(content), content + + +def test_no_reprompt_on_complete_artifact_with_prose_tag_mention(): + """Complete code/markup artifacts followed by ordinary prose that + mentions ```` or ```` tags are not mid-stream output.""" + samples = [ + "hi\nUse the tag as the root.", + ( + "First, here is the SVG: \n" + "Put it inside an page if needed." + ), + ] + for content in samples: + assert _has_answer_artifact(content), content + assert not _would_reprompt(content), content + + def test_no_reprompt_on_html_containing_backtick_literal(): """A complete answer whose body contains a JS string with literal backticks is still a complete page. The unclosed-fence