diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index ff7b1eb598..981df074eb 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -119,6 +119,16 @@ _MAX_REPROMPTS = 3 # `` or `` do not bypass the re-prompt. # * All `[\s\S]{...}?` runs are length-bounded so the search stays # linear on adversarial input (CRLF spam, repeated `` etc.). +_CLOSED_CODE_FENCE = re.compile( + r"(?`{3,})(?!`)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=bf)`*[ \t]*(?:\r?\n|\Z)" + r"|(?~{3,})(?!~)[^\r\n]{0,200}\r?\n[\s\S]{1,4000}?\r?\n[ \t]*(?P=tf)~*[ \t]*(?:\r?\n|\Z)", + re.IGNORECASE, +) +_CLOSED_MARKUP_ARTIFACT = re.compile( + r"(?:" + r"|", + re.IGNORECASE, +) _HAS_ANSWER_ARTIFACT = re.compile( # Closed backtick code fence (any markdown info string, optional indent # on close). CommonMark allows opening fences of 3+ backticks; the @@ -254,6 +264,18 @@ _EMPTY_MARKUP_SKELETON = re.compile( r"<(html|svg)\b[^>]*>\s*", re.IGNORECASE, ) +_DOCTYPE_PREFIX = re.compile( + r"^", + re.IGNORECASE, +) + + +def _is_empty_markup_skeleton(matched: str) -> bool: + """True if ``matched`` is just an empty / + (optionally with a `` prefix and surrounding whitespace). + These read as plan-only mentions, not substantive answers.""" + candidate = _DOCTYPE_PREFIX.sub("", matched.strip(), count=1).strip() + return _EMPTY_MARKUP_SKELETON.fullmatch(candidate) is not None # A numbered list whose item lines start with a strong work / tool @@ -283,16 +305,34 @@ _STRONG_INTENT_BEFORE_LIST = re.compile( re.IGNORECASE, ) +# Bare first-person intent immediately followed by ``:`` and a numbered +# list whose first item begins with a work verb. Catches +# ``I'll:\n1. Open the URL`` and ``Let me:\n1. Parse the JSON`` where +# no work verb appears between the intent phrase and the list. The +# verb set here is broader than _LOCAL_ACTION_VERBS because the +# tight ``intent + : + newline + numbered`` shape is itself the strong +# signal that this is a tool stall. +_BARE_INTENT_NUMBERED_PLAN = re.compile( + r"\b(?:i['’](?:ll|m going to|m gonna)|i am (?:going to|gonna)|" + r"i will|i shall|let me|allow me|now i|next i)\s*:[ \t]*" + r"(?:\r?\n)[ \t]*\d+\.[ \t]+" + r"(?:open|read|search|look up|check|verify|create|build|add|set up|" + r"load|inspect|parse|calculate|compute|analy[sz]e|extract|run|execute|" + r"fetch|download|query|summari[sz]e|implement|generate|draft|write)\b", + re.IGNORECASE, +) + def _looks_like_real_artifact(text: str) -> bool: - """Match _HAS_ANSWER_ARTIFACT but reject empty markup skeletons.""" - m = _HAS_ANSWER_ARTIFACT.search(text) - if not m: - return False - matched = m.group(0).strip() - if _EMPTY_MARKUP_SKELETON.fullmatch(matched): - return False - return True + """Match _HAS_ANSWER_ARTIFACT but reject empty markup skeletons. + + Iterates every artifact match in ``text`` so an empty + skeleton followed by a real complete page still classifies as a + real artifact (the second match wins).""" + for m in _HAS_ANSWER_ARTIFACT.finditer(text): + if not _is_empty_markup_skeleton(m.group(0)): + return True + return False def _has_answer_artifact(text: str) -> bool: @@ -310,9 +350,16 @@ def _has_answer_artifact(text: str) -> bool: answer, even when an earlier complete artifact is also present. Empty `` / `` skeletons do not count. """ - if _has_unclosed_code_fence(text): + # Cross-strip closed artifacts before the unclosed-state checks so + # delimiter-like content INSIDE a complete code fence (e.g. + # `html = ''` literal in a Python snippet) or INSIDE complete + # HTML (e.g. a JS string containing backticks) does not falsely + # disqualify the artifact path. + text_without_closed_fences = _CLOSED_CODE_FENCE.sub("", text) + text_without_closed_markup = _CLOSED_MARKUP_ARTIFACT.sub("", text) + if _has_unclosed_code_fence(text_without_closed_markup): return False - if _has_unclosed_markup_block(text): + if _has_unclosed_markup_block(text_without_closed_fences): return False if _looks_like_real_artifact(text): return True @@ -321,6 +368,8 @@ def _has_answer_artifact(text: str) -> bool: return False if _DIRECT_NUMBERED_PLAN_FRAMING.search(text): return False + if _BARE_INTENT_NUMBERED_PLAN.search(text): + return False # First-person pronoun intent + numbered list where the items # themselves start with a strong work verb ("First, I'll:\n # 1. Load...\n2. Run...") is a plan stall, even when no work diff --git a/studio/backend/tests/test_llama_cpp_reprompt_guard.py b/studio/backend/tests/test_llama_cpp_reprompt_guard.py index 00ba9f5ba1..683e433e64 100644 --- a/studio/backend/tests/test_llama_cpp_reprompt_guard.py +++ b/studio/backend/tests/test_llama_cpp_reprompt_guard.py @@ -956,6 +956,90 @@ def test_reprompts_on_empty_html_or_svg_skeleton_mention(): assert _would_reprompt(content), content +def test_no_reprompt_on_code_fence_containing_markup_literal(): + """A closed code fence whose body contains literal ````, + ````, ```` strings is still a complete code answer. + The unclosed-markup cross-check operates on text with closed + fences stripped out so code literals do not falsely trip it.""" + samples = [ + ( + "First, let me write the scraper.\n" + "```python\n" + "html = ''\n" + "svg = \"\"\n" + "print(html, svg)\n" + "```" + ), + ( + "First, let me write the parser.\n" + "```javascript\n" + "const open = '';\n" + "const fragment = '';\n" + "console.log(open, fragment);\n" + "```" + ), + ] + for content in samples: + assert _has_answer_artifact(content), content + assert not _would_reprompt(content), content + + +def test_no_reprompt_on_html_containing_backtick_literal(): + """A complete answer whose body contains a JS string with + literal backticks is still a complete page. The unclosed-fence + cross-check operates on text with closed markup stripped out.""" + content = ( + "First, here is the page.\n" + "" + ) + assert _has_answer_artifact(content) + assert not _would_reprompt(content) + + +def test_empty_markup_before_real_artifact_still_counts_real_artifact(): + """An empty / skeleton that PRECEDES a + real complete artifact must not hide it. _looks_like_real_artifact + iterates every match.""" + samples = [ + ( + "First, the minimal skeleton is . " + "Here is the full page:

Hello

" + ), + ( + "First, the icon skeleton is . " + "Here is the full SVG: " + "" + ), + ] + for content in samples: + assert _has_answer_artifact(content), content + assert not _would_reprompt(content), content + + +def test_doctype_empty_html_skeleton_still_reprompts(): + """```` is an empty skeleton even with + a doctype prefix; the artifact check must reject it.""" + content = ( + "First, I'll create a skeleton, then add CSS." + ) + assert not _has_answer_artifact(content) + assert _would_reprompt(content) + + +def test_reprompts_on_bare_intent_colon_numbered_plan(): + """Bare ``I'll:`` / ``Let me:`` immediately followed by a numbered + list of work verbs is a tool stall regardless of whether the verbs + sit before or in the list.""" + samples = [ + "I'll:\n1. Open the URL.\n2. Read the page.\n3. Summarize the answer.", + "First, I'll:\n1. Create the Python file.\n2. Build the game loop.\n3. Test it.", + "Let me:\n1. Parse the JSON.\n2. Calculate the average.", + ] + for content in samples: + assert not _has_answer_artifact(content), content + assert _would_reprompt(content), content + + def test_reprompts_on_intent_plus_numbered_action_items(): """``First, I'll:\\n1. Load CSV\\n2. Compute total`` is a plan stall: the work verbs are in the list ITEMS even though no work verb