diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index bf8a3c04df..ed22e329d6 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -71,6 +71,23 @@ _INTENT_SIGNAL = re.compile( ) _MAX_REPROMPTS = 3 +# Substantive answer artifacts. Re-prompt fires when the model emits +# intent-only language ("first I'll ...", "let me ...") without a tool +# call, but the same intent words appear in long explanations that +# accompany REAL code or markup. Without this guard, a complete reply +# like "First, let me set up pygame. ```python ... ```" trips the +# re-prompt and the next user-visible message wipes the code. We +# require ALL of (intent signal, length < _REPROMPT_MAX_CHARS, no +# answer artifact) to fire. +_HAS_ANSWER_ARTIFACT = re.compile( + r"```[a-zA-Z]*\n[\s\S]+?\n```" # closed code fence + r"|" # complete SVG + r"|(?:^|\n)\s*\d+\.\s+\S.*?\n\s*\d+\.", # 2+ numbered list items + re.IGNORECASE, +) + # Without max_tokens, llama-server defaults to n_predict = n_ctx (up to # 262144 for Qwen3.5), producing many-minute zombie decodes when cancel # fails. t_max_predict_ms is a wall-clock backstop applied unconditionally, @@ -4818,6 +4835,7 @@ class LlamaCppBackend: and _reprompt_count < _MAX_REPROMPTS and 0 < len(_stripped) < _REPROMPT_MAX_CHARS and _INTENT_SIGNAL.search(_stripped) + and not _HAS_ANSWER_ARTIFACT.search(_stripped) ): _reprompt_count += 1 logger.info( diff --git a/studio/backend/tests/test_llama_cpp_reprompt_guard.py b/studio/backend/tests/test_llama_cpp_reprompt_guard.py new file mode 100644 index 0000000000..fbe12e9727 --- /dev/null +++ b/studio/backend/tests/test_llama_cpp_reprompt_guard.py @@ -0,0 +1,208 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Tests for the plan-without-action re-prompt guard. + +The re-prompt path in ``LlamaCppEngine.chat_stream`` exists to nudge a +model that described what it *will* do (forward-looking language) +without actually calling a tool. Before the guard added in this PR, the +heuristic only checked ``len(content) < _REPROMPT_MAX_CHARS`` and the +intent regex, which over-fired on long-but-complete responses that +happened to contain phrases like "first" or "let me". Specifically, a +correct Python game answer of the form :: + + First, let me set up pygame. + ```python + import pygame; ... + ``` + +would still match (length < 2000, intent signal present) and the next +synthetic user turn ("STOP. Do NOT write code or explain.") wiped the +visible code from the conversation. + +The new ``_HAS_ANSWER_ARTIFACT`` regex blocks the re-prompt whenever +the response already contains a real answer artifact: a closed code +fence, an HTML page, a complete SVG, or a numbered list of items. +""" + +from __future__ import annotations + +import sys +import types as _types +from pathlib import Path + +_BACKEND_DIR = str(Path(__file__).resolve().parent.parent) +if _BACKEND_DIR not in sys.path: + sys.path.insert(0, _BACKEND_DIR) + +_loggers_stub = _types.ModuleType("loggers") +_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name) +sys.modules.setdefault("loggers", _loggers_stub) + +_structlog_stub = _types.ModuleType("structlog") +_structlog_stub.get_logger = lambda *a, **k: __import__("logging").getLogger("stub") +sys.modules.setdefault("structlog", _structlog_stub) + +from core.inference.llama_cpp import ( # noqa: E402 + _HAS_ANSWER_ARTIFACT, + _INTENT_SIGNAL, +) + + +# ── _INTENT_SIGNAL still matches plan-only stalls ────────────────── + + +def test_intent_signal_matches_plan_only_phrases(): + """Original behaviour is preserved: intent regex still matches the + plan-without-action phrases that motivated the re-prompt.""" + plan_only_samples = [ + "I'll search the web for that.", + "I will look that up.", + "I am going to search.", + "Let me search the web for the answer.", + "First, I need to look up the date.", + "Step 1: I'll search for the song list.", + "Now I need to call the tool.", + "Here's my plan: search for X.", + ] + for s in plan_only_samples: + assert _INTENT_SIGNAL.search(s), f"_INTENT_SIGNAL should match {s!r}" + + +def test_intent_signal_ignores_direct_answers(): + """Direct, complete answers do not match the intent regex.""" + direct_samples = [ + "4", + "Hello!", + "The answer is 42.", + "The capital of France is Paris.", + ] + for s in direct_samples: + assert not _INTENT_SIGNAL.search(s), f"_INTENT_SIGNAL must not match {s!r}" + + +# ── _HAS_ANSWER_ARTIFACT recognises substantive content ──────────── + + +def test_artifact_regex_detects_closed_code_fence(): + """Closed Python code fence is an answer artifact.""" + text = "First, let me set up pygame.\n```python\nimport pygame\npygame.init()\n```" + assert _HAS_ANSWER_ARTIFACT.search(text), ( + "Closed code fence must be detected as an answer artifact" + ) + + +def test_artifact_regex_detects_html_page(): + """HTML pages (doctype or root) are answer artifacts.""" + text_a = "
" + text_b = "Sure, here is the dashboard:\n..." + assert _HAS_ANSWER_ARTIFACT.search(text_a) + assert _HAS_ANSWER_ARTIFACT.search(text_b) + + +def test_artifact_regex_detects_complete_svg(): + """A complete is an answer artifact.""" + text = ( + "Here is the sloth SVG:\n" + "" + ) + assert _HAS_ANSWER_ARTIFACT.search(text) + + +def test_artifact_regex_detects_numbered_list(): + """A list of 2+ numbered items is an answer artifact.""" + text = ( + "Let me list these:\n" + "1. Animals — Maroon 5\n" + "2. Take Me to Church — Hozier\n" + "3. Love Me Like You Do — Ellie Goulding\n" + ) + assert _HAS_ANSWER_ARTIFACT.search(text) + + +def test_artifact_regex_ignores_open_code_fence(): + """An UNCLOSED code fence is not yet a complete artifact.""" + text = "Let me set up pygame.\n```python\nimport pygame" + assert not _HAS_ANSWER_ARTIFACT.search(text), ( + "Open code fence must not satisfy the artifact guard" + ) + + +def test_artifact_regex_ignores_plain_text(): + """Plain conversational text contains no artifact.""" + text = "First, I will search for the songs that charted #3 in 2015." + assert not _HAS_ANSWER_ARTIFACT.search(text) + + +# ── End-to-end guard semantics on realistic responses ────────────── + + +def _would_reprompt(content: str) -> bool: + """Return True if the re-prompt block at llama_cpp.py would fire.""" + from core.inference.llama_cpp import _REPROMPT_MAX_CHARS + stripped = content.strip() + return bool( + 0 < len(stripped) < _REPROMPT_MAX_CHARS + and _INTENT_SIGNAL.search(stripped) + and not _HAS_ANSWER_ARTIFACT.search(stripped) + ) + + +def test_no_reprompt_on_complete_python_game(): + """Response with intent phrasing + complete code does NOT re-prompt.""" + content = ( + "First, let me set up pygame.\n" + "```python\n" + "import pygame\n" + "pygame.init()\n" + "screen = pygame.display.set_mode((640, 480))\n" + "while True:\n" + " for e in pygame.event.get():\n" + " if e.type == pygame.QUIT: break\n" + "```" + ) + assert not _would_reprompt(content), ( + "Re-prompt must not fire after a complete code block was produced" + ) + + +def test_no_reprompt_on_complete_svg(): + """Response with intent phrasing + complete SVG does NOT re-prompt.""" + content = ( + "Let me draw a cute sloth:\n" + "" + ) + assert not _would_reprompt(content) + + +def test_no_reprompt_on_numbered_list_answer(): + """Response with intent + numbered list (Billboard-style) does NOT re-prompt.""" + content = ( + "Here's my list of #3 hits:\n" + "1. Animals — Maroon 5\n" + "2. Take Me to Church — Hozier\n" + "3. Drag Me Down — One Direction\n" + ) + assert not _would_reprompt(content) + + +def test_reprompts_on_plan_only_stall(): + """Response that is purely a plan and no artifact STILL re-prompts.""" + content = "I'll search the web for the answer." + assert _would_reprompt(content), ( + "Plan-only stalls must still trigger the re-prompt" + ) + + +def test_reprompts_on_intent_with_open_fence(): + """Open code fence is not a complete artifact, so we still re-prompt.""" + content = "First, let me write the code.\n```python\nimport" + assert _would_reprompt(content)