ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Studio: don't re-prompt after model already produced a complete answer

Browse source 
The plan-without-action re-prompt at
`studio/backend/core/inference/llama_cpp.py` fires when the model
emits intent-only language ("first I'll ...", "let me ...") without
calling a tool. Previously the heuristic only checked an intent regex
and a 2000-char length cap. The same intent words occur in long
explanations that accompany REAL code or markup, so a complete reply
like "First, let me set up pygame. ```python ... ```" still tripped
the re-prompt, and the synthetic follow-up ("STOP. Do NOT write code
or explain.") wiped the user-visible answer.

Reproduced at scale in a 900-run sweep across 15 Qwen3.5/3.6 GGUF
configs: prompts that emit code or markup (Create a Python game,
Create a Flappy Bird game, weather dashboard HTML, sloth SVG)
landed empty `final_text` for the majority of seeds even on the
strongest configs.

Fix adds a `_HAS_ANSWER_ARTIFACT` regex covering:
  - closed code fences (```...```)
  - HTML pages (<!doctype, <html)
  - complete SVG (<svg...</svg>)
  - 2+ item numbered lists

and a `and not _HAS_ANSWER_ARTIFACT.search(_stripped)` guard on the
re-prompt condition. Plan-only stalls still re-prompt; complete
responses no longer do.

13 new unit tests in `test_llama_cpp_reprompt_guard.py` pin both
directions (artifact present -> no re-prompt; plan-only -> still
re-prompts).
This commit is contained in:
Daniel Han 2026-05-22 15:41:11 +00:00 • committed by danielhanchen
commit 078ae64cdf
2 changed files with 226 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

18
studio/backend/core/inference/llama_cpp.py
View file

@ -71,6 +71,23 @@ _INTENT_SIGNAL = re.compile(
)
_MAX_REPROMPTS = 3
# Substantive answer artifacts. Re-prompt fires when the model emits
# intent-only language ("first I'll ...", "let me ...") without a tool
# call, but the same intent words appear in long explanations that
# accompany REAL code or markup. Without this guard, a complete reply
# like "First, let me set up pygame. ```python ... ```" trips the
# re-prompt and the next user-visible message wipes the code. We
# require ALL of (intent signal, length < _REPROMPT_MAX_CHARS, no
# answer artifact) to fire.
_HAS_ANSWER_ARTIFACT = re.compile(
r"```[a-zA-Z]*\n[\s\S]+?\n```" # closed code fence
r"|<!doctype\b" # HTML page
r"|<html\b"
r"|<svg\b[\s\S]*?</svg>" # complete SVG
r"|(?:^|\n)\s*\d+\.\s+\S.*?\n\s*\d+\.", # 2+ numbered list items
re.IGNORECASE,
)
# Without max_tokens, llama-server defaults to n_predict = n_ctx (up to
# 262144 for Qwen3.5), producing many-minute zombie decodes when cancel
# fails. t_max_predict_ms is a wall-clock backstop applied unconditionally,
@ -4818,6 +4835,7 @@ class LlamaCppBackend:
and _reprompt_count < _MAX_REPROMPTS
and 0 < len(_stripped) < _REPROMPT_MAX_CHARS
and _INTENT_SIGNAL.search(_stripped)
and not _HAS_ANSWER_ARTIFACT.search(_stripped)
):
_reprompt_count += 1
logger.info(

208
studio/backend/tests/test_llama_cpp_reprompt_guard.py Normal file
View file

@ -0,0 +1,208 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Tests for the plan-without-action re-prompt guard.
The re-prompt path in ``LlamaCppEngine.chat_stream`` exists to nudge a
model that described what it *will* do (forward-looking language)
without actually calling a tool. Before the guard added in this PR, the
heuristic only checked ``len(content) < _REPROMPT_MAX_CHARS`` and the
intent regex, which over-fired on long-but-complete responses that
happened to contain phrases like "first" or "let me". Specifically, a
correct Python game answer of the form ::
First, let me set up pygame.
```python
import pygame; ...
```
would still match (length < 2000, intent signal present) and the next
synthetic user turn ("STOP. Do NOT write code or explain.") wiped the
visible code from the conversation.
The new ``_HAS_ANSWER_ARTIFACT`` regex blocks the re-prompt whenever
the response already contains a real answer artifact: a closed code
fence, an HTML page, a complete SVG, or a numbered list of items.
"""
from __future__ import annotations
import sys
import types as _types
from pathlib import Path
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
_loggers_stub = _types.ModuleType("loggers")
_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
sys.modules.setdefault("loggers", _loggers_stub)
_structlog_stub = _types.ModuleType("structlog")
_structlog_stub.get_logger = lambda *a, **k: __import__("logging").getLogger("stub")
sys.modules.setdefault("structlog", _structlog_stub)
from core.inference.llama_cpp import ( # noqa: E402
_HAS_ANSWER_ARTIFACT,
_INTENT_SIGNAL,
)
# ── _INTENT_SIGNAL still matches plan-only stalls ──────────────────
def test_intent_signal_matches_plan_only_phrases():
"""Original behaviour is preserved: intent regex still matches the
plan-without-action phrases that motivated the re-prompt."""
plan_only_samples = [
"I'll search the web for that.",
"I will look that up.",
"I am going to search.",
"Let me search the web for the answer.",
"First, I need to look up the date.",
"Step 1: I'll search for the song list.",
"Now I need to call the tool.",
"Here's my plan: search for X.",
]
for s in plan_only_samples:
assert _INTENT_SIGNAL.search(s), f"_INTENT_SIGNAL should match {s!r}"
def test_intent_signal_ignores_direct_answers():
"""Direct, complete answers do not match the intent regex."""
direct_samples = [
"4",
"Hello!",
"The answer is 42.",
"The capital of France is Paris.",
]
for s in direct_samples:
assert not _INTENT_SIGNAL.search(s), f"_INTENT_SIGNAL must not match {s!r}"
# ── _HAS_ANSWER_ARTIFACT recognises substantive content ────────────
def test_artifact_regex_detects_closed_code_fence():
"""Closed Python code fence is an answer artifact."""
text = "First, let me set up pygame.\n```python\nimport pygame\npygame.init()\n```"
assert _HAS_ANSWER_ARTIFACT.search(text), (
"Closed code fence must be detected as an answer artifact"
)
def test_artifact_regex_detects_html_page():
"""HTML pages (doctype or <html> root) are answer artifacts."""
text_a = "<!doctype html><html><body><script>fetch('...')</script></body></html>"
text_b = "Sure, here is the dashboard:\n<html><body>...</body></html>"
assert _HAS_ANSWER_ARTIFACT.search(text_a)
assert _HAS_ANSWER_ARTIFACT.search(text_b)
def test_artifact_regex_detects_complete_svg():
"""A complete <svg>...</svg> is an answer artifact."""
text = (
"Here is the sloth SVG:\n"
"<svg width='200' height='100'>"
"<circle cx='50' cy='50' r='30'/>"
"<ellipse cx='100' cy='50' rx='40' ry='20'/>"
"</svg>"
)
assert _HAS_ANSWER_ARTIFACT.search(text)
def test_artifact_regex_detects_numbered_list():
"""A list of 2+ numbered items is an answer artifact."""
text = (
"Let me list these:\n"
"1. Animals — Maroon 5\n"
"2. Take Me to Church — Hozier\n"
"3. Love Me Like You Do — Ellie Goulding\n"
)
assert _HAS_ANSWER_ARTIFACT.search(text)
def test_artifact_regex_ignores_open_code_fence():
"""An UNCLOSED code fence is not yet a complete artifact."""
text = "Let me set up pygame.\n```python\nimport pygame"
assert not _HAS_ANSWER_ARTIFACT.search(text), (
"Open code fence must not satisfy the artifact guard"
)
def test_artifact_regex_ignores_plain_text():
"""Plain conversational text contains no artifact."""
text = "First, I will search for the songs that charted #3 in 2015."
assert not _HAS_ANSWER_ARTIFACT.search(text)
# ── End-to-end guard semantics on realistic responses ──────────────
def _would_reprompt(content: str) -> bool:
"""Return True if the re-prompt block at llama_cpp.py would fire."""
from core.inference.llama_cpp import _REPROMPT_MAX_CHARS
stripped = content.strip()
return bool(
0 < len(stripped) < _REPROMPT_MAX_CHARS
and _INTENT_SIGNAL.search(stripped)
and not _HAS_ANSWER_ARTIFACT.search(stripped)
)
def test_no_reprompt_on_complete_python_game():
"""Response with intent phrasing + complete code does NOT re-prompt."""
content = (
"First, let me set up pygame.\n"
"```python\n"
"import pygame\n"
"pygame.init()\n"
"screen = pygame.display.set_mode((640, 480))\n"
"while True:\n"
" for e in pygame.event.get():\n"
" if e.type == pygame.QUIT: break\n"
"```"
)
assert not _would_reprompt(content), (
"Re-prompt must not fire after a complete code block was produced"
)
def test_no_reprompt_on_complete_svg():
"""Response with intent phrasing + complete SVG does NOT re-prompt."""
content = (
"Let me draw a cute sloth:\n"
"<svg width='100' height='100'>"
"<circle cx='50' cy='50' r='30' fill='brown'/>"
"<circle cx='40' cy='45' r='3' fill='black'/>"
"<circle cx='60' cy='45' r='3' fill='black'/>"
"<path d='M40 60 Q50 70 60 60' stroke='black' fill='none'/>"
"</svg>"
)
assert not _would_reprompt(content)
def test_no_reprompt_on_numbered_list_answer():
"""Response with intent + numbered list (Billboard-style) does NOT re-prompt."""
content = (
"Here's my list of #3 hits:\n"
"1. Animals — Maroon 5\n"
"2. Take Me to Church — Hozier\n"
"3. Drag Me Down — One Direction\n"
)
assert not _would_reprompt(content)
def test_reprompts_on_plan_only_stall():
"""Response that is purely a plan and no artifact STILL re-prompts."""
content = "I'll search the web for the answer."
assert _would_reprompt(content), (
"Plan-only stalls must still trigger the re-prompt"
)
def test_reprompts_on_intent_with_open_fence():
"""Open code fence is not a complete artifact, so we still re-prompt."""
content = "First, let me write the code.\n```python\nimport"
assert _would_reprompt(content)