2db8b81854
Two robustness fixes for the `_HAS_ANSWER_ARTIFACT` regex from the parent commit, both caught by a thorough simulation suite covering Linux/Mac/Windows line-ending portability and adversarial inputs. 1. **CRLF line endings.** The original `\n` literals missed Windows- authored or CRLF-converted content (model echoing a pasted prompt, etc.). Replaced with `\r?\n` everywhere a newline is required, so closed code fences, numbered lists, and end-to-end re-prompt decisions all work on `\r\n` as well as `\n`. 2. **Catastrophic backtracking on whitespace spam.** The numbered-list alternative `(?:^|\r?\n)\s*\d+\.\s+\S.*?\r?\n\s*\d+\.` was O(n^2) on long whitespace runs: `\s*` greedy + `\d+` failing + `\s` matching `\r\n` led to repeated backtracking through the newline characters. Measured at ~630ms for 10KB of `\r\n` repeats. Fix: restrict the post-newline indent to `[ \t]*` (spaces / tabs only). After `\r?\n` we are at column 0 and only spaces / tabs are a sensible leading indent for a list item; greedy whitespace was never needed. New worst case on the same input: <1ms (1000x speedup). Added 5 in-tree tests: - test_artifact_regex_handles_crlf_code_fence - test_artifact_regex_handles_crlf_numbered_list - test_artifact_regex_handles_mixed_lf_crlf - test_no_backtrack_on_crlf_spam (asserts <50ms on 10KB \r\n) - test_no_reprompt_on_crlf_complete_python_game All 18 reprompt-guard tests pass. All 253 llama_cpp-related tests pass. Out-of-tree simulation suite (84 tests) passes on both Python 3.12 and Python 3.13 inside isolated uv venvs.
263 lines
9.2 KiB
Python
263 lines
9.2 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-only
|
|
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
|
|
|
"""Tests for the plan-without-action re-prompt guard.
|
|
|
|
The re-prompt path in ``LlamaCppEngine.chat_stream`` exists to nudge a
|
|
model that described what it *will* do (forward-looking language)
|
|
without actually calling a tool. Before the guard added in this PR, the
|
|
heuristic only checked ``len(content) < _REPROMPT_MAX_CHARS`` and the
|
|
intent regex, which over-fired on long-but-complete responses that
|
|
happened to contain phrases like "first" or "let me". Specifically, a
|
|
correct Python game answer of the form ::
|
|
|
|
First, let me set up pygame.
|
|
```python
|
|
import pygame; ...
|
|
```
|
|
|
|
would still match (length < 2000, intent signal present) and the next
|
|
synthetic user turn ("STOP. Do NOT write code or explain.") wiped the
|
|
visible code from the conversation.
|
|
|
|
The new ``_HAS_ANSWER_ARTIFACT`` regex blocks the re-prompt whenever
|
|
the response already contains a real answer artifact: a closed code
|
|
fence, an HTML page, a complete SVG, or a numbered list of items.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
import types as _types
|
|
from pathlib import Path
|
|
|
|
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
|
|
if _BACKEND_DIR not in sys.path:
|
|
sys.path.insert(0, _BACKEND_DIR)
|
|
|
|
_loggers_stub = _types.ModuleType("loggers")
|
|
_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
|
|
sys.modules.setdefault("loggers", _loggers_stub)
|
|
|
|
_structlog_stub = _types.ModuleType("structlog")
|
|
_structlog_stub.get_logger = lambda *a, **k: __import__("logging").getLogger("stub")
|
|
sys.modules.setdefault("structlog", _structlog_stub)
|
|
|
|
from core.inference.llama_cpp import ( # noqa: E402
|
|
_HAS_ANSWER_ARTIFACT,
|
|
_INTENT_SIGNAL,
|
|
)
|
|
|
|
|
|
# ── _INTENT_SIGNAL still matches plan-only stalls ──────────────────
|
|
|
|
|
|
def test_intent_signal_matches_plan_only_phrases():
|
|
"""Original behaviour is preserved: intent regex still matches the
|
|
plan-without-action phrases that motivated the re-prompt."""
|
|
plan_only_samples = [
|
|
"I'll search the web for that.",
|
|
"I will look that up.",
|
|
"I am going to search.",
|
|
"Let me search the web for the answer.",
|
|
"First, I need to look up the date.",
|
|
"Step 1: I'll search for the song list.",
|
|
"Now I need to call the tool.",
|
|
"Here's my plan: search for X.",
|
|
]
|
|
for s in plan_only_samples:
|
|
assert _INTENT_SIGNAL.search(s), f"_INTENT_SIGNAL should match {s!r}"
|
|
|
|
|
|
def test_intent_signal_ignores_direct_answers():
|
|
"""Direct, complete answers do not match the intent regex."""
|
|
direct_samples = [
|
|
"4",
|
|
"Hello!",
|
|
"The answer is 42.",
|
|
"The capital of France is Paris.",
|
|
]
|
|
for s in direct_samples:
|
|
assert not _INTENT_SIGNAL.search(s), f"_INTENT_SIGNAL must not match {s!r}"
|
|
|
|
|
|
# ── _HAS_ANSWER_ARTIFACT recognises substantive content ────────────
|
|
|
|
|
|
def test_artifact_regex_detects_closed_code_fence():
|
|
"""Closed Python code fence is an answer artifact."""
|
|
text = "First, let me set up pygame.\n```python\nimport pygame\npygame.init()\n```"
|
|
assert _HAS_ANSWER_ARTIFACT.search(
|
|
text
|
|
), "Closed code fence must be detected as an answer artifact"
|
|
|
|
|
|
def test_artifact_regex_detects_html_page():
|
|
"""HTML pages (doctype or <html> root) are answer artifacts."""
|
|
text_a = "<!doctype html><html><body><script>fetch('...')</script></body></html>"
|
|
text_b = "Sure, here is the dashboard:\n<html><body>...</body></html>"
|
|
assert _HAS_ANSWER_ARTIFACT.search(text_a)
|
|
assert _HAS_ANSWER_ARTIFACT.search(text_b)
|
|
|
|
|
|
def test_artifact_regex_detects_complete_svg():
|
|
"""A complete <svg>...</svg> is an answer artifact."""
|
|
text = (
|
|
"Here is the sloth SVG:\n"
|
|
"<svg width='200' height='100'>"
|
|
"<circle cx='50' cy='50' r='30'/>"
|
|
"<ellipse cx='100' cy='50' rx='40' ry='20'/>"
|
|
"</svg>"
|
|
)
|
|
assert _HAS_ANSWER_ARTIFACT.search(text)
|
|
|
|
|
|
def test_artifact_regex_detects_numbered_list():
|
|
"""A list of 2+ numbered items is an answer artifact."""
|
|
text = (
|
|
"Let me list these:\n"
|
|
"1. Animals — Maroon 5\n"
|
|
"2. Take Me to Church — Hozier\n"
|
|
"3. Love Me Like You Do — Ellie Goulding\n"
|
|
)
|
|
assert _HAS_ANSWER_ARTIFACT.search(text)
|
|
|
|
|
|
def test_artifact_regex_ignores_open_code_fence():
|
|
"""An UNCLOSED code fence is not yet a complete artifact."""
|
|
text = "Let me set up pygame.\n```python\nimport pygame"
|
|
assert not _HAS_ANSWER_ARTIFACT.search(
|
|
text
|
|
), "Open code fence must not satisfy the artifact guard"
|
|
|
|
|
|
def test_artifact_regex_ignores_plain_text():
|
|
"""Plain conversational text contains no artifact."""
|
|
text = "First, I will search for the songs that charted #3 in 2015."
|
|
assert not _HAS_ANSWER_ARTIFACT.search(text)
|
|
|
|
|
|
# ── End-to-end guard semantics on realistic responses ──────────────
|
|
|
|
|
|
def _would_reprompt(content: str) -> bool:
|
|
"""Return True if the re-prompt block at llama_cpp.py would fire."""
|
|
from core.inference.llama_cpp import _REPROMPT_MAX_CHARS
|
|
|
|
stripped = content.strip()
|
|
return bool(
|
|
0 < len(stripped) < _REPROMPT_MAX_CHARS
|
|
and _INTENT_SIGNAL.search(stripped)
|
|
and not _HAS_ANSWER_ARTIFACT.search(stripped)
|
|
)
|
|
|
|
|
|
def test_no_reprompt_on_complete_python_game():
|
|
"""Response with intent phrasing + complete code does NOT re-prompt."""
|
|
content = (
|
|
"First, let me set up pygame.\n"
|
|
"```python\n"
|
|
"import pygame\n"
|
|
"pygame.init()\n"
|
|
"screen = pygame.display.set_mode((640, 480))\n"
|
|
"while True:\n"
|
|
" for e in pygame.event.get():\n"
|
|
" if e.type == pygame.QUIT: break\n"
|
|
"```"
|
|
)
|
|
assert not _would_reprompt(
|
|
content
|
|
), "Re-prompt must not fire after a complete code block was produced"
|
|
|
|
|
|
def test_no_reprompt_on_complete_svg():
|
|
"""Response with intent phrasing + complete SVG does NOT re-prompt."""
|
|
content = (
|
|
"Let me draw a cute sloth:\n"
|
|
"<svg width='100' height='100'>"
|
|
"<circle cx='50' cy='50' r='30' fill='brown'/>"
|
|
"<circle cx='40' cy='45' r='3' fill='black'/>"
|
|
"<circle cx='60' cy='45' r='3' fill='black'/>"
|
|
"<path d='M40 60 Q50 70 60 60' stroke='black' fill='none'/>"
|
|
"</svg>"
|
|
)
|
|
assert not _would_reprompt(content)
|
|
|
|
|
|
def test_no_reprompt_on_numbered_list_answer():
|
|
"""Response with intent + numbered list (Billboard-style) does NOT re-prompt."""
|
|
content = (
|
|
"Here's my list of #3 hits:\n"
|
|
"1. Animals — Maroon 5\n"
|
|
"2. Take Me to Church — Hozier\n"
|
|
"3. Drag Me Down — One Direction\n"
|
|
)
|
|
assert not _would_reprompt(content)
|
|
|
|
|
|
def test_reprompts_on_plan_only_stall():
|
|
"""Response that is purely a plan and no artifact STILL re-prompts."""
|
|
content = "I'll search the web for the answer."
|
|
assert _would_reprompt(content), "Plan-only stalls must still trigger the re-prompt"
|
|
|
|
|
|
def test_reprompts_on_intent_with_open_fence():
|
|
"""Open code fence is not a complete artifact, so we still re-prompt."""
|
|
content = "First, let me write the code.\n```python\nimport"
|
|
assert _would_reprompt(content)
|
|
|
|
|
|
# ── Cross-platform line endings ────────────────────────────────────
|
|
|
|
|
|
def test_artifact_regex_handles_crlf_code_fence():
|
|
"""Windows / CRLF-converted content still detects a closed fence."""
|
|
content = "First, let me code.\r\n```python\r\nimport sys\r\nprint('hi')\r\n```"
|
|
assert _HAS_ANSWER_ARTIFACT.search(content), (
|
|
"CRLF (\\r\\n) line endings inside a code fence must still match"
|
|
)
|
|
|
|
|
|
def test_artifact_regex_handles_crlf_numbered_list():
|
|
"""CRLF numbered list also matches."""
|
|
content = "Here's the plan:\r\n1. one\r\n2. two\r\n"
|
|
assert _HAS_ANSWER_ARTIFACT.search(content)
|
|
|
|
|
|
def test_artifact_regex_handles_mixed_lf_crlf():
|
|
"""Mixed line endings (real-world: paste-and-edit on Windows)."""
|
|
content = "Here's the code:\r\n```python\nimport sys\r\n```"
|
|
assert _HAS_ANSWER_ARTIFACT.search(content)
|
|
|
|
|
|
def test_no_backtrack_on_crlf_spam():
|
|
"""10K of `\\r\\n` repeats must complete fast.
|
|
|
|
Pre-fix the numbered-list alternative `(?:^|\\r?\\n)\\s*\\d+\\.` would
|
|
O(n^2)-backtrack on this kind of input (measured at ~630ms for 10KB
|
|
of `\\r\\n` repeats). The post-fix `[ \\t]*` indent restriction
|
|
keeps it linear.
|
|
"""
|
|
import time
|
|
payload = "\r\n" * 5000
|
|
t0 = time.time()
|
|
_HAS_ANSWER_ARTIFACT.search(payload)
|
|
elapsed_ms = (time.time() - t0) * 1000
|
|
assert elapsed_ms < 50, f"regex took {elapsed_ms:.1f}ms on 10KB CRLF spam"
|
|
|
|
|
|
def test_no_reprompt_on_crlf_complete_python_game():
|
|
"""End-to-end CRLF: complete fence -> no re-prompt."""
|
|
content = (
|
|
"First, let me set up pygame.\r\n"
|
|
"```python\r\n"
|
|
"import pygame\r\n"
|
|
"pygame.init()\r\n"
|
|
"while True:\r\n"
|
|
" for e in pygame.event.get():\r\n"
|
|
" if e.type == pygame.QUIT: break\r\n"
|
|
"```"
|
|
)
|
|
assert not _would_reprompt(content), (
|
|
"CRLF-encoded complete fence must also suppress the re-prompt"
|
|
)
|