fix: add tool-use system prompt nudge and strip leaked tool-call XML
Small models (e.g. Qwen3.5-4B) frequently skip tool calls when tools are enabled, defaulting to plain text answers instead. This adds a short, model-agnostic system prompt nudge when tools are active that encourages the model to use them for math/code/search tasks. The nudge varies depending on which tools are enabled (code, web, or both). Also fixes tool-call XML leaking into visible chat output: - Strip <tool_call> / <function=...> tags from outgoing SSE content deltas before they reach the frontend. - Sanitize stale <tool_call> XML from conversation history messages so prior leaked text does not re-enter the model context. - Strip raw content_accum in the false-positive DRAINING path that previously yielded unprocessed text.
This commit is contained in:
parent
c4e34c88c8
commit
38abdbe0b0
2 changed files with 74 additions and 12 deletions
|
|
@ -2493,7 +2493,9 @@ class LlamaCppBackend:
|
|||
# iterations so they are not silently dropped.
|
||||
yield {"type": "status", "text": ""}
|
||||
if content_accum:
|
||||
yield {"type": "content", "text": content_accum}
|
||||
_safe = _strip_tool_markup(content_accum, final=True)
|
||||
if _safe.strip():
|
||||
yield {"type": "content", "text": _safe}
|
||||
_fu = _iter_usage or {}
|
||||
_fc = _fu.get("completion_tokens", 0)
|
||||
_fp = _fu.get("prompt_tokens", 0)
|
||||
|
|
|
|||
|
|
@ -1034,17 +1034,6 @@ async def openai_chat_completions(
|
|||
status_code = 400, detail = f"Failed to process image: {e}"
|
||||
)
|
||||
|
||||
# Build message list with system prompt prepended
|
||||
gguf_messages = []
|
||||
if system_prompt:
|
||||
gguf_messages.append({"role": "system", "content": system_prompt})
|
||||
gguf_messages.extend(chat_messages)
|
||||
|
||||
cancel_event = threading.Event()
|
||||
|
||||
completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"
|
||||
created = int(time.time())
|
||||
|
||||
# ── Tool-calling path (agentic loop) ──────────────────
|
||||
use_tools = (
|
||||
payload.enable_tools and llama_backend.supports_tools and not image_b64
|
||||
|
|
@ -1062,6 +1051,72 @@ async def openai_chat_completions(
|
|||
else:
|
||||
tools_to_use = ALL_TOOLS
|
||||
|
||||
# Build a model-agnostic nudge tailored to the active tools.
|
||||
# No format instructions -- each model's chat template handles
|
||||
# that. Keeps tool use optional ("prefer", not "must").
|
||||
tool_names = {t["function"]["name"] for t in tools_to_use}
|
||||
has_web = "web_search" in tool_names
|
||||
has_code = "python" in tool_names or "terminal" in tool_names
|
||||
|
||||
nudge_parts = ["You are a helpful assistant with access to tools."]
|
||||
if has_code and has_web:
|
||||
nudge_parts.append(
|
||||
"For tasks involving math, calculations, code, or data "
|
||||
"analysis, prefer using the code execution tools to "
|
||||
"produce a verified answer rather than answering from "
|
||||
"memory alone. For questions requiring up-to-date or "
|
||||
"real-time information, use web search."
|
||||
)
|
||||
elif has_code:
|
||||
nudge_parts.append(
|
||||
"For tasks involving math, calculations, code, or data "
|
||||
"analysis, prefer using the code execution tools to "
|
||||
"produce a verified answer rather than answering from "
|
||||
"memory alone."
|
||||
)
|
||||
elif has_web:
|
||||
nudge_parts.append(
|
||||
"For questions requiring up-to-date or real-time "
|
||||
"information, prefer using web search rather than "
|
||||
"answering from memory alone."
|
||||
)
|
||||
|
||||
tool_nudge = " ".join(nudge_parts)
|
||||
|
||||
if not system_prompt:
|
||||
system_prompt = tool_nudge
|
||||
else:
|
||||
system_prompt = system_prompt.rstrip() + "\n\n" + tool_nudge
|
||||
|
||||
# Strip stale tool-call XML from conversation history
|
||||
# (the frontend may include prior assistant <tool_call> text
|
||||
# in follow-up user messages).
|
||||
import re as _re
|
||||
_tool_xml_strip = [
|
||||
_re.compile(r"<tool_call>.*?</tool_call>", _re.DOTALL),
|
||||
_re.compile(r"<tool_call>.*$", _re.DOTALL),
|
||||
_re.compile(r"<function=\w+>.*?</function>", _re.DOTALL),
|
||||
_re.compile(r"<function=\w+>.*$", _re.DOTALL),
|
||||
]
|
||||
for msg in chat_messages:
|
||||
c = msg.get("content", "")
|
||||
if isinstance(c, str) and "<tool_call>" in c:
|
||||
for pat in _tool_xml_strip:
|
||||
c = pat.sub("", c)
|
||||
msg["content"] = c.strip()
|
||||
|
||||
# Build message list with system prompt prepended
|
||||
gguf_messages = []
|
||||
if system_prompt:
|
||||
gguf_messages.append({"role": "system", "content": system_prompt})
|
||||
gguf_messages.extend(chat_messages)
|
||||
|
||||
cancel_event = threading.Event()
|
||||
|
||||
completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"
|
||||
created = int(time.time())
|
||||
|
||||
if use_tools:
|
||||
def gguf_generate_with_tools():
|
||||
return llama_backend.generate_chat_completion_with_tools(
|
||||
messages = gguf_messages,
|
||||
|
|
@ -1143,6 +1198,11 @@ async def openai_chat_completions(
|
|||
|
||||
# "content" type -- cumulative text
|
||||
cumulative = event.get("text", "")
|
||||
# Strip tool-call XML that may have leaked
|
||||
# through the backend's content stream.
|
||||
for pat in _tool_xml_strip:
|
||||
cumulative = pat.sub("", cumulative)
|
||||
cumulative = cumulative.rstrip()
|
||||
new_text = cumulative[len(prev_text) :]
|
||||
prev_text = cumulative
|
||||
if not new_text:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue