Compare commits

...
Sign in to create a new pull request.

9 commits

Author SHA1 Message Date
Daniel Han
a2c77f4932 fix: remove auto-fetch on search, keep url param for explicit fetch
The auto-fetch of top search results added ~2s latency per search
without meaningfully improving results for small models that kept
searching the wrong pages.  Searches now return snippets only (fast)
with a hint telling the model it can fetch any URL explicitly via the
url parameter.  Direct URL fetch remains available for when the model
finds a relevant link and wants the full content.
2026-03-27 14:21:38 +00:00
pre-commit-ci[bot]
92d22fd69c [pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
2026-03-27 14:16:12 +00:00
Daniel Han
e451d2dae4 fix: only strip closed tool-call XML pairs in SSE output
The open-ended <tool_call>.*$ regex was stripping everything from
<tool_call> to end of string, including legitimate model text that
followed a tool block.  This caused the stream to appear stuck since
the cumulative text never grew past the strip point.

Switch to only stripping closed <tool_call>...</tool_call> pairs so
text after a tool block is preserved.
2026-03-27 14:15:59 +00:00
pre-commit-ci[bot]
88aa3ff451 [pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
2026-03-27 14:08:44 +00:00
Daniel Han
ebbe2d66d0 fix: add direct URL fetch to web_search tool
The web_search tool now accepts an optional `url` parameter that
fetches a specific page directly (skipping the search step).  This
lets the model follow URLs it found in earlier search results instead
of re-searching with increasingly desperate queries.

Also updated the tool description to mention that page content is
returned, so models know they will get more than just snippets.
2026-03-27 14:08:32 +00:00
pre-commit-ci[bot]
1e66c83649 [pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
2026-03-27 13:35:03 +00:00
Daniel Han
cab3ed41e9 fix: fetch top search result page content for web_search tool
The web_search tool previously returned only titles, URLs, and short
snippets from DuckDuckGo.  Small models would loop repeatedly trying
different search queries because the snippets never contained enough
data to answer the question.

Now fetches the actual page content for the top 1-2 search results
(best-effort, capped at 6000 chars and 10s timeout per fetch) and
includes it in the tool response.  This gives the model real data to
work with on the first search instead of endlessly retrying.

Uses only stdlib (urllib + re) so no new dependencies are needed.
2026-03-27 13:34:52 +00:00
pre-commit-ci[bot]
2ef375a7fa [pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
2026-03-27 13:33:58 +00:00
Daniel Han
38abdbe0b0 fix: add tool-use system prompt nudge and strip leaked tool-call XML
Small models (e.g. Qwen3.5-4B) frequently skip tool calls when tools
are enabled, defaulting to plain text answers instead.  This adds a
short, model-agnostic system prompt nudge when tools are active that
encourages the model to use them for math/code/search tasks.  The nudge
varies depending on which tools are enabled (code, web, or both).

Also fixes tool-call XML leaking into visible chat output:
- Strip <tool_call> / <function=...> tags from outgoing SSE content
  deltas before they reach the frontend.
- Sanitize stale <tool_call> XML from conversation history messages
  so prior leaked text does not re-enter the model context.
- Strip raw content_accum in the false-positive DRAINING path that
  previously yielded unprocessed text.
2026-03-27 13:33:20 +00:00
3 changed files with 150 additions and 18 deletions

View file

@ -2493,7 +2493,9 @@ class LlamaCppBackend:
# iterations so they are not silently dropped.
yield {"type": "status", "text": ""}
if content_accum:
yield {"type": "content", "text": content_accum}
_safe = _strip_tool_markup(content_accum, final = True)
if _safe.strip():
yield {"type": "content", "text": _safe}
_fu = _iter_usage or {}
_fc = _fu.get("completion_tokens", 0)
_fp = _fu.get("prompt_tokens", 0)

View file

@ -57,14 +57,18 @@ WEB_SEARCH_TOOL = {
"type": "function",
"function": {
"name": "web_search",
"description": "Search the web for current information, recent events, or facts you are uncertain about.",
"description": "Search the web and fetch page content from the top result. Returns snippets for all results plus the full text of the best matching page.",
"parameters": {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "The search query",
}
"description": "The search query. Be specific and concise.",
},
"url": {
"type": "string",
"description": "Optional: fetch this specific URL directly instead of searching. Use when you already know the page you need.",
},
},
"required": ["query"],
},
@ -131,7 +135,11 @@ def execute_tool(
)
effective_timeout = _EXEC_TIMEOUT if timeout is _TIMEOUT_UNSET else timeout
if name == "web_search":
return _web_search(arguments.get("query", ""), timeout = effective_timeout)
return _web_search(
arguments.get("query", ""),
url = arguments.get("url", ""),
timeout = effective_timeout,
)
if name == "python":
return _python_exec(
arguments.get("code", ""), cancel_event, effective_timeout, session_id
@ -143,8 +151,52 @@ def execute_tool(
return f"Unknown tool: {name}"
def _web_search(query: str, max_results: int = 5, timeout: int = _EXEC_TIMEOUT) -> str:
"""Search the web using DuckDuckGo and return formatted results."""
def _fetch_page_text(url: str, max_chars: int = 4000, timeout: int = 10) -> str:
"""Fetch a URL and extract plain text content (best-effort)."""
import urllib.request
import urllib.error
try:
req = urllib.request.Request(url, headers = {"User-Agent": "Mozilla/5.0"})
with urllib.request.urlopen(req, timeout = timeout) as resp:
raw = resp.read(200_000).decode("utf-8", errors = "replace")
# Strip HTML tags (lightweight, no extra deps)
import re
text = re.sub(
r"<script[^>]*>.*?</script>", "", raw, flags = re.DOTALL | re.IGNORECASE
)
text = re.sub(
r"<style[^>]*>.*?</style>", "", text, flags = re.DOTALL | re.IGNORECASE
)
text = re.sub(r"<[^>]+>", " ", text)
text = re.sub(r"\s+", " ", text).strip()
# Decode HTML entities
import html
text = html.unescape(text)
if len(text) > max_chars:
text = text[:max_chars] + "..."
return text
except Exception:
return ""
def _web_search(
query: str, url: str = "", max_results: int = 5, timeout: int = _EXEC_TIMEOUT
) -> str:
"""Search the web using DuckDuckGo and return formatted results.
If *url* is provided, fetches that page directly (skips search).
Otherwise searches and fetches page content for the top result.
"""
# Direct URL fetch mode
if url and url.strip():
text = _fetch_page_text(url.strip(), max_chars = 8000, timeout = 15)
if text:
return f"Page content from {url}:\n\n{text}"
return f"Failed to fetch {url}"
if not query.strip():
return "No query provided."
try:
@ -153,6 +205,7 @@ def _web_search(query: str, max_results: int = 5, timeout: int = _EXEC_TIMEOUT)
results = DDGS(timeout = timeout).text(query, max_results = max_results)
if not results:
return "No results found."
parts = []
for r in results:
parts.append(
@ -160,6 +213,10 @@ def _web_search(query: str, max_results: int = 5, timeout: int = _EXEC_TIMEOUT)
f"URL: {r.get('href', '')}\n"
f"Snippet: {r.get('body', '')}"
)
parts.append(
"\nTip: To read the full content of any page above, "
"call web_search again with the url parameter."
)
return "\n\n---\n\n".join(parts)
except Exception as e:
return f"Search failed: {e}"

View file

@ -1034,17 +1034,6 @@ async def openai_chat_completions(
status_code = 400, detail = f"Failed to process image: {e}"
)
# Build message list with system prompt prepended
gguf_messages = []
if system_prompt:
gguf_messages.append({"role": "system", "content": system_prompt})
gguf_messages.extend(chat_messages)
cancel_event = threading.Event()
completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"
created = int(time.time())
# ── Tool-calling path (agentic loop) ──────────────────
use_tools = (
payload.enable_tools and llama_backend.supports_tools and not image_b64
@ -1062,6 +1051,74 @@ async def openai_chat_completions(
else:
tools_to_use = ALL_TOOLS
# Build a model-agnostic nudge tailored to the active tools.
# No format instructions -- each model's chat template handles
# that. Keeps tool use optional ("prefer", not "must").
tool_names = {t["function"]["name"] for t in tools_to_use}
has_web = "web_search" in tool_names
has_code = "python" in tool_names or "terminal" in tool_names
nudge_parts = ["You are a helpful assistant with access to tools."]
if has_code and has_web:
nudge_parts.append(
"For tasks involving math, calculations, code, or data "
"analysis, prefer using the code execution tools to "
"produce a verified answer rather than answering from "
"memory alone. For questions requiring up-to-date or "
"real-time information, use web search."
)
elif has_code:
nudge_parts.append(
"For tasks involving math, calculations, code, or data "
"analysis, prefer using the code execution tools to "
"produce a verified answer rather than answering from "
"memory alone."
)
elif has_web:
nudge_parts.append(
"For questions requiring up-to-date or real-time "
"information, prefer using web search rather than "
"answering from memory alone."
)
tool_nudge = " ".join(nudge_parts)
if not system_prompt:
system_prompt = tool_nudge
else:
system_prompt = system_prompt.rstrip() + "\n\n" + tool_nudge
# Strip stale tool-call XML from conversation history
# (the frontend may include prior assistant <tool_call> text
# in follow-up user messages).
import re as _re
_tool_xml_strip = [
_re.compile(r"<tool_call>.*?</tool_call>", _re.DOTALL),
_re.compile(r"<tool_call>.*$", _re.DOTALL),
_re.compile(r"<function=\w+>.*?</function>", _re.DOTALL),
_re.compile(r"<function=\w+>.*$", _re.DOTALL),
]
for msg in chat_messages:
c = msg.get("content", "")
if isinstance(c, str) and "<tool_call>" in c:
for pat in _tool_xml_strip:
c = pat.sub("", c)
msg["content"] = c.strip()
# Build message list with system prompt prepended
gguf_messages = []
if system_prompt:
gguf_messages.append({"role": "system", "content": system_prompt})
gguf_messages.extend(chat_messages)
cancel_event = threading.Event()
completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"
created = int(time.time())
if use_tools:
def gguf_generate_with_tools():
return llama_backend.generate_chat_completion_with_tools(
messages = gguf_messages,
@ -1143,6 +1200,22 @@ async def openai_chat_completions(
# "content" type -- cumulative text
cumulative = event.get("text", "")
# Strip closed tool-call XML pairs that may
# have leaked through the backend stream.
# Only strip closed pairs here (not open-ended)
# so legitimate text after a tool block is kept.
cumulative = _re.sub(
r"<tool_call>.*?</tool_call>",
"",
cumulative,
flags = _re.DOTALL,
)
cumulative = _re.sub(
r"<function=\w+>.*?</function>",
"",
cumulative,
flags = _re.DOTALL,
)
new_text = cumulative[len(prev_text) :]
prev_text = cumulative
if not new_text: