From 38abdbe0b090126e4bb7094c986a85e233e36d61 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 27 Mar 2026 13:33:20 +0000 Subject: [PATCH 1/9] fix: add tool-use system prompt nudge and strip leaked tool-call XML Small models (e.g. Qwen3.5-4B) frequently skip tool calls when tools are enabled, defaulting to plain text answers instead. This adds a short, model-agnostic system prompt nudge when tools are active that encourages the model to use them for math/code/search tasks. The nudge varies depending on which tools are enabled (code, web, or both). Also fixes tool-call XML leaking into visible chat output: - Strip / tags from outgoing SSE content deltas before they reach the frontend. - Sanitize stale XML from conversation history messages so prior leaked text does not re-enter the model context. - Strip raw content_accum in the false-positive DRAINING path that previously yielded unprocessed text. --- studio/backend/core/inference/llama_cpp.py | 4 +- studio/backend/routes/inference.py | 82 +++++++++++++++++++--- 2 files changed, 74 insertions(+), 12 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 7909af8a23..6798d43145 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -2493,7 +2493,9 @@ class LlamaCppBackend: # iterations so they are not silently dropped. yield {"type": "status", "text": ""} if content_accum: - yield {"type": "content", "text": content_accum} + _safe = _strip_tool_markup(content_accum, final=True) + if _safe.strip(): + yield {"type": "content", "text": _safe} _fu = _iter_usage or {} _fc = _fu.get("completion_tokens", 0) _fp = _fu.get("prompt_tokens", 0) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 6f44a3c69f..0c0f0929cf 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1034,17 +1034,6 @@ async def openai_chat_completions( status_code = 400, detail = f"Failed to process image: {e}" ) - # Build message list with system prompt prepended - gguf_messages = [] - if system_prompt: - gguf_messages.append({"role": "system", "content": system_prompt}) - gguf_messages.extend(chat_messages) - - cancel_event = threading.Event() - - completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}" - created = int(time.time()) - # ── Tool-calling path (agentic loop) ────────────────── use_tools = ( payload.enable_tools and llama_backend.supports_tools and not image_b64 @@ -1062,6 +1051,72 @@ async def openai_chat_completions( else: tools_to_use = ALL_TOOLS + # Build a model-agnostic nudge tailored to the active tools. + # No format instructions -- each model's chat template handles + # that. Keeps tool use optional ("prefer", not "must"). + tool_names = {t["function"]["name"] for t in tools_to_use} + has_web = "web_search" in tool_names + has_code = "python" in tool_names or "terminal" in tool_names + + nudge_parts = ["You are a helpful assistant with access to tools."] + if has_code and has_web: + nudge_parts.append( + "For tasks involving math, calculations, code, or data " + "analysis, prefer using the code execution tools to " + "produce a verified answer rather than answering from " + "memory alone. For questions requiring up-to-date or " + "real-time information, use web search." + ) + elif has_code: + nudge_parts.append( + "For tasks involving math, calculations, code, or data " + "analysis, prefer using the code execution tools to " + "produce a verified answer rather than answering from " + "memory alone." + ) + elif has_web: + nudge_parts.append( + "For questions requiring up-to-date or real-time " + "information, prefer using web search rather than " + "answering from memory alone." + ) + + tool_nudge = " ".join(nudge_parts) + + if not system_prompt: + system_prompt = tool_nudge + else: + system_prompt = system_prompt.rstrip() + "\n\n" + tool_nudge + + # Strip stale tool-call XML from conversation history + # (the frontend may include prior assistant text + # in follow-up user messages). + import re as _re + _tool_xml_strip = [ + _re.compile(r".*?", _re.DOTALL), + _re.compile(r".*$", _re.DOTALL), + _re.compile(r".*?", _re.DOTALL), + _re.compile(r".*$", _re.DOTALL), + ] + for msg in chat_messages: + c = msg.get("content", "") + if isinstance(c, str) and "" in c: + for pat in _tool_xml_strip: + c = pat.sub("", c) + msg["content"] = c.strip() + + # Build message list with system prompt prepended + gguf_messages = [] + if system_prompt: + gguf_messages.append({"role": "system", "content": system_prompt}) + gguf_messages.extend(chat_messages) + + cancel_event = threading.Event() + + completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}" + created = int(time.time()) + + if use_tools: def gguf_generate_with_tools(): return llama_backend.generate_chat_completion_with_tools( messages = gguf_messages, @@ -1143,6 +1198,11 @@ async def openai_chat_completions( # "content" type -- cumulative text cumulative = event.get("text", "") + # Strip tool-call XML that may have leaked + # through the backend's content stream. + for pat in _tool_xml_strip: + cumulative = pat.sub("", cumulative) + cumulative = cumulative.rstrip() new_text = cumulative[len(prev_text) :] prev_text = cumulative if not new_text: From 2ef375a7faf5f1b6819036f6c1e70f3708a102f0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 13:33:56 +0000 Subject: [PATCH 2/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/core/inference/llama_cpp.py | 2 +- studio/backend/routes/inference.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 6798d43145..b0746a55a9 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -2493,7 +2493,7 @@ class LlamaCppBackend: # iterations so they are not silently dropped. yield {"type": "status", "text": ""} if content_accum: - _safe = _strip_tool_markup(content_accum, final=True) + _safe = _strip_tool_markup(content_accum, final = True) if _safe.strip(): yield {"type": "content", "text": _safe} _fu = _iter_usage or {} diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 0c0f0929cf..99910808f8 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1055,8 +1055,8 @@ async def openai_chat_completions( # No format instructions -- each model's chat template handles # that. Keeps tool use optional ("prefer", not "must"). tool_names = {t["function"]["name"] for t in tools_to_use} - has_web = "web_search" in tool_names - has_code = "python" in tool_names or "terminal" in tool_names + has_web = "web_search" in tool_names + has_code = "python" in tool_names or "terminal" in tool_names nudge_parts = ["You are a helpful assistant with access to tools."] if has_code and has_web: @@ -1092,6 +1092,7 @@ async def openai_chat_completions( # (the frontend may include prior assistant text # in follow-up user messages). import re as _re + _tool_xml_strip = [ _re.compile(r".*?", _re.DOTALL), _re.compile(r".*$", _re.DOTALL), @@ -1117,6 +1118,7 @@ async def openai_chat_completions( created = int(time.time()) if use_tools: + def gguf_generate_with_tools(): return llama_backend.generate_chat_completion_with_tools( messages = gguf_messages, From cab3ed41e94b26f4dcb7491c8c99898e665e16ca Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 27 Mar 2026 13:34:41 +0000 Subject: [PATCH 3/9] fix: fetch top search result page content for web_search tool The web_search tool previously returned only titles, URLs, and short snippets from DuckDuckGo. Small models would loop repeatedly trying different search queries because the snippets never contained enough data to answer the question. Now fetches the actual page content for the top 1-2 search results (best-effort, capped at 6000 chars and 10s timeout per fetch) and includes it in the tool response. This gives the model real data to work with on the first search instead of endlessly retrying. Uses only stdlib (urllib + re) so no new dependencies are needed. --- studio/backend/core/inference/tools.py | 47 ++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 55bfa095f9..04fdae54cc 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -143,8 +143,36 @@ def execute_tool( return f"Unknown tool: {name}" +def _fetch_page_text(url: str, max_chars: int = 4000, timeout: int = 10) -> str: + """Fetch a URL and extract plain text content (best-effort).""" + import urllib.request + import urllib.error + try: + req = urllib.request.Request(url, headers = {"User-Agent": "Mozilla/5.0"}) + with urllib.request.urlopen(req, timeout = timeout) as resp: + raw = resp.read(200_000).decode("utf-8", errors = "replace") + # Strip HTML tags (lightweight, no extra deps) + import re + text = re.sub(r"]*>.*?", "", raw, flags = re.DOTALL | re.IGNORECASE) + text = re.sub(r"]*>.*?", "", text, flags = re.DOTALL | re.IGNORECASE) + text = re.sub(r"<[^>]+>", " ", text) + text = re.sub(r"\s+", " ", text).strip() + # Decode HTML entities + import html + text = html.unescape(text) + if len(text) > max_chars: + text = text[:max_chars] + "..." + return text + except Exception: + return "" + + def _web_search(query: str, max_results: int = 5, timeout: int = _EXEC_TIMEOUT) -> str: - """Search the web using DuckDuckGo and return formatted results.""" + """Search the web using DuckDuckGo and return formatted results. + + For the top result, also fetches the actual page content so the + model has real data to work with instead of just snippets. + """ if not query.strip(): return "No query provided." try: @@ -153,13 +181,26 @@ def _web_search(query: str, max_results: int = 5, timeout: int = _EXEC_TIMEOUT) results = DDGS(timeout = timeout).text(query, max_results = max_results) if not results: return "No results found." + + # Fetch full page content for the top result (best-effort, + # capped at 10s so it does not block the agentic loop). + top_page = "" + for r in results[:2]: + href = r.get("href", "") + if href: + top_page = _fetch_page_text(href, max_chars = 6000, timeout = 10) + if len(top_page) > 200: + break parts = [] - for r in results: - parts.append( + for i, r in enumerate(results): + entry = ( f"Title: {r.get('title', '')}\n" f"URL: {r.get('href', '')}\n" f"Snippet: {r.get('body', '')}" ) + if i == 0 and top_page: + entry += f"\n\nPage content:\n{top_page}" + parts.append(entry) return "\n\n---\n\n".join(parts) except Exception as e: return f"Search failed: {e}" From 1e66c83649bddaf39eeb1d11caba8f35f8b48587 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 13:35:01 +0000 Subject: [PATCH 4/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/core/inference/tools.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 04fdae54cc..0752200ef5 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -147,18 +147,25 @@ def _fetch_page_text(url: str, max_chars: int = 4000, timeout: int = 10) -> str: """Fetch a URL and extract plain text content (best-effort).""" import urllib.request import urllib.error + try: req = urllib.request.Request(url, headers = {"User-Agent": "Mozilla/5.0"}) with urllib.request.urlopen(req, timeout = timeout) as resp: raw = resp.read(200_000).decode("utf-8", errors = "replace") # Strip HTML tags (lightweight, no extra deps) import re - text = re.sub(r"]*>.*?", "", raw, flags = re.DOTALL | re.IGNORECASE) - text = re.sub(r"]*>.*?", "", text, flags = re.DOTALL | re.IGNORECASE) + + text = re.sub( + r"]*>.*?", "", raw, flags = re.DOTALL | re.IGNORECASE + ) + text = re.sub( + r"]*>.*?", "", text, flags = re.DOTALL | re.IGNORECASE + ) text = re.sub(r"<[^>]+>", " ", text) text = re.sub(r"\s+", " ", text).strip() # Decode HTML entities import html + text = html.unescape(text) if len(text) > max_chars: text = text[:max_chars] + "..." From ebbe2d66d07ccc1230a48d2bfa9d0257ef6ed381 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 27 Mar 2026 14:08:26 +0000 Subject: [PATCH 5/9] fix: add direct URL fetch to web_search tool The web_search tool now accepts an optional `url` parameter that fetches a specific page directly (skipping the search step). This lets the model follow URLs it found in earlier search results instead of re-searching with increasingly desperate queries. Also updated the tool description to mention that page content is returned, so models know they will get more than just snippets. --- studio/backend/core/inference/tools.py | 29 +++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 0752200ef5..8ce8f628cc 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -57,14 +57,18 @@ WEB_SEARCH_TOOL = { "type": "function", "function": { "name": "web_search", - "description": "Search the web for current information, recent events, or facts you are uncertain about.", + "description": "Search the web and fetch page content from the top result. Returns snippets for all results plus the full text of the best matching page.", "parameters": { "type": "object", "properties": { "query": { "type": "string", - "description": "The search query", - } + "description": "The search query. Be specific and concise.", + }, + "url": { + "type": "string", + "description": "Optional: fetch this specific URL directly instead of searching. Use when you already know the page you need.", + }, }, "required": ["query"], }, @@ -131,7 +135,11 @@ def execute_tool( ) effective_timeout = _EXEC_TIMEOUT if timeout is _TIMEOUT_UNSET else timeout if name == "web_search": - return _web_search(arguments.get("query", ""), timeout = effective_timeout) + return _web_search( + arguments.get("query", ""), + url = arguments.get("url", ""), + timeout = effective_timeout, + ) if name == "python": return _python_exec( arguments.get("code", ""), cancel_event, effective_timeout, session_id @@ -174,12 +182,19 @@ def _fetch_page_text(url: str, max_chars: int = 4000, timeout: int = 10) -> str: return "" -def _web_search(query: str, max_results: int = 5, timeout: int = _EXEC_TIMEOUT) -> str: +def _web_search(query: str, url: str = "", max_results: int = 5, timeout: int = _EXEC_TIMEOUT) -> str: """Search the web using DuckDuckGo and return formatted results. - For the top result, also fetches the actual page content so the - model has real data to work with instead of just snippets. + If *url* is provided, fetches that page directly (skips search). + Otherwise searches and fetches page content for the top result. """ + # Direct URL fetch mode + if url and url.strip(): + text = _fetch_page_text(url.strip(), max_chars = 8000, timeout = 15) + if text: + return f"Page content from {url}:\n\n{text}" + return f"Failed to fetch {url}" + if not query.strip(): return "No query provided." try: From 88aa3ff451e5d616d3c6f77826673041ae3da8b9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 14:08:42 +0000 Subject: [PATCH 6/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/core/inference/tools.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 8ce8f628cc..b27a75cde1 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -182,7 +182,9 @@ def _fetch_page_text(url: str, max_chars: int = 4000, timeout: int = 10) -> str: return "" -def _web_search(query: str, url: str = "", max_results: int = 5, timeout: int = _EXEC_TIMEOUT) -> str: +def _web_search( + query: str, url: str = "", max_results: int = 5, timeout: int = _EXEC_TIMEOUT +) -> str: """Search the web using DuckDuckGo and return formatted results. If *url* is provided, fetches that page directly (skips search). From e451d2dae468f958db5261f6b2d71333ccc8ebe4 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 27 Mar 2026 14:15:54 +0000 Subject: [PATCH 7/9] fix: only strip closed tool-call XML pairs in SSE output The open-ended .*$ regex was stripping everything from to end of string, including legitimate model text that followed a tool block. This caused the stream to appear stuck since the cumulative text never grew past the strip point. Switch to only stripping closed ... pairs so text after a tool block is preserved. --- studio/backend/routes/inference.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 99910808f8..40e35cc0f6 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1200,11 +1200,18 @@ async def openai_chat_completions( # "content" type -- cumulative text cumulative = event.get("text", "") - # Strip tool-call XML that may have leaked - # through the backend's content stream. - for pat in _tool_xml_strip: - cumulative = pat.sub("", cumulative) - cumulative = cumulative.rstrip() + # Strip closed tool-call XML pairs that may + # have leaked through the backend stream. + # Only strip closed pairs here (not open-ended) + # so legitimate text after a tool block is kept. + cumulative = _re.sub( + r".*?", "", + cumulative, flags = _re.DOTALL, + ) + cumulative = _re.sub( + r".*?", "", + cumulative, flags = _re.DOTALL, + ) new_text = cumulative[len(prev_text) :] prev_text = cumulative if not new_text: From 92d22fd69c2ccf553a03ea083d561c76359d8f6a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 14:16:10 +0000 Subject: [PATCH 8/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/routes/inference.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 40e35cc0f6..8efe6e1699 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1205,12 +1205,16 @@ async def openai_chat_completions( # Only strip closed pairs here (not open-ended) # so legitimate text after a tool block is kept. cumulative = _re.sub( - r".*?", "", - cumulative, flags = _re.DOTALL, + r".*?", + "", + cumulative, + flags = _re.DOTALL, ) cumulative = _re.sub( - r".*?", "", - cumulative, flags = _re.DOTALL, + r".*?", + "", + cumulative, + flags = _re.DOTALL, ) new_text = cumulative[len(prev_text) :] prev_text = cumulative From a2c77f49325dd697ff6048b0f553bc2831149317 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 27 Mar 2026 14:21:38 +0000 Subject: [PATCH 9/9] fix: remove auto-fetch on search, keep url param for explicit fetch The auto-fetch of top search results added ~2s latency per search without meaningfully improving results for small models that kept searching the wrong pages. Searches now return snippets only (fast) with a hint telling the model it can fetch any URL explicitly via the url parameter. Direct URL fetch remains available for when the model finds a relevant link and wants the full content. --- studio/backend/core/inference/tools.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index b27a75cde1..a52819eedb 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -206,25 +206,17 @@ def _web_search( if not results: return "No results found." - # Fetch full page content for the top result (best-effort, - # capped at 10s so it does not block the agentic loop). - top_page = "" - for r in results[:2]: - href = r.get("href", "") - if href: - top_page = _fetch_page_text(href, max_chars = 6000, timeout = 10) - if len(top_page) > 200: - break parts = [] - for i, r in enumerate(results): - entry = ( + for r in results: + parts.append( f"Title: {r.get('title', '')}\n" f"URL: {r.get('href', '')}\n" f"Snippet: {r.get('body', '')}" ) - if i == 0 and top_page: - entry += f"\n\nPage content:\n{top_page}" - parts.append(entry) + parts.append( + "\nTip: To read the full content of any page above, " + "call web_search again with the url parameter." + ) return "\n\n---\n\n".join(parts) except Exception as e: return f"Search failed: {e}"