diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 7909af8a23..6798d43145 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -2493,7 +2493,9 @@ class LlamaCppBackend: # iterations so they are not silently dropped. yield {"type": "status", "text": ""} if content_accum: - yield {"type": "content", "text": content_accum} + _safe = _strip_tool_markup(content_accum, final=True) + if _safe.strip(): + yield {"type": "content", "text": _safe} _fu = _iter_usage or {} _fc = _fu.get("completion_tokens", 0) _fp = _fu.get("prompt_tokens", 0) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 6f44a3c69f..0c0f0929cf 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1034,17 +1034,6 @@ async def openai_chat_completions( status_code = 400, detail = f"Failed to process image: {e}" ) - # Build message list with system prompt prepended - gguf_messages = [] - if system_prompt: - gguf_messages.append({"role": "system", "content": system_prompt}) - gguf_messages.extend(chat_messages) - - cancel_event = threading.Event() - - completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}" - created = int(time.time()) - # ── Tool-calling path (agentic loop) ────────────────── use_tools = ( payload.enable_tools and llama_backend.supports_tools and not image_b64 @@ -1062,6 +1051,72 @@ async def openai_chat_completions( else: tools_to_use = ALL_TOOLS + # Build a model-agnostic nudge tailored to the active tools. + # No format instructions -- each model's chat template handles + # that. Keeps tool use optional ("prefer", not "must"). + tool_names = {t["function"]["name"] for t in tools_to_use} + has_web = "web_search" in tool_names + has_code = "python" in tool_names or "terminal" in tool_names + + nudge_parts = ["You are a helpful assistant with access to tools."] + if has_code and has_web: + nudge_parts.append( + "For tasks involving math, calculations, code, or data " + "analysis, prefer using the code execution tools to " + "produce a verified answer rather than answering from " + "memory alone. For questions requiring up-to-date or " + "real-time information, use web search." + ) + elif has_code: + nudge_parts.append( + "For tasks involving math, calculations, code, or data " + "analysis, prefer using the code execution tools to " + "produce a verified answer rather than answering from " + "memory alone." + ) + elif has_web: + nudge_parts.append( + "For questions requiring up-to-date or real-time " + "information, prefer using web search rather than " + "answering from memory alone." + ) + + tool_nudge = " ".join(nudge_parts) + + if not system_prompt: + system_prompt = tool_nudge + else: + system_prompt = system_prompt.rstrip() + "\n\n" + tool_nudge + + # Strip stale tool-call XML from conversation history + # (the frontend may include prior assistant text + # in follow-up user messages). + import re as _re + _tool_xml_strip = [ + _re.compile(r".*?", _re.DOTALL), + _re.compile(r".*$", _re.DOTALL), + _re.compile(r".*?", _re.DOTALL), + _re.compile(r".*$", _re.DOTALL), + ] + for msg in chat_messages: + c = msg.get("content", "") + if isinstance(c, str) and "" in c: + for pat in _tool_xml_strip: + c = pat.sub("", c) + msg["content"] = c.strip() + + # Build message list with system prompt prepended + gguf_messages = [] + if system_prompt: + gguf_messages.append({"role": "system", "content": system_prompt}) + gguf_messages.extend(chat_messages) + + cancel_event = threading.Event() + + completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}" + created = int(time.time()) + + if use_tools: def gguf_generate_with_tools(): return llama_backend.generate_chat_completion_with_tools( messages = gguf_messages, @@ -1143,6 +1198,11 @@ async def openai_chat_completions( # "content" type -- cumulative text cumulative = event.get("text", "") + # Strip tool-call XML that may have leaked + # through the backend's content stream. + for pat in _tool_xml_strip: + cumulative = pat.sub("", cumulative) + cumulative = cumulative.rstrip() new_text = cumulative[len(prev_text) :] prev_text = cumulative if not new_text: