From 4f9986ecb991c51b26c206ffcf33f9ff23291d7b Mon Sep 17 00:00:00 2001
From: Daniel Han <danielhanchen@gmail.com>
Date: Thu, 2 Apr 2026 08:59:02 -0700
Subject: [PATCH] fix(studio): improve tool-calling re-prompt for small models
 (#4783)

Small GGUF models (<9B) frequently generate full code or lengthy
explanations instead of calling tools, bypassing the existing
plan-without-action re-prompt mechanism. Three issues:

1. _REPROMPT_MAX_CHARS=500 was too low -- models that output full
   HTML/code responses (often 1000+ chars) never triggered the
   re-prompt at all, since it only fires on short responses.

2. _MAX_REPROMPTS=1 gave the model only one chance to comply.
   Small models often need 2-3 nudges before switching from
   text generation to tool calling.

3. The re-prompt text ("Please use the available tools...") was
   too polite for small models to follow reliably.

4. Tool-calling detection missed chat templates using Jinja
   whitespace-trimming syntax ({%- if tools -%}) since only
   ({%- if tools %}) and ({% if tools %}) were checked.

Changes:
- Raise _REPROMPT_MAX_CHARS from 500 to 2000 so longer responses
  (code blocks, multi-paragraph plans) still trigger re-prompts
- Raise _MAX_REPROMPTS from 1 to 3 for more retry budget
- Use direct, imperative re-prompt language that small models
  follow more reliably ("STOP. You MUST call a tool NOW.")
- Strengthen the system prompt tool nudge to explicitly forbid
  outputting code blocks (redirect to the python tool instead)
- Add Jinja whitespace-trimmed variants to the tool_markers
  list so all template styles are detected correctly
---
 studio/backend/core/inference/llama_cpp.py | 9 +++++----
 studio/backend/routes/inference.py         | 4 +++-
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index b436f4dc6d..44c700bf3d 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -45,8 +45,8 @@ _INTENT_SIGNAL = re.compile(
     r"\b(?:now i|next i)\b"
     r")"
 )
-_MAX_REPROMPTS = 1
-_REPROMPT_MAX_CHARS = 500
+_MAX_REPROMPTS = 3
+_REPROMPT_MAX_CHARS = 2000
 
 # ── Pre-compiled patterns for GGUF shard detection ───────────
 _SHARD_FULL_RE = re.compile(r"^(.*)-(\d{5})-of-(\d{5})\.gguf$")
@@ -2659,8 +2659,9 @@ class LlamaCppBackend:
                                 {
                                     "role": "user",
                                     "content": (
-                                        "Please use the available tools to complete "
-                                        "the task instead of describing what to do."
+                                        "STOP. Do NOT write code or explain. "
+                                        "You MUST call a tool NOW. "
+                                        "Call web_search or python immediately."
                                     ),
                                 }
                             )
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 8f058bd0f2..ced24c1d5f 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -96,8 +96,10 @@ router = APIRouter()
 
 # Appended to tool-use nudge to discourage plan-without-action
 _TOOL_ACTION_NUDGE = (
-    " Always call tools directly."
+    " IMPORTANT: Always call tools directly -- never write code yourself."
     " Never describe what you plan to do -- just call the tool immediately."
+    " For any code request, call the python tool. For any factual question, call web_search."
+    " Do NOT output code blocks -- use the python tool instead."
 )
 
 # Regex for stripping leaked tool-call XML from assistant messages/stream