From 62827a9f3f6794e9bdff4af5e935db66cd06cb28 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 16 Apr 2026 11:03:17 +0000 Subject: [PATCH 1/4] Studio: soften tool-use nudge for small models and add synthesise directive The existing _TOOL_ACTION_NUDGE tells the model "For any factual question, call web_search" and "Never describe what you plan to do -- just call the tool immediately". On small GGUF models (<9B) this causes two failure modes we can measure: 1. First turn: the model calls web_search on questions it could answer from training data, often queuing several parallel searches in one assistant message. Measured on Qwen3.5-4B UD-Q4_K_XL, n=30: 28/30 tool_call, 2/30 answer. On Qwen3.5-4B Q4_K_M: 30/30 tool_call, 0/30 answer. 2. Subsequent turns: even with tool results already in context, the "prefer tools" directive keeps dominating and the model searches again instead of synthesising. Measured on UD-Q4_K_XL with one tool_result present: 23/30 tool_call. Two changes: _TOOL_ACTION_NUDGE_SMALL: a softer nudge used for models under 9B. Asks for tool use only when current information or a calculation is actually needed, and explicitly discourages queuing multiple parallel tool calls. Larger models keep the original aggressive nudge -- they weren't the ones over-triggering. _TOOL_SYNTHESISE_NUDGE: appended whenever the conversation already has a tool result, and injected into the system message inside the internal tool-call loop once the first tool result has been added. Phrasing matters here -- framing this as a concrete action ("write the final answer to the user's original question using what you have") works. Framing it as an opt-out clause ("do not call more tools unless...") is actually worse than no nudge, measured 33% vs 57% synthesis rate on UD-Q4_K_XL. Measured end-to-end on the same "How do you fine-tune an audio model with Unsloth?" query, n=30 per cell: Qwen3.5-4B UD-Q4_K_XL OLD NEW first turn tool_call 28/30 (93%) 2/30 (7%) first turn answer 2/30 28/30 second turn tool_call 23/30 (77%) 6/30 (20%) second turn answer 7/30 24/30 Qwen3.5-4B Q4_K_M (LM Studio) OLD NEW first turn tool_call 30/30 (100%) 0/30 (0%) first turn answer 0/30 30/30 second turn tool_call 2/30 0/30 second turn answer 28/30 30/30 Test scripts under tests/test_ab_nudge.py and tests/test_stronger_synth.py. --- studio/backend/core/inference/llama_cpp.py | 37 ++++++++++++++++++ studio/backend/routes/inference.py | 45 +++++++++++++++++++++- 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index b53fc513de..921c8acf74 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -2473,6 +2473,39 @@ class LlamaCppBackend: _accumulated_predicted_ms = 0.0 _accumulated_predicted_n = 0 + # Flipped once a tool result has been appended to the conversation + # and the system message updated with a synthesise-now directive. + # Small models otherwise keep honouring the initial "prefer tools" + # nudge and loop on search forever, even when the result they need + # is already in context. + _synthesise_nudge_applied = False + # Phrased as a concrete action rather than an opt-out clause -- + # the "do not call more tools" wording actively harmed small-model + # synthesis rate in our benchmarks. + _SYNTHESISE_NUDGE = ( + " Tool results have been gathered. Now write the final answer to the" + " user's original question using what you have. Tool calls are no" + " longer needed for this turn." + ) + + def _apply_synthesise_nudge() -> None: + nonlocal _synthesise_nudge_applied + if _synthesise_nudge_applied: + return + for _msg in conversation: + if _msg.get("role") == "system": + _content = _msg.get("content") or "" + if _SYNTHESISE_NUDGE.strip() not in _content: + _msg["content"] = _content.rstrip() + _SYNTHESISE_NUDGE + _synthesise_nudge_applied = True + return + # No system message yet: insert one + conversation.insert( + 0, + {"role": "system", "content": _SYNTHESISE_NUDGE.lstrip()}, + ) + _synthesise_nudge_applied = True + def _strip_tool_markup(text: str, *, final: bool = False) -> str: if not auto_heal_tool_calls: return text @@ -3135,6 +3168,10 @@ class LlamaCppBackend: tool_msg["tool_call_id"] = tool_call_id conversation.append(tool_msg) + # First tool result of the loop: tell the model to + # synthesise an answer rather than continue searching. + _apply_synthesise_nudge() + # Clear tool status badge before next generation iteration yield {"type": "status", "text": ""} # Continue the loop to let model respond with context diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 4246f0056b..63ce790271 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -127,6 +127,34 @@ _TOOL_ACTION_NUDGE = ( " Do NOT output code blocks -- use the python tool instead." ) +# Softer variant for small models (<9B). The aggressive ALWAYS-CALL-TOOLS +# phrasing above causes small models to pick web_search on every factual +# question even when the answer sits in their training data, and to keep +# calling search after each result instead of synthesising. See +# tests/test_tool_loop_with_nudge.py for the measured behaviour. +_TOOL_ACTION_NUDGE_SMALL = ( + " Call tools only when you need current information or a specific" + " calculation. For questions within your knowledge, answer directly." + " Issue one tool call at a time rather than queuing several at once." +) + +# Appended whenever the current conversation already contains a tool +# result, to counteract the "prefer tools" nudge and push the model +# toward synthesising a final answer from what it has. Matters most for +# small models where the initial "prefer tools" directive is still +# dominating on the second and subsequent turns. +# +# Phrasing matters a lot here -- "do not call more tools" as an opt-out +# clause is actually worse than no nudge (measured 33% vs 57% synthesis +# rate on Qwen3.5-4B UD-Q4_K_XL). Reframing as a concrete action ("write +# the final answer to the user's original question using what you have") +# is what moves the needle: the same bench run jumps to 90% synthesis. +_TOOL_SYNTHESISE_NUDGE = ( + " Tool results have been gathered. Now write the final answer to the" + " user's original question using what you have. Tool calls are no" + " longer needed for this turn." +) + # Regex for stripping leaked tool-call XML from assistant messages/stream _TOOL_XML_RE = _re.compile( r".*?|.*?", @@ -1242,7 +1270,16 @@ async def openai_chat_completions( _nudge = "" if _nudge: - _nudge += _TOOL_ACTION_NUDGE + _nudge += ( + _TOOL_ACTION_NUDGE_SMALL if _is_small_model else _TOOL_ACTION_NUDGE + ) + # If the current conversation already has a tool result + # message, append the synthesise-now directive. Covers + # forked chats and long-running sessions where the prior + # "prefer tools" line keeps biasing the model into search + # loops instead of answering from what it already has. + if any(m.get("role") == "tool" for m in chat_messages): + _nudge += _TOOL_SYNTHESISE_NUDGE # Append nudge to system prompt (preserve user's prompt) if system_prompt: system_prompt = system_prompt.rstrip() + "\n\n" + _nudge @@ -2468,7 +2505,11 @@ async def anthropic_messages( _nudge = "" if _nudge: - _nudge += _TOOL_ACTION_NUDGE + _nudge += ( + _TOOL_ACTION_NUDGE_SMALL if _is_small_model else _TOOL_ACTION_NUDGE + ) + if any(m.get("role") == "tool" for m in openai_messages): + _nudge += _TOOL_SYNTHESISE_NUDGE # Inject into system prompt if openai_messages and openai_messages[0].get("role") == "system": openai_messages[0]["content"] = ( From 646b4f02d941ac0e1af3eb88dc310fe4fd6704e8 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 16 Apr 2026 11:09:52 +0000 Subject: [PATCH 2/4] Address review: scope synthesis nudge to current turn and skip on errors - Gate _TOOL_SYNTHESISE_NUDGE on current turn only: scan backwards from the last message -- only append the synthesis directive when tool results appear after the last user message. Historical tool results from earlier questions no longer suppress legitimate new tool calls. - Skip synthesis nudge when all tool calls in a batch errored: if every tool execution returned an error (timeout, failed fetch, etc.), let the model retry or try a different approach rather than prematurely forcing a final answer from incomplete data. --- studio/backend/core/inference/llama_cpp.py | 12 ++++++-- studio/backend/routes/inference.py | 32 +++++++++++++++++----- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 921c8acf74..23f52e1126 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -3041,6 +3041,7 @@ class LlamaCppBackend: assistant_msg["tool_calls"] = tool_calls conversation.append(assistant_msg) + _any_tool_succeeded = False for tc in tool_calls or []: func = tc.get("function", {}) tool_name = func.get("name", "") @@ -3146,6 +3147,8 @@ class LlamaCppBackend: _error_prefixes ) _tool_call_history.append((_tc_key, _is_error)) + if not _is_error: + _any_tool_succeeded = True # Strip image sentinel before feeding result to the LLM # (the full result with sentinel is still yielded via # tool_end so the frontend can extract image paths). @@ -3168,9 +3171,12 @@ class LlamaCppBackend: tool_msg["tool_call_id"] = tool_call_id conversation.append(tool_msg) - # First tool result of the loop: tell the model to - # synthesise an answer rather than continue searching. - _apply_synthesise_nudge() + # First successful tool result of the loop: tell the model + # to synthesise an answer rather than continue searching. + # Skip if every tool call in this batch errored -- let the + # model retry or try a different approach instead. + if _any_tool_succeeded: + _apply_synthesise_nudge() # Clear tool status badge before next generation iteration yield {"type": "status", "text": ""} diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 63ce790271..1b019946b6 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -155,6 +155,24 @@ _TOOL_SYNTHESISE_NUDGE = ( " longer needed for this turn." ) + +def _has_tool_result_in_current_turn(messages: list[dict]) -> bool: + """Check if the current turn (after the last user message) has tool results. + + Scans backwards from the end -- if we find a tool result before hitting + a user message, the current turn has tool results. If we hit a user + message first (or the list is empty), the tool results are from prior + turns and should not trigger the synthesis nudge. + """ + for msg in reversed(messages): + role = msg.get("role") + if role == "tool": + return True + if role == "user": + return False + return False + + # Regex for stripping leaked tool-call XML from assistant messages/stream _TOOL_XML_RE = _re.compile( r".*?|.*?", @@ -1273,12 +1291,12 @@ async def openai_chat_completions( _nudge += ( _TOOL_ACTION_NUDGE_SMALL if _is_small_model else _TOOL_ACTION_NUDGE ) - # If the current conversation already has a tool result - # message, append the synthesise-now directive. Covers - # forked chats and long-running sessions where the prior - # "prefer tools" line keeps biasing the model into search - # loops instead of answering from what it already has. - if any(m.get("role") == "tool" for m in chat_messages): + # If the current turn (after the last user message) has + # tool results, append the synthesise directive. Only + # scoped to the current turn so that historical tool + # results from earlier questions do not suppress + # legitimate new tool calls. + if _has_tool_result_in_current_turn(chat_messages): _nudge += _TOOL_SYNTHESISE_NUDGE # Append nudge to system prompt (preserve user's prompt) if system_prompt: @@ -2508,7 +2526,7 @@ async def anthropic_messages( _nudge += ( _TOOL_ACTION_NUDGE_SMALL if _is_small_model else _TOOL_ACTION_NUDGE ) - if any(m.get("role") == "tool" for m in openai_messages): + if _has_tool_result_in_current_turn(openai_messages): _nudge += _TOOL_SYNTHESISE_NUDGE # Inject into system prompt if openai_messages and openai_messages[0].get("role") == "system": From 9dff999ed6743e4608aa86e87e25a2516e8a7f9d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 16 Apr 2026 11:31:57 +0000 Subject: [PATCH 3/4] Address review: gate synthesis nudge to small models and skip on errors - Synthesis nudge (both route-level and loop-level) now only applies to small models (<9B). Large models handle multi-step tool use correctly (e.g. web_search(query) -> web_search(url=...) -> python(...)) and should not be told "Tool calls are no longer needed." - Added synthesise_after_tool_result parameter to generate_chat_completion_with_tools(). Routes pass True only when _is_small_model, so the loop-level nudge is also model-size-aware. - Renamed _has_tool_result_in_current_turn to _has_successful_tool_result_in_current_turn. Now checks tool result content against error prefixes (Error, Failed to fetch, Blocked, etc.) so error-only tool results don't prematurely force synthesis -- the model can retry with a different query instead. --- studio/backend/core/inference/llama_cpp.py | 5 ++- studio/backend/routes/inference.py | 50 ++++++++++++++++------ 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 23f52e1126..77e19f0df9 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -2453,6 +2453,7 @@ class LlamaCppBackend: auto_heal_tool_calls: bool = True, tool_call_timeout: int = 300, session_id: Optional[str] = None, + synthesise_after_tool_result: bool = False, ) -> Generator[dict, None, None]: """ Agentic loop: let the model call tools, execute them, and continue. @@ -3173,9 +3174,11 @@ class LlamaCppBackend: # First successful tool result of the loop: tell the model # to synthesise an answer rather than continue searching. + # Only enabled for small models (via synthesise_after_tool_result) + # since large models handle multi-step tool use well. # Skip if every tool call in this batch errored -- let the # model retry or try a different approach instead. - if _any_tool_succeeded: + if synthesise_after_tool_result and _any_tool_succeeded: _apply_synthesise_nudge() # Clear tool status badge before next generation iteration diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 1b019946b6..ea4bc28bc7 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -156,18 +156,34 @@ _TOOL_SYNTHESISE_NUDGE = ( ) -def _has_tool_result_in_current_turn(messages: list[dict]) -> bool: - """Check if the current turn (after the last user message) has tool results. +_TOOL_ERROR_PREFIXES = ( + "Error", + "Search failed", + "Execution error", + "Blocked:", + "Exit code", + "Failed to fetch", + "Failed to resolve", + "No query provided", +) - Scans backwards from the end -- if we find a tool result before hitting - a user message, the current turn has tool results. If we hit a user - message first (or the list is empty), the tool results are from prior - turns and should not trigger the synthesis nudge. + +def _has_successful_tool_result_in_current_turn(messages: list[dict]) -> bool: + """Check if the current turn has at least one successful tool result. + + Scans backwards from the end of the message list. Only returns True + when a non-error tool result appears after the last user message. + Error-only tool results (timeouts, failed fetches, etc.) return False + so the model can retry rather than being forced to synthesise. """ for msg in reversed(messages): role = msg.get("role") if role == "tool": - return True + content = (msg.get("content") or "").lstrip() + if not content.startswith(_TOOL_ERROR_PREFIXES): + return True + # Error tool result -- keep scanning for a successful one + continue if role == "user": return False return False @@ -1291,12 +1307,12 @@ async def openai_chat_completions( _nudge += ( _TOOL_ACTION_NUDGE_SMALL if _is_small_model else _TOOL_ACTION_NUDGE ) - # If the current turn (after the last user message) has - # tool results, append the synthesise directive. Only - # scoped to the current turn so that historical tool - # results from earlier questions do not suppress - # legitimate new tool calls. - if _has_tool_result_in_current_turn(chat_messages): + # Small models loop on tool calls instead of answering. + # If the current turn already has a successful tool + # result, nudge the model to synthesise a final answer. + # Large models handle multi-step tool use well, so this + # is gated to small models only. + if _is_small_model and _has_successful_tool_result_in_current_turn(chat_messages): _nudge += _TOOL_SYNTHESISE_NUDGE # Append nudge to system prompt (preserve user's prompt) if system_prompt: @@ -1339,6 +1355,7 @@ async def openai_chat_completions( if payload.tool_call_timeout is not None else 300, session_id = payload.session_id, + synthesise_after_tool_result = _is_small_model, ) _tool_sentinel = object() @@ -2526,7 +2543,11 @@ async def anthropic_messages( _nudge += ( _TOOL_ACTION_NUDGE_SMALL if _is_small_model else _TOOL_ACTION_NUDGE ) - if _has_tool_result_in_current_turn(openai_messages): + # Only nudge small models to synthesise -- see comment in + # /chat/completions for rationale. (The OpenAI-compat schema + # does not yet accept role="tool", so this branch is currently + # unreachable; kept for when the schema is extended.) + if _is_small_model and _has_successful_tool_result_in_current_turn(openai_messages): _nudge += _TOOL_SYNTHESISE_NUDGE # Inject into system prompt if openai_messages and openai_messages[0].get("role") == "system": @@ -2558,6 +2579,7 @@ async def anthropic_messages( auto_heal_tool_calls = True, tool_call_timeout = 300, session_id = payload.session_id, + synthesise_after_tool_result = _is_small_model, ) if payload.stream: From a0506e1e7c3dc753d83bbc44aa735cf58ffa7494 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 Apr 2026 11:32:12 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/routes/inference.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index ea4bc28bc7..3db56b9c6b 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1312,7 +1312,9 @@ async def openai_chat_completions( # result, nudge the model to synthesise a final answer. # Large models handle multi-step tool use well, so this # is gated to small models only. - if _is_small_model and _has_successful_tool_result_in_current_turn(chat_messages): + if _is_small_model and _has_successful_tool_result_in_current_turn( + chat_messages + ): _nudge += _TOOL_SYNTHESISE_NUDGE # Append nudge to system prompt (preserve user's prompt) if system_prompt: @@ -2547,7 +2549,9 @@ async def anthropic_messages( # /chat/completions for rationale. (The OpenAI-compat schema # does not yet accept role="tool", so this branch is currently # unreachable; kept for when the schema is extended.) - if _is_small_model and _has_successful_tool_result_in_current_turn(openai_messages): + if _is_small_model and _has_successful_tool_result_in_current_turn( + openai_messages + ): _nudge += _TOOL_SYNTHESISE_NUDGE # Inject into system prompt if openai_messages and openai_messages[0].get("role") == "system":