From 67e371934c8d889ebecae58da48bb982e0033dda Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 17:35:05 +0000 Subject: [PATCH] Enforce parallel_tool_calls=False client-side on local GGUF (PR #5711) Two related findings from round 12 reviewers: 1. The local GGUF tool loop in `generate_chat_completion_with_tools` iterates every entry of `tool_calls` returned by llama-server, even when the caller explicitly opted out of parallel tool calls. The `parallel_tool_calls` flag is forwarded to llama-server, but llama .cpp does not enforce it on every jinja template (https://github.com/ggml-org/llama.cpp/issues/22043), so a model that ignores the flag still ran multiple tools per turn. Cap `tool_calls` to the first entry when the flag is False so the client-side contract holds regardless of upstream behavior. 2. llama-server documents `parallel_tool_calls` as defaulting to FALSE (https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md), so the previous chat-adapter shape (forward only on explicit false) meant the UI's default-on state could never enable parallel tool calls there. Always forward the user's preference on the local path so the toggle actually does what it says. External providers default to true everywhere, so the external branch is unchanged. Test pins the GGUF tool-loop cap by source-level assertion (the loop itself is integration-only). --- studio/backend/core/inference/llama_cpp.py | 9 +++++++++ .../tests/test_sampling_params_routing.py | 20 +++++++++++++++++++ .../src/features/chat/api/chat-adapter.ts | 12 +++++++---- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index b26ddaf1cf..0a8a86136c 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -5015,6 +5015,15 @@ class LlamaCppBackend: _accumulated_predicted_ms += _it.get("predicted_ms", 0) _accumulated_predicted_n += _it.get("predicted_n", 0) + # When the caller opted out of parallel tool calls + # (parallel_tool_calls=False), enforce at most one call + # per assistant turn even if llama-server emitted more. + # llama.cpp's parallel_tool_calls flag isn't enforced by + # every jinja template (see ggml-org/llama.cpp#22043), + # so this client-side cap is the only guarantee. + if parallel_tool_calls is False and tool_calls: + tool_calls = tool_calls[:1] + assistant_msg = {"role": "assistant", "content": content_text} if tool_calls: assistant_msg["tool_calls"] = tool_calls diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index 431ff4cd65..0143b7770d 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -876,6 +876,26 @@ def test_local_anthropic_disable_parallel_tool_use_translation(): assert _extract({"type": "auto", "disable_parallel_tool_use": "yes"}) is None +def test_gguf_tool_loop_enforces_parallel_tool_calls_false(): + """llama.cpp's `parallel_tool_calls` flag is not enforced by every + jinja template (see ggml-org/llama.cpp#22043), so when the caller + opted out we must cap tool_calls to the first entry before the + agentic loop executes them. The cap is a single-line slice in + `generate_chat_completion_with_tools`; pin the contract.""" + from pathlib import Path + + src = Path(__file__).resolve().parent.parent / "core/inference/llama_cpp.py" + text = src.read_text() + assert "if parallel_tool_calls is False and tool_calls" in text, ( + "GGUF tool loop must enforce parallel_tool_calls=False by " + "truncating tool_calls before assistant_msg is built; that " + "is the client-side guarantee llama-server's flag does not " + "give us. See routes/inference.py and chat-adapter.ts for " + "the wire-side forwarding of the same flag." + ) + assert "tool_calls = tool_calls[:1]" in text + + def test_local_anthropic_passthrough_helpers_accept_parallel_tool_calls(): """The Anthropic-compat client-tool passthrough helpers (`_anthropic_passthrough_stream` / diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index 6f01c0becd..97533c4f7d 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -1529,15 +1529,19 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { // Optional sampling extensions; local llama-server already // accepts `stop` / `seed` / `frequency_penalty` via // _build_passthrough_payload (routes/inference.py:4884) and - // silently ignores fields it does not recognise. + // silently ignores fields it does not recognise. llama-server + // documents `parallel_tool_calls` defaulting to FALSE + // (https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md); + // forward the user's preference unconditionally so the + // default-on UI state actually enables parallel tool calls + // there. External providers default to true everywhere; the + // external branch above keeps its opt-in-on-false shape. ...(params.frequencyPenalty !== 0 ? { frequency_penalty: params.frequencyPenalty } : {}), ...(params.seed !== null ? { seed: params.seed } : {}), ...(params.stop.length > 0 ? { stop: params.stop } : {}), - ...(params.parallelToolCalls === false - ? { parallel_tool_calls: false } - : {}), + parallel_tool_calls: params.parallelToolCalls, image_base64: imageBase64, audio_base64: audioBase64, cancel_id: cancelId,