From 9806e36aa4980ead7289d809df80616eabfddb4e Mon Sep 17 00:00:00 2001 From: Lee Jackson <130007945+Imagineer99@users.noreply.github.com> Date: Fri, 5 Jun 2026 11:46:04 +0100 Subject: [PATCH] Studio: enable GGUF tools with vision inputs (#6009) * fix: enable GGUF tools with vision inputs * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix: GGUF vision tool routing * Dedupe system messages on GGUF vision tool path for PR #6009 --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Daniel Han --- studio/backend/routes/inference.py | 30 ++-- .../tests/test_openai_tool_passthrough.py | 138 +++++++++++++++++- 2 files changed, 156 insertions(+), 12 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 15d73405cb..1d505e31f6 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -2780,7 +2780,7 @@ async def openai_chat_completions( detail = "Audio input is not supported for GGUF chat models yet.", ) - gguf_messages, has_gguf_image = _openai_messages_for_gguf_chat( + gguf_messages, _ = _openai_messages_for_gguf_chat( payload, llama_backend.is_vision, ) @@ -2804,11 +2804,7 @@ async def openai_chat_completions( _cli_policy = _get_tool_policy_g() _tools_on = _effective_enable_tools(payload) _mcp_allowed = bool(payload.mcp_enabled) and _cli_policy is not False - use_tools = ( - (_tools_on or _mcp_allowed) - and llama_backend.supports_tools - and not has_gguf_image - ) + use_tools = (_tools_on or _mcp_allowed) and llama_backend.supports_tools if use_tools: from core.inference.tools import ALL_TOOLS, get_enabled_mcp_tools @@ -2897,11 +2893,9 @@ async def openai_chat_completions( system_prompt = system_prompt.rstrip() + "\n\n" + _nudge else: system_prompt = _nudge - # Rebuild gguf_messages with updated system prompt - gguf_messages = [] - if system_prompt: - gguf_messages.append({"role": "system", "content": system_prompt}) - gguf_messages.extend(chat_messages) + gguf_messages = _set_or_prepend_system_message( + gguf_messages, system_prompt + ) # ── Strip stale tool-call XML from conversation history ─ for _msg in gguf_messages: @@ -4860,6 +4854,20 @@ def _normalize_anthropic_openai_images( return has_image +def _set_or_prepend_system_message( + messages: Optional[list[dict]], system_prompt: str +) -> list[dict]: + """Return messages with a single leading system prompt, preserving multimodal parts.""" + safe_messages = messages or [] + if not system_prompt: + return safe_messages + + # Drop existing system turns so the backend never sees duplicate or + # conflicting system instructions, then prepend the resolved prompt. + others = [dict(msg) for msg in safe_messages if msg.get("role") != "system"] + return [{"role": "system", "content": system_prompt}, *others] + + @router.post("/messages") async def anthropic_messages( payload: AnthropicMessagesRequest, diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py index 84f3e41998..05efee1595 100644 --- a/studio/backend/tests/test_openai_tool_passthrough.py +++ b/studio/backend/tests/test_openai_tool_passthrough.py @@ -20,6 +20,8 @@ No running server or GPU required. import os import sys +import asyncio +from types import SimpleNamespace _backend = os.path.join(os.path.dirname(__file__), "..") sys.path.insert(0, _backend) @@ -36,7 +38,13 @@ from models.inference import ( from core.inference.anthropic_compat import ( anthropic_tool_choice_to_openai, ) -from routes.inference import _build_passthrough_payload, _friendly_error +from routes.inference import ( + _build_passthrough_payload, + _friendly_error, + _set_or_prepend_system_message, + openai_chat_completions, +) +from state.tool_policy import reset_tool_policy # ===================================================================== @@ -725,3 +733,131 @@ class TestGgufVisionMessages: with pytest.raises(HTTPException) as exc_info: _openai_messages_for_gguf_chat(req, is_vision = False) assert "does not support vision" in str(exc_info.value) + + def test_tool_nudge_system_update_preserves_image_parts(self): + messages = [ + {"role": "system", "content": "Base instructions."}, + { + "role": "user", + "content": [ + {"type": "text", "text": "describe this"}, + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{self._PNG_B64}", + }, + }, + ], + }, + ] + + updated = _set_or_prepend_system_message( + messages, "Base instructions.\n\nUse tools when appropriate." + ) + + assert updated[0] == { + "role": "system", + "content": "Base instructions.\n\nUse tools when appropriate.", + } + assert updated[1]["content"][1]["type"] == "image_url" + assert messages[1]["content"][1]["type"] == "image_url" + + def test_tool_nudge_system_update_handles_none_messages(self): + assert _set_or_prepend_system_message(None, "") == [] + assert _set_or_prepend_system_message(None, "Use tools.") == [ + {"role": "system", "content": "Use tools."} + ] + + def test_tool_nudge_system_update_dedupes_non_leading_system(self): + messages = [ + {"role": "user", "content": "earlier"}, + {"role": "system", "content": "Mid instructions."}, + {"role": "user", "content": "now"}, + ] + + updated = _set_or_prepend_system_message( + messages, "Mid instructions.\n\nUse tools." + ) + + assert [m["role"] for m in updated] == ["system", "user", "user"] + assert updated[0]["content"] == "Mid instructions.\n\nUse tools." + + +class TestGgufVisionToolRouting: + class _Request: + async def is_disconnected(self): + return False + + @staticmethod + def _drive(coro): + return asyncio.run(coro) + + @staticmethod + def _consume_response(response): + async def _consume(): + chunks = [] + async for chunk in response.body_iterator: + chunks.append(chunk) + return chunks + + return TestGgufVisionToolRouting._drive(_consume()) + + def test_image_request_with_enabled_tools_enters_gguf_tool_loop(self, monkeypatch): + import routes.inference as inf_mod + + reset_tool_policy() + captured = {} + + def _plain(**kwargs): + raise AssertionError("plain GGUF path should not be used") + + def _tools(**kwargs): + captured["kwargs"] = kwargs + yield {"type": "content", "text": "done"} + + backend = SimpleNamespace( + is_loaded = True, + is_vision = True, + supports_tools = True, + model_identifier = "gemma-4-12b-it-GGUF", + generate_chat_completion = _plain, + generate_chat_completion_with_tools = _tools, + ) + monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: backend) + + payload = ChatCompletionRequest( + model = "default", + enable_tools = True, + enabled_tools = ["web_search"], + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": { + "url": ( + "data:image/png;base64," + f"{TestGgufVisionMessages._PNG_B64}" + ), + }, + }, + ], + }, + ], + ) + + response = self._drive( + openai_chat_completions( + payload, request = self._Request(), current_subject = "test" + ) + ) + self._consume_response(response) + + assert "kwargs" in captured + assert captured["kwargs"]["tools"] + tool_messages = captured["kwargs"]["messages"] + assert tool_messages[0]["role"] == "system" + assert tool_messages[1]["role"] == "user" + assert tool_messages[1]["content"][1]["type"] == "image_url"