Studio: enable GGUF tools with vision inputs (#6009)

* fix: enable GGUF tools with vision inputs

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix: GGUF vision tool routing

* Dedupe system messages on GGUF vision tool path for PR #6009

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Daniel Han <danielhanchen@gmail.com>

studio/backend/routes/inference.py

@@ -2780,7 +2780,7 @@ async def openai_chat_completions(
                 detail = "Audio input is not supported for GGUF chat models yet.",
             )
 
-        gguf_messages, has_gguf_image = _openai_messages_for_gguf_chat(
+        gguf_messages, _ = _openai_messages_for_gguf_chat(
             payload,
             llama_backend.is_vision,
         )
@@ -2804,11 +2804,7 @@ async def openai_chat_completions(
         _cli_policy = _get_tool_policy_g()
         _tools_on = _effective_enable_tools(payload)
         _mcp_allowed = bool(payload.mcp_enabled) and _cli_policy is not False
-        use_tools = (
-            (_tools_on or _mcp_allowed)
-            and llama_backend.supports_tools
-            and not has_gguf_image
-        )
+        use_tools = (_tools_on or _mcp_allowed) and llama_backend.supports_tools
 
         if use_tools:
             from core.inference.tools import ALL_TOOLS, get_enabled_mcp_tools
@@ -2897,11 +2893,9 @@ async def openai_chat_completions(
                     system_prompt = system_prompt.rstrip() + "\n\n" + _nudge
                 else:
                     system_prompt = _nudge
-                # Rebuild gguf_messages with updated system prompt
-                gguf_messages = []
-                if system_prompt:
-                    gguf_messages.append({"role": "system", "content": system_prompt})
-                gguf_messages.extend(chat_messages)
+                gguf_messages = _set_or_prepend_system_message(
+                    gguf_messages, system_prompt
+                )
 
             # ── Strip stale tool-call XML from conversation history ─
             for _msg in gguf_messages:
@@ -4860,6 +4854,20 @@ def _normalize_anthropic_openai_images(
     return has_image
 
 
+def _set_or_prepend_system_message(
+    messages: Optional[list[dict]], system_prompt: str
+) -> list[dict]:
+    """Return messages with a single leading system prompt, preserving multimodal parts."""
+    safe_messages = messages or []
+    if not system_prompt:
+        return safe_messages
+
+    # Drop existing system turns so the backend never sees duplicate or
+    # conflicting system instructions, then prepend the resolved prompt.
+    others = [dict(msg) for msg in safe_messages if msg.get("role") != "system"]
+    return [{"role": "system", "content": system_prompt}, *others]
+
+
 @router.post("/messages")
 async def anthropic_messages(
     payload: AnthropicMessagesRequest,

studio/backend/tests/test_openai_tool_passthrough.py

@@ -20,6 +20,8 @@ No running server or GPU required.
 
 import os
 import sys
+import asyncio
+from types import SimpleNamespace
 
 _backend = os.path.join(os.path.dirname(__file__), "..")
 sys.path.insert(0, _backend)
@@ -36,7 +38,13 @@ from models.inference import (
 from core.inference.anthropic_compat import (
     anthropic_tool_choice_to_openai,
 )
-from routes.inference import _build_passthrough_payload, _friendly_error
+from routes.inference import (
+    _build_passthrough_payload,
+    _friendly_error,
+    _set_or_prepend_system_message,
+    openai_chat_completions,
+)
+from state.tool_policy import reset_tool_policy
 
 
 # =====================================================================
@@ -725,3 +733,131 @@ class TestGgufVisionMessages:
         with pytest.raises(HTTPException) as exc_info:
             _openai_messages_for_gguf_chat(req, is_vision = False)
         assert "does not support vision" in str(exc_info.value)
+
+    def test_tool_nudge_system_update_preserves_image_parts(self):
+        messages = [
+            {"role": "system", "content": "Base instructions."},
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "describe this"},
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": f"data:image/png;base64,{self._PNG_B64}",
+                        },
+                    },
+                ],
+            },
+        ]
+
+        updated = _set_or_prepend_system_message(
+            messages, "Base instructions.\n\nUse tools when appropriate."
+        )
+
+        assert updated[0] == {
+            "role": "system",
+            "content": "Base instructions.\n\nUse tools when appropriate.",
+        }
+        assert updated[1]["content"][1]["type"] == "image_url"
+        assert messages[1]["content"][1]["type"] == "image_url"
+
+    def test_tool_nudge_system_update_handles_none_messages(self):
+        assert _set_or_prepend_system_message(None, "") == []
+        assert _set_or_prepend_system_message(None, "Use tools.") == [
+            {"role": "system", "content": "Use tools."}
+        ]
+
+    def test_tool_nudge_system_update_dedupes_non_leading_system(self):
+        messages = [
+            {"role": "user", "content": "earlier"},
+            {"role": "system", "content": "Mid instructions."},
+            {"role": "user", "content": "now"},
+        ]
+
+        updated = _set_or_prepend_system_message(
+            messages, "Mid instructions.\n\nUse tools."
+        )
+
+        assert [m["role"] for m in updated] == ["system", "user", "user"]
+        assert updated[0]["content"] == "Mid instructions.\n\nUse tools."
+
+
+class TestGgufVisionToolRouting:
+    class _Request:
+        async def is_disconnected(self):
+            return False
+
+    @staticmethod
+    def _drive(coro):
+        return asyncio.run(coro)
+
+    @staticmethod
+    def _consume_response(response):
+        async def _consume():
+            chunks = []
+            async for chunk in response.body_iterator:
+                chunks.append(chunk)
+            return chunks
+
+        return TestGgufVisionToolRouting._drive(_consume())
+
+    def test_image_request_with_enabled_tools_enters_gguf_tool_loop(self, monkeypatch):
+        import routes.inference as inf_mod
+
+        reset_tool_policy()
+        captured = {}
+
+        def _plain(**kwargs):
+            raise AssertionError("plain GGUF path should not be used")
+
+        def _tools(**kwargs):
+            captured["kwargs"] = kwargs
+            yield {"type": "content", "text": "done"}
+
+        backend = SimpleNamespace(
+            is_loaded = True,
+            is_vision = True,
+            supports_tools = True,
+            model_identifier = "gemma-4-12b-it-GGUF",
+            generate_chat_completion = _plain,
+            generate_chat_completion_with_tools = _tools,
+        )
+        monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: backend)
+
+        payload = ChatCompletionRequest(
+            model = "default",
+            enable_tools = True,
+            enabled_tools = ["web_search"],
+            messages = [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": "What is in this image?"},
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": (
+                                    "data:image/png;base64,"
+                                    f"{TestGgufVisionMessages._PNG_B64}"
+                                ),
+                            },
+                        },
+                    ],
+                },
+            ],
+        )
+
+        response = self._drive(
+            openai_chat_completions(
+                payload, request = self._Request(), current_subject = "test"
+            )
+        )
+        self._consume_response(response)
+
+        assert "kwargs" in captured
+        assert captured["kwargs"]["tools"]
+        tool_messages = captured["kwargs"]["messages"]
+        assert tool_messages[0]["role"] == "system"
+        assert tool_messages[1]["role"] == "user"
+        assert tool_messages[1]["content"][1]["type"] == "image_url"