ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(studio): forward OpenAI tools/tool_choice to llama-server (#4999)

Browse source 
Studio's /v1/chat/completions silently stripped standard OpenAI `tools`
and `tool_choice` fields, so clients using standard function calling
(opencode, Claude Code, Cursor, Continue, ...) never got structured
tool_calls back. Adds a client-side pass-through path mirroring the
existing Anthropic /v1/messages flow: when `tools` is present without
Studio's `enable_tools` shorthand, the request is forwarded to
llama-server verbatim so the client sees native id, finish_reason
("tool_calls"), delta.tool_calls, and accurate usage tokens.

Also wires Anthropic tool_choice forwarding: /v1/messages previously
accepted tool_choice on the request model but silently dropped it with
a warning. Translate the four Anthropic shapes to OpenAI format and
forward them so agentic clients can actually enforce tool use.

- ChatCompletionRequest: add tools, tool_choice, stop; extra="allow"
- ChatMessage: accept role="tool", optional tool_call_id / tool_calls /
  name; content is now optional (assistant with only tool_calls)
- routes/inference.py: _openai_passthrough_stream /
  _openai_passthrough_non_streaming helpers, routing branch in
  openai_chat_completions, vision+tools via content-parts injection
- _build_passthrough_payload: tool_choice parameter (default "auto")
- anthropic_compat: anthropic_tool_choice_to_openai() translator
- tests/test_openai_tool_passthrough.py: Pydantic + translator unit tests
- tests/test_studio_api.py: 5 new E2E tests (non-stream, stream,
  multi-turn, OpenAI SDK, Anthropic tool_choice=any regression)
This commit is contained in:
Roland Tannous 2026-04-15 17:18:35 +04:00
commit 5043333c94
5 changed files with 1009 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

33
studio/backend/core/inference/anthropic_compat.py
View file

@ -114,6 +114,39 @@ def anthropic_tools_to_openai(tools: list) -> list[dict]:
return result
def anthropic_tool_choice_to_openai(tc: Any) -> Any:
"""Translate Anthropic `tool_choice` into OpenAI `tool_choice`.
Anthropic formats (all dict shapes with a ``type`` discriminator):
- ``{"type": "auto"}`` → ``"auto"``
- ``{"type": "any"}`` → ``"required"``
- ``{"type": "none"}`` → ``"none"``
- ``{"type": "tool", "name": "get_weather"}``
→ ``{"type": "function", "function": {"name": "get_weather"}}``
Returns ``None`` for ``None`` or any unrecognized shape (caller may
then fall back to its own default, typically ``"auto"``).
"""
if tc is None:
return None
if not isinstance(tc, dict):
return None
t = tc.get("type")
if t == "auto":
return "auto"
if t == "any":
return "required"
if t == "none":
return "none"
if t == "tool":
name = tc.get("name")
if not name:
return None
return {"type": "function", "function": {"name": name}}
return None
def build_anthropic_sse_event(event_type: str, data: dict) -> str:
"""Format a single Anthropic SSE event."""
return f"event: {event_type}\ndata: {json.dumps(data)}\n\n"

47
studio/backend/models/inference.py
View file

@ -338,13 +338,29 @@ class ChatMessage(BaseModel):
``content`` may be a plain string (text-only) or a list of
content parts for multimodal messages (OpenAI vision format).
Assistant messages that only contain tool calls may set ``content``
to ``None`` with ``tool_calls`` populated. ``role="tool"`` messages
carry the result of a client-executed tool call and require
``tool_call_id`` per the OpenAI spec.
"""
role: Literal["system", "user", "assistant"] = Field(
role: Literal["system", "user", "assistant", "tool"] = Field(
..., description = "Message role"
)
content: Union[str, list[ContentPart]] = Field(
..., description = "Message content (string or multimodal parts)"
content: Optional[Union[str, list[ContentPart]]] = Field(
None, description = "Message content (string or multimodal parts)"
)
tool_call_id: Optional[str] = Field(
None,
description = "OpenAI tool-result messages: id of the tool call this result belongs to.",
)
tool_calls: Optional[list[dict]] = Field(
None,
description = "OpenAI assistant messages: structured tool calls the model decided to make.",
)
name: Optional[str] = Field(
None,
description = "OpenAI tool-result messages: name of the tool whose result this is.",
)
@ -355,6 +371,12 @@ class ChatCompletionRequest(BaseModel):
Extensions (non-OpenAI fields) are marked with 'x-unsloth'.
"""
# Accept unknown fields defensively so future OpenAI fields (seed,
# response_format, logprobs, frequency_penalty, etc.) don't get
# silently dropped by Pydantic before route code runs. Mirrors
# AnthropicMessagesRequest and ResponsesRequest.
model_config = {"extra": "allow"}
model: str = Field(
"default",
description = "Model identifier (informational; the active model is used)",
@ -367,6 +389,25 @@ class ChatCompletionRequest(BaseModel):
None, ge = 1, description = "Maximum tokens to generate (None = until EOS)"
)
presence_penalty: float = Field(0.0, ge = 0.0, le = 2.0, description = "Presence penalty")
stop: Optional[Union[str, list[str]]] = Field(
None,
description = "OpenAI stop sequences: a single string or list of strings at which generation halts.",
)
tools: Optional[list[dict]] = Field(
None,
description = (
"OpenAI function-tool definitions. When provided without `enable_tools=true`, "
"Studio forwards the tools to the backend so the model returns structured "
"tool_calls for the client to execute (standard OpenAI function calling)."
),
)
tool_choice: Optional[Union[str, dict]] = Field(
None,
description = (
"OpenAI tool choice: 'auto' | 'required' | 'none' | "
"{'type': 'function', 'function': {'name': ...}}"
),
)
# ── Unsloth extensions (ignored by standard OpenAI clients) ──
top_k: int = Field(20, ge = -1, le = 100, description = "[x-unsloth] Top-k sampling")

274
studio/backend/routes/inference.py
View file

@ -106,6 +106,7 @@ from models.inference import (
from core.inference.anthropic_compat import (
anthropic_messages_to_openai,
anthropic_tools_to_openai,
anthropic_tool_choice_to_openai,
AnthropicStreamEmitter,
AnthropicPassthroughEmitter,
)
@ -1122,6 +1123,39 @@ async def openai_chat_completions(
)
return JSONResponse(content = response.model_dump())
# ── Standard OpenAI function-calling pass-through (GGUF only) ────
# When a client (opencode / Claude Code via OpenAI compat / Cursor /
# Continue / ...) sends standard OpenAI `tools` without Studio's
# `enable_tools` shorthand, forward the request to llama-server
# verbatim so structured `tool_calls` flow back to the client. This
# branch runs BEFORE `_extract_content_parts` because that helper is
# unaware of `role="tool"` messages and assistant messages that only
# carry `tool_calls` (content=None) — both of which are valid in
# multi-turn client-side tool loops.
_has_tool_messages = any(m.role == "tool" or m.tool_calls for m in payload.messages)
if (
using_gguf
and llama_backend.supports_tools
and not payload.enable_tools
and ((payload.tools and len(payload.tools) > 0) or _has_tool_messages)
):
cancel_event = threading.Event()
completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"
if payload.stream:
return await _openai_passthrough_stream(
request,
cancel_event,
llama_backend,
payload,
model_name,
completion_id,
)
return await _openai_passthrough_non_streaming(
llama_backend,
payload,
model_name,
)
# ── Parse messages (handles multimodal content parts) ─────
system_prompt, chat_messages, extracted_image_b64 = _extract_content_parts(
payload.messages
@ -2339,22 +2373,12 @@ async def anthropic_messages(
)
stop = payload.stop_sequences or None
# tool_choice is declared on AnthropicMessagesRequest for Anthropic SDK
# compatibility (the SDK often sets it by default), but it is not
# currently honored by Unsloth's backend. Warn once per request so the
# silent drop is visible to operators instead of looking like a model
# quality issue to clients.
if payload.tool_choice is not None:
logger.warning(
"anthropic_messages.tool_choice_ignored",
tool_choice = payload.tool_choice,
note = (
"tool_choice is accepted for Anthropic SDK compatibility but not "
"honored by Unsloth. Use enable_tools / enabled_tools (server-side "
"built-in tools) or restrict the `tools` array (client-side) to "
"control which tools the model sees."
),
)
# Translate Anthropic tool_choice to OpenAI format for forwarding to
# llama-server. Falls back to "auto" when unset or unrecognized, which
# matches the prior hardcoded behavior.
openai_tool_choice = anthropic_tool_choice_to_openai(payload.tool_choice)
if openai_tool_choice is None:
openai_tool_choice = "auto"
cancel_event = threading.Event()
@ -2392,6 +2416,7 @@ async def anthropic_messages(
min_p = min_p,
repetition_penalty = repetition_penalty,
presence_penalty = presence_penalty,
tool_choice = openai_tool_choice,
)
return await _anthropic_passthrough_non_streaming(
llama_backend,
@ -2407,6 +2432,7 @@ async def anthropic_messages(
min_p = min_p,
repetition_penalty = repetition_penalty,
presence_penalty = presence_penalty,
tool_choice = openai_tool_choice,
)
if server_tools:
@ -2750,11 +2776,12 @@ def _build_passthrough_payload(
min_p = None,
repetition_penalty = None,
presence_penalty = None,
tool_choice = "auto",
):
body = {
"messages": openai_messages,
"tools": openai_tools,
"tool_choice": "auto",
"tool_choice": tool_choice,
"temperature": temperature,
"top_p": top_p,
"top_k": top_k,
@ -2792,6 +2819,7 @@ async def _anthropic_passthrough_stream(
min_p = None,
repetition_penalty = None,
presence_penalty = None,
tool_choice = "auto",
):
"""Streaming client-side pass-through: forward tools to llama-server and
translate its streaming response to Anthropic SSE without executing anything."""
@ -2808,6 +2836,7 @@ async def _anthropic_passthrough_stream(
min_p = min_p,
repetition_penalty = repetition_penalty,
presence_penalty = presence_penalty,
tool_choice = tool_choice,
)
async def _stream():
@ -2897,6 +2926,7 @@ async def _anthropic_passthrough_non_streaming(
min_p = None,
repetition_penalty = None,
presence_penalty = None,
tool_choice = "auto",
):
"""Non-streaming client-side pass-through."""
target_url = f"{llama_backend.base_url}/v1/chat/completions"
@ -2912,6 +2942,7 @@ async def _anthropic_passthrough_non_streaming(
min_p = min_p,
repetition_penalty = repetition_penalty,
presence_penalty = presence_penalty,
tool_choice = tool_choice,
)
async with httpx.AsyncClient() as client:
@ -2969,3 +3000,212 @@ async def _anthropic_passthrough_non_streaming(
),
)
return JSONResponse(content = resp_obj.model_dump())
# =====================================================================
# Client-side tool pass-through (OpenAI-native /v1/chat/completions)
# =====================================================================
def _openai_messages_for_passthrough(payload) -> list[dict]:
"""Build OpenAI-format message dicts for the /v1/chat/completions
passthrough path.
Messages from ``payload.messages`` are dumped through Pydantic (dropping
unset optional fields) so they are already in standard OpenAI format
— including ``role="tool"`` tool-result messages and assistant messages
that carry structured ``tool_calls``. Content-parts images already in
the message list are left untouched.
When a client uses Studio's legacy ``image_base64`` top-level field, the
image is re-encoded to PNG (llama-server's stb_image has limited format
support) and spliced into the last user message as an OpenAI
``image_url`` content part so vision + function-calling requests work
transparently.
"""
messages = [m.model_dump(exclude_none = True) for m in payload.messages]
if not payload.image_base64:
return messages
try:
import base64 as _b64
from io import BytesIO as _BytesIO
from PIL import Image as _Image
raw = _b64.b64decode(payload.image_base64)
img = _Image.open(_BytesIO(raw))
if img.mode == "RGBA":
img = img.convert("RGB")
buf = _BytesIO()
img.save(buf, format = "PNG")
png_b64 = _b64.b64encode(buf.getvalue()).decode("ascii")
except Exception as e:
raise HTTPException(
status_code = 400,
detail = f"Failed to process image: {e}",
)
data_url = f"data:image/png;base64,{png_b64}"
image_part = {"type": "image_url", "image_url": {"url": data_url}}
for msg in reversed(messages):
if msg.get("role") != "user":
continue
existing = msg.get("content")
if isinstance(existing, str):
msg["content"] = [{"type": "text", "text": existing}, image_part]
elif isinstance(existing, list):
existing.append(image_part)
else:
msg["content"] = [image_part]
break
else:
messages.append({"role": "user", "content": [image_part]})
return messages
def _build_openai_passthrough_body(payload) -> dict:
"""Assemble the llama-server request body from a ChatCompletionRequest.
Only explicitly-known OpenAI / llama-server fields are forwarded so that
Studio-specific extensions (``enable_tools``, ``enabled_tools``,
``session_id``, ...) never leak to the backend.
"""
messages = _openai_messages_for_passthrough(payload)
tool_choice = payload.tool_choice if payload.tool_choice is not None else "auto"
return _build_passthrough_payload(
messages,
payload.tools,
payload.temperature,
payload.top_p,
payload.top_k,
payload.max_tokens,
payload.stream,
stop = payload.stop,
min_p = payload.min_p,
repetition_penalty = payload.repetition_penalty,
presence_penalty = payload.presence_penalty,
tool_choice = tool_choice,
)
async def _openai_passthrough_stream(
request,
cancel_event,
llama_backend,
payload,
model_name,
completion_id,
):
"""Streaming client-side pass-through for /v1/chat/completions.
Forwards the client's OpenAI function-calling request to llama-server and
relays the SSE stream back verbatim. This preserves llama-server's
native response ``id``, ``finish_reason`` (including ``"tool_calls"``),
``delta.tool_calls``, and the trailing ``usage`` chunk so the client
observes a standard OpenAI response.
"""
target_url = f"{llama_backend.base_url}/v1/chat/completions"
body = _build_openai_passthrough_body(payload)
async def _stream():
# Same httpx lifecycle pattern as _anthropic_passthrough_stream:
# avoid `async with` on the client/response to sidestep the Python
# 3.13 + httpcore 1.0.x anyio cancel-scope bug when the async
# generator is garbage-collected from a different task than the
# one that originally entered the context managers.
client = httpx.AsyncClient(timeout = 600)
resp = None
try:
req = client.build_request("POST", target_url, json = body)
resp = await client.send(req, stream = True)
if resp.status_code != 200:
err_bytes = await resp.aread()
err_text = err_bytes.decode("utf-8", errors = "replace")
logger.error(
"openai passthrough upstream error: status=%s body=%s",
resp.status_code,
err_text[:500],
)
err = {
"error": {
"message": f"llama-server error: {err_text[:500]}",
"type": "server_error",
},
}
yield f"data: {json.dumps(err)}\n\n"
return
async for raw_line in resp.aiter_lines():
if await request.is_disconnected():
cancel_event.set()
break
if not raw_line:
continue
if not raw_line.startswith("data: "):
continue
# Relay the llama-server SSE chunk verbatim so the client
# sees its native `id`, `finish_reason`, `delta.tool_calls`,
# and final `usage` unchanged.
yield raw_line + "\n\n"
if raw_line[6:].strip() == "[DONE]":
break
except Exception as e:
logger.error("openai passthrough stream error: %s", e)
err = {
"error": {
"message": _friendly_error(e),
"type": "server_error",
},
}
yield f"data: {json.dumps(err)}\n\n"
finally:
if resp is not None:
try:
await resp.aclose()
except Exception:
pass
try:
await client.aclose()
except Exception:
pass
return StreamingResponse(
_stream(),
media_type = "text/event-stream",
headers = {
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"X-Accel-Buffering": "no",
},
)
async def _openai_passthrough_non_streaming(
llama_backend,
payload,
model_name,
):
"""Non-streaming client-side pass-through for /v1/chat/completions.
Returns llama-server's JSON response verbatim (via JSONResponse) so the
client sees the native response ``id``, ``finish_reason`` (including
``"tool_calls"``), structured ``tool_calls``, and accurate ``usage``
token counts.
"""
target_url = f"{llama_backend.base_url}/v1/chat/completions"
body = _build_openai_passthrough_body(payload)
async with httpx.AsyncClient() as client:
resp = await client.post(target_url, json = body, timeout = 600)
if resp.status_code != 200:
raise HTTPException(
status_code = resp.status_code,
detail = f"llama-server error: {resp.text[:500]}",
)
return JSONResponse(content = resp.json())

326
studio/backend/tests/test_openai_tool_passthrough.py Normal file
View file

@ -0,0 +1,326 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""
Tests for the OpenAI /v1/chat/completions client-side tool pass-through.
Covers:
- ChatCompletionRequest accepts standard OpenAI `tools` / `tool_choice` / `stop`.
- ChatMessage accepts role="tool" with `tool_call_id` and role="assistant"
with `content: None` + `tool_calls`.
- ChatCompletionRequest carries unknown fields via `extra="allow"`.
- anthropic_tool_choice_to_openai() covers all four Anthropic shapes.
- _build_passthrough_payload() honors a caller-supplied tool_choice and
defaults to "auto" when unset.
No running server or GPU required.
"""
import os
import sys
_backend = os.path.join(os.path.dirname(__file__), "..")
sys.path.insert(0, _backend)
import pytest
from pydantic import ValidationError
from models.inference import (
ChatCompletionRequest,
ChatMessage,
)
from core.inference.anthropic_compat import (
anthropic_tool_choice_to_openai,
)
from routes.inference import _build_passthrough_payload
# =====================================================================
# ChatMessage — tool role, tool_calls, optional content
# =====================================================================
class TestChatMessageToolRoles:
def test_tool_role_with_tool_call_id(self):
msg = ChatMessage(
role = "tool",
tool_call_id = "call_abc123",
content = '{"temperature": 72}',
)
assert msg.role == "tool"
assert msg.tool_call_id == "call_abc123"
assert msg.content == '{"temperature": 72}'
def test_tool_role_with_name(self):
msg = ChatMessage(
role = "tool",
tool_call_id = "call_abc123",
name = "get_weather",
content = '{"temperature": 72}',
)
assert msg.name == "get_weather"
def test_assistant_with_tool_calls_no_content(self):
msg = ChatMessage(
role = "assistant",
content = None,
tool_calls = [
{
"id": "call_1",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"city": "Paris"}',
},
}
],
)
assert msg.role == "assistant"
assert msg.content is None
assert msg.tool_calls is not None
assert len(msg.tool_calls) == 1
assert msg.tool_calls[0]["function"]["name"] == "get_weather"
def test_assistant_with_content_and_tool_calls(self):
msg = ChatMessage(
role = "assistant",
content = "Let me check the weather.",
tool_calls = [
{
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": "{}"},
}
],
)
assert msg.content == "Let me check the weather."
assert msg.tool_calls[0]["id"] == "call_1"
def test_plain_user_message_still_works(self):
msg = ChatMessage(role = "user", content = "Hello")
assert msg.role == "user"
assert msg.tool_call_id is None
assert msg.tool_calls is None
assert msg.name is None
def test_invalid_role_rejected(self):
with pytest.raises(ValidationError):
ChatMessage(role = "function", content = "x")
def test_content_absent_defaults_to_none(self):
msg = ChatMessage(role = "assistant")
assert msg.content is None
# =====================================================================
# ChatCompletionRequest — standard OpenAI tool fields
# =====================================================================
class TestChatCompletionRequestToolFields:
def _make(self, **kwargs):
base = {"messages": [{"role": "user", "content": "Hi"}]}
base.update(kwargs)
return ChatCompletionRequest(**base)
def test_tools_parses(self):
req = self._make(
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Return the weather in a city",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
}
],
)
assert req.tools is not None
assert len(req.tools) == 1
assert req.tools[0]["function"]["name"] == "get_weather"
def test_tool_choice_string_auto(self):
assert self._make(tool_choice = "auto").tool_choice == "auto"
def test_tool_choice_string_required(self):
assert self._make(tool_choice = "required").tool_choice == "required"
def test_tool_choice_string_none(self):
assert self._make(tool_choice = "none").tool_choice == "none"
def test_tool_choice_named_function(self):
tc = {"type": "function", "function": {"name": "get_weather"}}
assert self._make(tool_choice = tc).tool_choice == tc
def test_stop_string(self):
assert self._make(stop = "\nUser:").stop == "\nUser:"
def test_stop_list(self):
assert self._make(stop = ["\nUser:", "\nAssistant:"]).stop == [
"\nUser:",
"\nAssistant:",
]
def test_tools_default_none(self):
req = self._make()
assert req.tools is None
assert req.tool_choice is None
assert req.stop is None
def test_extra_fields_accepted(self):
# `frequency_penalty`, `seed`, `response_format` are not yet
# explicitly declared but must survive Pydantic parsing now that
# extra="allow" is set.
req = self._make(
frequency_penalty = 0.5,
seed = 42,
response_format = {"type": "json_object"},
)
# Extras land in model_extra
assert req.model_extra is not None
assert req.model_extra.get("frequency_penalty") == 0.5
assert req.model_extra.get("seed") == 42
assert req.model_extra.get("response_format") == {"type": "json_object"}
def test_unsloth_extensions_still_work(self):
req = self._make(
enable_tools = True,
enabled_tools = ["web_search", "python"],
session_id = "abc",
)
assert req.enable_tools is True
assert req.enabled_tools == ["web_search", "python"]
assert req.session_id == "abc"
def test_multiturn_tool_loop_messages(self):
req = ChatCompletionRequest(
messages = [
{"role": "user", "content": "What's the weather in Paris?"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"city": "Paris"}',
},
}
],
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": '{"temperature": 14, "unit": "celsius"}',
},
],
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {"type": "object"},
},
}
],
)
assert len(req.messages) == 3
assert req.messages[1].role == "assistant"
assert req.messages[1].content is None
assert req.messages[1].tool_calls[0]["id"] == "call_1"
assert req.messages[2].role == "tool"
assert req.messages[2].tool_call_id == "call_1"
# =====================================================================
# anthropic_tool_choice_to_openai — pure translation helper
# =====================================================================
class TestAnthropicToolChoiceToOpenAI:
def test_auto(self):
assert anthropic_tool_choice_to_openai({"type": "auto"}) == "auto"
def test_any_becomes_required(self):
assert anthropic_tool_choice_to_openai({"type": "any"}) == "required"
def test_none(self):
assert anthropic_tool_choice_to_openai({"type": "none"}) == "none"
def test_tool_named(self):
result = anthropic_tool_choice_to_openai(
{"type": "tool", "name": "get_weather"}
)
assert result == {
"type": "function",
"function": {"name": "get_weather"},
}
def test_tool_missing_name_returns_none(self):
assert anthropic_tool_choice_to_openai({"type": "tool"}) is None
def test_none_input_returns_none(self):
assert anthropic_tool_choice_to_openai(None) is None
def test_unrecognized_shape_returns_none(self):
assert anthropic_tool_choice_to_openai({"type": "wibble"}) is None
assert anthropic_tool_choice_to_openai("auto") is None
assert anthropic_tool_choice_to_openai(42) is None
# =====================================================================
# _build_passthrough_payload — tool_choice propagation
# =====================================================================
class TestBuildPassthroughPayloadToolChoice:
def _args(self):
return dict(
openai_messages = [{"role": "user", "content": "Hi"}],
openai_tools = [
{
"type": "function",
"function": {"name": "f", "parameters": {"type": "object"}},
}
],
temperature = 0.6,
top_p = 0.95,
top_k = 20,
max_tokens = 128,
stream = False,
)
def test_default_tool_choice_is_auto(self):
body = _build_passthrough_payload(**self._args())
assert body["tool_choice"] == "auto"
def test_override_tool_choice_required(self):
body = _build_passthrough_payload(**self._args(), tool_choice = "required")
assert body["tool_choice"] == "required"
def test_override_tool_choice_none(self):
body = _build_passthrough_payload(**self._args(), tool_choice = "none")
assert body["tool_choice"] == "none"
def test_override_tool_choice_named_function(self):
tc = {"type": "function", "function": {"name": "f"}}
body = _build_passthrough_payload(**self._args(), tool_choice = tc)
assert body["tool_choice"] == tc
def test_stream_adds_include_usage(self):
args = self._args()
args["stream"] = True
body = _build_passthrough_payload(**args)
assert body.get("stream_options") == {"include_usage": True}
def test_repetition_penalty_renamed(self):
body = _build_passthrough_payload(**self._args(), repetition_penalty = 1.1)
assert body.get("repeat_penalty") == 1.1
assert "repetition_penalty" not in body

367
studio/backend/tests/test_studio_api.py
View file

@ -11,11 +11,16 @@ authentication and the CLI's ``--help`` output:
1. curl -- basic chat completions (non-streaming)
2. curl -- streaming chat completions
3. Python OpenAI SDK -- streaming completions
4. curl -- with tools (web_search + python)
5. Anthropic Messages API -- basic non-streaming
6. Anthropic Messages API -- streaming SSE
7. Anthropic Python SDK -- non-streaming
8. Anthropic Messages API -- streaming with tools
4. curl -- Studio server-side tools (enable_tools=true)
5. curl -- Standard OpenAI function calling (non-streaming)
6. curl -- Standard OpenAI function calling (streaming)
7. curl -- Standard OpenAI function calling (multi-turn tool loop)
8. OpenAI Python SDK -- Standard function calling
9. Anthropic Messages API -- basic non-streaming
10. Anthropic Messages API -- streaming SSE
11. Anthropic Python SDK -- non-streaming
12. Anthropic Messages API -- streaming with tools
13. Anthropic Messages API -- tool_choice={"type":"any"} honored
Training, export, fine-tuning, and chat-UI concerns are out of scope —
see the unit suites elsewhere under ``studio/backend/tests/`` for those.
@ -266,6 +271,250 @@ def test_curl_with_tools(base_url: str, api_key: str):
print(f" PASS curl with tools: {len(chunks)} chunks, {len(full)} chars content")
# ── Standard OpenAI function-calling pass-through tests ─────────────
#
# Regression coverage for unslothai/unsloth#4999: Studio's
# /v1/chat/completions used to silently strip standard OpenAI `tools`
# and `tool_choice` fields, so clients (opencode, Claude Code, Cursor,
# Continue, ...) could never get structured tool_calls back. These
# tests exercise the client-side pass-through path that forwards those
# fields to llama-server verbatim.
#
# They require a tool-capable GGUF (``supports_tools=True`` — e.g.
# Qwen3, Qwen2.5-Coder, Llama-3.1-Instruct). The default test model
# ``unsloth/Qwen3-1.7B-GGUF`` advertises tool support via its chat
# template metadata.
_WEATHER_TOOL = {
"type": "function",
"function": {
"name": "get_weather",
"description": "Look up the current weather for a given city.",
"parameters": {
"type": "object",
"properties": {
"city": {
"type": "string",
"description": "The name of the city, e.g. 'Paris'.",
},
},
"required": ["city"],
},
},
}
def _collect_streamed_tool_calls(chunks: list[dict]) -> list[dict]:
"""Reassemble OpenAI streaming delta.tool_calls into full tool calls.
OpenAI streams partial tool calls across chunks — the first chunk for
a given index carries ``id`` + ``function.name``, and subsequent
chunks append fragments to ``function.arguments``.
"""
by_index: dict[int, dict] = {}
for c in chunks:
choices = c.get("choices") or []
if not choices:
continue
delta = choices[0].get("delta") or {}
tool_calls = delta.get("tool_calls") or []
for tc in tool_calls:
idx = tc.get("index", 0)
slot = by_index.setdefault(
idx,
{
"id": None,
"type": "function",
"function": {"name": None, "arguments": ""},
},
)
if tc.get("id"):
slot["id"] = tc["id"]
fn = tc.get("function") or {}
if fn.get("name"):
slot["function"]["name"] = fn["name"]
if fn.get("arguments"):
slot["function"]["arguments"] += fn["arguments"]
return [by_index[i] for i in sorted(by_index)]
def _final_finish_reason(chunks: list[dict]) -> str | None:
for c in reversed(chunks):
choices = c.get("choices") or []
if not choices:
continue
fr = choices[0].get("finish_reason")
if fr is not None:
return fr
return None
def test_openai_tools_nonstream(base_url: str, api_key: str):
"""Standard OpenAI function calling, non-streaming, tool_choice='required'.
Regression: before the fix, Studio silently stripped `tools` and the
model returned plain text with finish_reason='stop'. After the fix,
llama-server's response is forwarded verbatim so the client sees
finish_reason='tool_calls' with a structured tool_calls array and
non-zero usage.prompt_tokens.
"""
status, text = _http(
"POST",
f"{base_url}/v1/chat/completions",
body = {
"messages": [{"role": "user", "content": "What is the weather in Paris?"}],
"tools": [_WEATHER_TOOL],
"tool_choice": "required",
"stream": False,
},
headers = {"Authorization": f"Bearer {api_key}"},
timeout = 120,
)
assert status == 200, f"Expected 200, got {status}: {text[:500]}"
data = json.loads(text)
assert "choices" in data, f"Missing 'choices': {text[:300]}"
choice = data["choices"][0]
assert (
choice["finish_reason"] == "tool_calls"
), f"Expected finish_reason='tool_calls', got {choice['finish_reason']!r}"
msg = choice["message"]
tool_calls = msg.get("tool_calls") or []
assert len(tool_calls) >= 1, f"No tool_calls in response: {msg}"
first = tool_calls[0]
assert first["type"] == "function"
assert (
first["function"]["name"] == "get_weather"
), f"Wrong tool name: {first['function']['name']!r}"
# arguments must be valid JSON
parsed = json.loads(first["function"]["arguments"])
assert "city" in parsed, f"Tool call missing required 'city' arg: {parsed}"
# Usage must be non-zero (was 0 before the fix)
usage = data.get("usage") or {}
assert (
usage.get("prompt_tokens", 0) > 0
), f"Expected non-zero prompt_tokens; got {usage}"
assert data.get("id"), "Missing response id"
print(
f" PASS openai tools non-stream: "
f"tool={first['function']['name']}, args={parsed}, "
f"prompt_tokens={usage['prompt_tokens']}"
)
def test_openai_tools_stream(base_url: str, api_key: str):
"""Standard OpenAI function calling, streaming, tool_choice='required'."""
status, chunks = _stream_http(
f"{base_url}/v1/chat/completions",
body = {
"messages": [{"role": "user", "content": "What is the weather in Tokyo?"}],
"tools": [_WEATHER_TOOL],
"tool_choice": "required",
"stream": True,
},
headers = {"Authorization": f"Bearer {api_key}"},
timeout = 120,
)
assert status == 200, f"Expected 200, got {status}"
assert len(chunks) > 0, "No SSE chunks received"
assert _final_finish_reason(chunks) == "tool_calls", (
f"Expected final finish_reason='tool_calls', got "
f"{_final_finish_reason(chunks)!r}"
)
assembled = _collect_streamed_tool_calls(chunks)
assert len(assembled) >= 1, "No tool_calls reassembled from stream"
first = assembled[0]
assert first["function"]["name"] == "get_weather"
parsed = json.loads(first["function"]["arguments"])
assert "city" in parsed
print(
f" PASS openai tools stream: {len(chunks)} chunks, "
f"tool={first['function']['name']}, args={parsed}"
)
def test_openai_tools_multiturn(base_url: str, api_key: str):
"""Multi-turn client-side tool loop: validates that role='tool' result
messages and assistant messages carrying tool_calls are accepted.
Regression: before the fix, ChatMessage.role was restricted to
{system,user,assistant} and rejected role='tool' at the Pydantic
validation stage. This test sends a full round trip so the model
receives the simulated tool result and responds with final text.
"""
status, text = _http(
"POST",
f"{base_url}/v1/chat/completions",
body = {
"messages": [
{"role": "user", "content": "What is the weather in Paris?"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_test_1",
"type": "function",
"function": {
"name": "get_weather",
"arguments": '{"city": "Paris"}',
},
}
],
},
{
"role": "tool",
"tool_call_id": "call_test_1",
"content": '{"temperature_c": 14, "condition": "cloudy"}',
},
],
"tools": [_WEATHER_TOOL],
"stream": False,
},
headers = {"Authorization": f"Bearer {api_key}"},
timeout = 120,
)
assert status == 200, f"Expected 200, got {status}: {text[:500]}"
data = json.loads(text)
msg = data["choices"][0]["message"]
# The model should respond with text now that it has the tool result
content = msg.get("content") or ""
assert len(content) > 0 or msg.get(
"tool_calls"
), f"Expected text or follow-up tool call, got empty message: {msg}"
print(f" PASS openai tools multiturn: {content[:80]!r}")
def test_openai_sdk_tool_calling(base_url: str, api_key: str):
"""OpenAI Python SDK round trip — the real client shape opencode et al. use."""
try:
from openai import OpenAI
except ImportError:
print(" SKIP openai SDK not installed")
return
client = OpenAI(base_url = f"{base_url}/v1", api_key = api_key)
resp = client.chat.completions.create(
model = "current",
messages = [{"role": "user", "content": "What's the weather in Berlin?"}],
tools = [_WEATHER_TOOL],
tool_choice = "required",
stream = False,
)
assert resp.choices[0].finish_reason == "tool_calls", (
f"Expected finish_reason='tool_calls', got "
f"{resp.choices[0].finish_reason!r}"
)
tool_calls = resp.choices[0].message.tool_calls
assert tool_calls and len(tool_calls) >= 1, "No tool_calls from SDK"
tc = tool_calls[0]
assert tc.function.name == "get_weather"
parsed = json.loads(tc.function.arguments)
assert "city" in parsed
print(
f" PASS openai SDK tool calling: " f"tool={tc.function.name}, args={parsed}"
)
def test_invalid_key_rejected(base_url: str):
"""Requests with a bad API key should be rejected."""
status, _text = _http(
@ -464,6 +713,73 @@ def test_anthropic_with_tools(base_url: str, api_key: str):
)
def test_anthropic_tool_choice_any(base_url: str, api_key: str):
"""Anthropic Messages API: ``tool_choice: {"type": "any"}`` must be
honored (forwarded as OpenAI ``tool_choice: "required"`` to
llama-server). Regression for the secondary fix bundled with #4999 —
previously this field was accepted on the request model but silently
dropped with a warning log, so the model was free to answer from
memory instead of using the tool.
"""
status, events = _stream_anthropic_http(
f"{base_url}/v1/messages",
body = {
"model": "default",
"max_tokens": 256,
"messages": [
# A question the model could easily answer from memory if
# tool_choice were not enforced.
{
"role": "user",
"content": "What is the weather in London right now?",
}
],
"tools": [
{
"name": "get_weather",
"description": "Look up current weather for a city.",
"input_schema": {
"type": "object",
"properties": {
"city": {"type": "string"},
},
"required": ["city"],
},
}
],
"tool_choice": {"type": "any"},
"stream": True,
},
headers = {"Authorization": f"Bearer {api_key}"},
timeout = 120,
)
assert status == 200, f"Expected 200, got {status}"
assert len(events) > 0, "No SSE events received"
# With tool_choice=any, stop_reason must be tool_use (not end_turn)
stop_reason = None
for etype, data in events:
if etype == "message_delta":
stop_reason = data.get("delta", {}).get("stop_reason") or stop_reason
assert stop_reason == "tool_use", (
f"Expected stop_reason='tool_use' with tool_choice=any, got "
f"{stop_reason!r} — tool_choice may not be forwarded to llama-server."
)
# And at least one tool_use content block must be emitted
tool_use_starts = [
e
for e in events
if e[0] == "content_block_start"
and e[1].get("content_block", {}).get("type") == "tool_use"
]
assert len(tool_use_starts) >= 1, "No tool_use content block emitted"
print(
f" PASS anthropic tool_choice=any honored: "
f"{len(tool_use_starts)} tool_use blocks, stop_reason={stop_reason}"
)
# ── Server lifecycle ─────────────────────────────────────────────────
@ -578,10 +894,10 @@ def main():
print(f" ERROR {fn.__name__}: {type(exc).__name__}: {exc}")
# ── 1. Test --help (no server needed) ────────────────────────────
print("\n[1/11] Testing --help output")
print("\n[1/16] Testing --help output")
run_test(test_help_output)
# ── 2-11. Start server and run API tests ─────────────────────────
# ── 2-16. Start server and run API tests ─────────────────────────
print(
f"\nStarting server: {args.model} (variant={args.gguf_variant}) on port {PORT}..."
)
@ -591,39 +907,54 @@ def main():
base_url = f"http://{HOST}:{PORT}"
print(f"Server ready. API Key: {api_key[:20]}...\n")
print("[2/11] Testing curl basic (non-streaming)")
print("[2/16] Testing curl basic (non-streaming)")
run_test(test_curl_basic, base_url, api_key)
print("[3/11] Testing curl streaming")
print("[3/16] Testing curl streaming")
run_test(test_curl_streaming, base_url, api_key)
print("[4/11] Testing OpenAI Python SDK (streaming)")
print("[4/16] Testing OpenAI Python SDK (streaming)")
run_test(test_openai_sdk, base_url, api_key)
print("[5/11] Testing curl with tools")
print("[5/16] Testing curl with tools (server-side enable_tools)")
run_test(test_curl_with_tools, base_url, api_key)
print("[6/11] Testing invalid API key rejection")
print("[6/16] Testing OpenAI standard tools (non-streaming)")
run_test(test_openai_tools_nonstream, base_url, api_key)
print("[7/16] Testing OpenAI standard tools (streaming)")
run_test(test_openai_tools_stream, base_url, api_key)
print("[8/16] Testing OpenAI standard tools (multi-turn)")
run_test(test_openai_tools_multiturn, base_url, api_key)
print("[9/16] Testing OpenAI SDK tool calling")
run_test(test_openai_sdk_tool_calling, base_url, api_key)
print("[10/16] Testing invalid API key rejection")
run_test(test_invalid_key_rejected, base_url)
print("[7/11] Testing no API key rejection")
print("[11/16] Testing no API key rejection")
run_test(test_no_key_rejected, base_url)
print("[8/11] Testing Anthropic basic (non-streaming)")
print("[12/16] Testing Anthropic basic (non-streaming)")
run_test(test_anthropic_basic, base_url, api_key)
print("[9/11] Testing Anthropic streaming")
print("[13/16] Testing Anthropic streaming")
run_test(test_anthropic_streaming, base_url, api_key)
print("[10/11] Testing Anthropic Python SDK")
print("[14/16] Testing Anthropic Python SDK")
run_test(test_anthropic_sdk, base_url, api_key)
print("[11/11] Testing Anthropic with tools")
print("[15/16] Testing Anthropic with tools")
run_test(test_anthropic_with_tools, base_url, api_key)
print("[16/16] Testing Anthropic tool_choice=any honored")
run_test(test_anthropic_tool_choice_any, base_url, api_key)
except RuntimeError as exc:
print(f"\nFATAL: Server failed to start: {exc}")
failed += 11 # count remaining tests as failed
failed += 16 # count remaining tests as failed
finally:
if proc:
print("\nStopping server...")