From 5043333c941aff3b874abb12227be0b7d7d0261f Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Wed, 15 Apr 2026 17:18:35 +0400 Subject: [PATCH 01/12] fix(studio): forward OpenAI tools/tool_choice to llama-server (#4999) Studio's /v1/chat/completions silently stripped standard OpenAI `tools` and `tool_choice` fields, so clients using standard function calling (opencode, Claude Code, Cursor, Continue, ...) never got structured tool_calls back. Adds a client-side pass-through path mirroring the existing Anthropic /v1/messages flow: when `tools` is present without Studio's `enable_tools` shorthand, the request is forwarded to llama-server verbatim so the client sees native id, finish_reason ("tool_calls"), delta.tool_calls, and accurate usage tokens. Also wires Anthropic tool_choice forwarding: /v1/messages previously accepted tool_choice on the request model but silently dropped it with a warning. Translate the four Anthropic shapes to OpenAI format and forward them so agentic clients can actually enforce tool use. - ChatCompletionRequest: add tools, tool_choice, stop; extra="allow" - ChatMessage: accept role="tool", optional tool_call_id / tool_calls / name; content is now optional (assistant with only tool_calls) - routes/inference.py: _openai_passthrough_stream / _openai_passthrough_non_streaming helpers, routing branch in openai_chat_completions, vision+tools via content-parts injection - _build_passthrough_payload: tool_choice parameter (default "auto") - anthropic_compat: anthropic_tool_choice_to_openai() translator - tests/test_openai_tool_passthrough.py: Pydantic + translator unit tests - tests/test_studio_api.py: 5 new E2E tests (non-stream, stream, multi-turn, OpenAI SDK, Anthropic tool_choice=any regression) --- .../core/inference/anthropic_compat.py | 33 ++ studio/backend/models/inference.py | 47 ++- studio/backend/routes/inference.py | 274 ++++++++++++- .../tests/test_openai_tool_passthrough.py | 326 ++++++++++++++++ studio/backend/tests/test_studio_api.py | 367 +++++++++++++++++- 5 files changed, 1009 insertions(+), 38 deletions(-) create mode 100644 studio/backend/tests/test_openai_tool_passthrough.py diff --git a/studio/backend/core/inference/anthropic_compat.py b/studio/backend/core/inference/anthropic_compat.py index e7b40a60ce..400e6053e0 100644 --- a/studio/backend/core/inference/anthropic_compat.py +++ b/studio/backend/core/inference/anthropic_compat.py @@ -114,6 +114,39 @@ def anthropic_tools_to_openai(tools: list) -> list[dict]: return result +def anthropic_tool_choice_to_openai(tc: Any) -> Any: + """Translate Anthropic `tool_choice` into OpenAI `tool_choice`. + + Anthropic formats (all dict shapes with a ``type`` discriminator): + + - ``{"type": "auto"}`` → ``"auto"`` + - ``{"type": "any"}`` → ``"required"`` + - ``{"type": "none"}`` → ``"none"`` + - ``{"type": "tool", "name": "get_weather"}`` + → ``{"type": "function", "function": {"name": "get_weather"}}`` + + Returns ``None`` for ``None`` or any unrecognized shape (caller may + then fall back to its own default, typically ``"auto"``). + """ + if tc is None: + return None + if not isinstance(tc, dict): + return None + t = tc.get("type") + if t == "auto": + return "auto" + if t == "any": + return "required" + if t == "none": + return "none" + if t == "tool": + name = tc.get("name") + if not name: + return None + return {"type": "function", "function": {"name": name}} + return None + + def build_anthropic_sse_event(event_type: str, data: dict) -> str: """Format a single Anthropic SSE event.""" return f"event: {event_type}\ndata: {json.dumps(data)}\n\n" diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index 4917a14579..44ca03d6b9 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -338,13 +338,29 @@ class ChatMessage(BaseModel): ``content`` may be a plain string (text-only) or a list of content parts for multimodal messages (OpenAI vision format). + Assistant messages that only contain tool calls may set ``content`` + to ``None`` with ``tool_calls`` populated. ``role="tool"`` messages + carry the result of a client-executed tool call and require + ``tool_call_id`` per the OpenAI spec. """ - role: Literal["system", "user", "assistant"] = Field( + role: Literal["system", "user", "assistant", "tool"] = Field( ..., description = "Message role" ) - content: Union[str, list[ContentPart]] = Field( - ..., description = "Message content (string or multimodal parts)" + content: Optional[Union[str, list[ContentPart]]] = Field( + None, description = "Message content (string or multimodal parts)" + ) + tool_call_id: Optional[str] = Field( + None, + description = "OpenAI tool-result messages: id of the tool call this result belongs to.", + ) + tool_calls: Optional[list[dict]] = Field( + None, + description = "OpenAI assistant messages: structured tool calls the model decided to make.", + ) + name: Optional[str] = Field( + None, + description = "OpenAI tool-result messages: name of the tool whose result this is.", ) @@ -355,6 +371,12 @@ class ChatCompletionRequest(BaseModel): Extensions (non-OpenAI fields) are marked with 'x-unsloth'. """ + # Accept unknown fields defensively so future OpenAI fields (seed, + # response_format, logprobs, frequency_penalty, etc.) don't get + # silently dropped by Pydantic before route code runs. Mirrors + # AnthropicMessagesRequest and ResponsesRequest. + model_config = {"extra": "allow"} + model: str = Field( "default", description = "Model identifier (informational; the active model is used)", @@ -367,6 +389,25 @@ class ChatCompletionRequest(BaseModel): None, ge = 1, description = "Maximum tokens to generate (None = until EOS)" ) presence_penalty: float = Field(0.0, ge = 0.0, le = 2.0, description = "Presence penalty") + stop: Optional[Union[str, list[str]]] = Field( + None, + description = "OpenAI stop sequences: a single string or list of strings at which generation halts.", + ) + tools: Optional[list[dict]] = Field( + None, + description = ( + "OpenAI function-tool definitions. When provided without `enable_tools=true`, " + "Studio forwards the tools to the backend so the model returns structured " + "tool_calls for the client to execute (standard OpenAI function calling)." + ), + ) + tool_choice: Optional[Union[str, dict]] = Field( + None, + description = ( + "OpenAI tool choice: 'auto' | 'required' | 'none' | " + "{'type': 'function', 'function': {'name': ...}}" + ), + ) # ── Unsloth extensions (ignored by standard OpenAI clients) ── top_k: int = Field(20, ge = -1, le = 100, description = "[x-unsloth] Top-k sampling") diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 4246f0056b..3b438c8612 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -106,6 +106,7 @@ from models.inference import ( from core.inference.anthropic_compat import ( anthropic_messages_to_openai, anthropic_tools_to_openai, + anthropic_tool_choice_to_openai, AnthropicStreamEmitter, AnthropicPassthroughEmitter, ) @@ -1122,6 +1123,39 @@ async def openai_chat_completions( ) return JSONResponse(content = response.model_dump()) + # ── Standard OpenAI function-calling pass-through (GGUF only) ──── + # When a client (opencode / Claude Code via OpenAI compat / Cursor / + # Continue / ...) sends standard OpenAI `tools` without Studio's + # `enable_tools` shorthand, forward the request to llama-server + # verbatim so structured `tool_calls` flow back to the client. This + # branch runs BEFORE `_extract_content_parts` because that helper is + # unaware of `role="tool"` messages and assistant messages that only + # carry `tool_calls` (content=None) — both of which are valid in + # multi-turn client-side tool loops. + _has_tool_messages = any(m.role == "tool" or m.tool_calls for m in payload.messages) + if ( + using_gguf + and llama_backend.supports_tools + and not payload.enable_tools + and ((payload.tools and len(payload.tools) > 0) or _has_tool_messages) + ): + cancel_event = threading.Event() + completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}" + if payload.stream: + return await _openai_passthrough_stream( + request, + cancel_event, + llama_backend, + payload, + model_name, + completion_id, + ) + return await _openai_passthrough_non_streaming( + llama_backend, + payload, + model_name, + ) + # ── Parse messages (handles multimodal content parts) ───── system_prompt, chat_messages, extracted_image_b64 = _extract_content_parts( payload.messages @@ -2339,22 +2373,12 @@ async def anthropic_messages( ) stop = payload.stop_sequences or None - # tool_choice is declared on AnthropicMessagesRequest for Anthropic SDK - # compatibility (the SDK often sets it by default), but it is not - # currently honored by Unsloth's backend. Warn once per request so the - # silent drop is visible to operators instead of looking like a model - # quality issue to clients. - if payload.tool_choice is not None: - logger.warning( - "anthropic_messages.tool_choice_ignored", - tool_choice = payload.tool_choice, - note = ( - "tool_choice is accepted for Anthropic SDK compatibility but not " - "honored by Unsloth. Use enable_tools / enabled_tools (server-side " - "built-in tools) or restrict the `tools` array (client-side) to " - "control which tools the model sees." - ), - ) + # Translate Anthropic tool_choice to OpenAI format for forwarding to + # llama-server. Falls back to "auto" when unset or unrecognized, which + # matches the prior hardcoded behavior. + openai_tool_choice = anthropic_tool_choice_to_openai(payload.tool_choice) + if openai_tool_choice is None: + openai_tool_choice = "auto" cancel_event = threading.Event() @@ -2392,6 +2416,7 @@ async def anthropic_messages( min_p = min_p, repetition_penalty = repetition_penalty, presence_penalty = presence_penalty, + tool_choice = openai_tool_choice, ) return await _anthropic_passthrough_non_streaming( llama_backend, @@ -2407,6 +2432,7 @@ async def anthropic_messages( min_p = min_p, repetition_penalty = repetition_penalty, presence_penalty = presence_penalty, + tool_choice = openai_tool_choice, ) if server_tools: @@ -2750,11 +2776,12 @@ def _build_passthrough_payload( min_p = None, repetition_penalty = None, presence_penalty = None, + tool_choice = "auto", ): body = { "messages": openai_messages, "tools": openai_tools, - "tool_choice": "auto", + "tool_choice": tool_choice, "temperature": temperature, "top_p": top_p, "top_k": top_k, @@ -2792,6 +2819,7 @@ async def _anthropic_passthrough_stream( min_p = None, repetition_penalty = None, presence_penalty = None, + tool_choice = "auto", ): """Streaming client-side pass-through: forward tools to llama-server and translate its streaming response to Anthropic SSE without executing anything.""" @@ -2808,6 +2836,7 @@ async def _anthropic_passthrough_stream( min_p = min_p, repetition_penalty = repetition_penalty, presence_penalty = presence_penalty, + tool_choice = tool_choice, ) async def _stream(): @@ -2897,6 +2926,7 @@ async def _anthropic_passthrough_non_streaming( min_p = None, repetition_penalty = None, presence_penalty = None, + tool_choice = "auto", ): """Non-streaming client-side pass-through.""" target_url = f"{llama_backend.base_url}/v1/chat/completions" @@ -2912,6 +2942,7 @@ async def _anthropic_passthrough_non_streaming( min_p = min_p, repetition_penalty = repetition_penalty, presence_penalty = presence_penalty, + tool_choice = tool_choice, ) async with httpx.AsyncClient() as client: @@ -2969,3 +3000,212 @@ async def _anthropic_passthrough_non_streaming( ), ) return JSONResponse(content = resp_obj.model_dump()) + + +# ===================================================================== +# Client-side tool pass-through (OpenAI-native /v1/chat/completions) +# ===================================================================== + + +def _openai_messages_for_passthrough(payload) -> list[dict]: + """Build OpenAI-format message dicts for the /v1/chat/completions + passthrough path. + + Messages from ``payload.messages`` are dumped through Pydantic (dropping + unset optional fields) so they are already in standard OpenAI format + — including ``role="tool"`` tool-result messages and assistant messages + that carry structured ``tool_calls``. Content-parts images already in + the message list are left untouched. + + When a client uses Studio's legacy ``image_base64`` top-level field, the + image is re-encoded to PNG (llama-server's stb_image has limited format + support) and spliced into the last user message as an OpenAI + ``image_url`` content part so vision + function-calling requests work + transparently. + """ + messages = [m.model_dump(exclude_none = True) for m in payload.messages] + + if not payload.image_base64: + return messages + + try: + import base64 as _b64 + from io import BytesIO as _BytesIO + from PIL import Image as _Image + + raw = _b64.b64decode(payload.image_base64) + img = _Image.open(_BytesIO(raw)) + if img.mode == "RGBA": + img = img.convert("RGB") + buf = _BytesIO() + img.save(buf, format = "PNG") + png_b64 = _b64.b64encode(buf.getvalue()).decode("ascii") + except Exception as e: + raise HTTPException( + status_code = 400, + detail = f"Failed to process image: {e}", + ) + + data_url = f"data:image/png;base64,{png_b64}" + image_part = {"type": "image_url", "image_url": {"url": data_url}} + + for msg in reversed(messages): + if msg.get("role") != "user": + continue + existing = msg.get("content") + if isinstance(existing, str): + msg["content"] = [{"type": "text", "text": existing}, image_part] + elif isinstance(existing, list): + existing.append(image_part) + else: + msg["content"] = [image_part] + break + else: + messages.append({"role": "user", "content": [image_part]}) + + return messages + + +def _build_openai_passthrough_body(payload) -> dict: + """Assemble the llama-server request body from a ChatCompletionRequest. + + Only explicitly-known OpenAI / llama-server fields are forwarded so that + Studio-specific extensions (``enable_tools``, ``enabled_tools``, + ``session_id``, ...) never leak to the backend. + """ + messages = _openai_messages_for_passthrough(payload) + tool_choice = payload.tool_choice if payload.tool_choice is not None else "auto" + return _build_passthrough_payload( + messages, + payload.tools, + payload.temperature, + payload.top_p, + payload.top_k, + payload.max_tokens, + payload.stream, + stop = payload.stop, + min_p = payload.min_p, + repetition_penalty = payload.repetition_penalty, + presence_penalty = payload.presence_penalty, + tool_choice = tool_choice, + ) + + +async def _openai_passthrough_stream( + request, + cancel_event, + llama_backend, + payload, + model_name, + completion_id, +): + """Streaming client-side pass-through for /v1/chat/completions. + + Forwards the client's OpenAI function-calling request to llama-server and + relays the SSE stream back verbatim. This preserves llama-server's + native response ``id``, ``finish_reason`` (including ``"tool_calls"``), + ``delta.tool_calls``, and the trailing ``usage`` chunk so the client + observes a standard OpenAI response. + """ + target_url = f"{llama_backend.base_url}/v1/chat/completions" + body = _build_openai_passthrough_body(payload) + + async def _stream(): + # Same httpx lifecycle pattern as _anthropic_passthrough_stream: + # avoid `async with` on the client/response to sidestep the Python + # 3.13 + httpcore 1.0.x anyio cancel-scope bug when the async + # generator is garbage-collected from a different task than the + # one that originally entered the context managers. + client = httpx.AsyncClient(timeout = 600) + resp = None + try: + req = client.build_request("POST", target_url, json = body) + resp = await client.send(req, stream = True) + + if resp.status_code != 200: + err_bytes = await resp.aread() + err_text = err_bytes.decode("utf-8", errors = "replace") + logger.error( + "openai passthrough upstream error: status=%s body=%s", + resp.status_code, + err_text[:500], + ) + err = { + "error": { + "message": f"llama-server error: {err_text[:500]}", + "type": "server_error", + }, + } + yield f"data: {json.dumps(err)}\n\n" + return + + async for raw_line in resp.aiter_lines(): + if await request.is_disconnected(): + cancel_event.set() + break + if not raw_line: + continue + if not raw_line.startswith("data: "): + continue + # Relay the llama-server SSE chunk verbatim so the client + # sees its native `id`, `finish_reason`, `delta.tool_calls`, + # and final `usage` unchanged. + yield raw_line + "\n\n" + if raw_line[6:].strip() == "[DONE]": + break + except Exception as e: + logger.error("openai passthrough stream error: %s", e) + err = { + "error": { + "message": _friendly_error(e), + "type": "server_error", + }, + } + yield f"data: {json.dumps(err)}\n\n" + finally: + if resp is not None: + try: + await resp.aclose() + except Exception: + pass + try: + await client.aclose() + except Exception: + pass + + return StreamingResponse( + _stream(), + media_type = "text/event-stream", + headers = { + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + }, + ) + + +async def _openai_passthrough_non_streaming( + llama_backend, + payload, + model_name, +): + """Non-streaming client-side pass-through for /v1/chat/completions. + + Returns llama-server's JSON response verbatim (via JSONResponse) so the + client sees the native response ``id``, ``finish_reason`` (including + ``"tool_calls"``), structured ``tool_calls``, and accurate ``usage`` + token counts. + """ + target_url = f"{llama_backend.base_url}/v1/chat/completions" + body = _build_openai_passthrough_body(payload) + + async with httpx.AsyncClient() as client: + resp = await client.post(target_url, json = body, timeout = 600) + + if resp.status_code != 200: + raise HTTPException( + status_code = resp.status_code, + detail = f"llama-server error: {resp.text[:500]}", + ) + + return JSONResponse(content = resp.json()) diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py new file mode 100644 index 0000000000..e3f4d6069e --- /dev/null +++ b/studio/backend/tests/test_openai_tool_passthrough.py @@ -0,0 +1,326 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +""" +Tests for the OpenAI /v1/chat/completions client-side tool pass-through. + +Covers: +- ChatCompletionRequest accepts standard OpenAI `tools` / `tool_choice` / `stop`. +- ChatMessage accepts role="tool" with `tool_call_id` and role="assistant" + with `content: None` + `tool_calls`. +- ChatCompletionRequest carries unknown fields via `extra="allow"`. +- anthropic_tool_choice_to_openai() covers all four Anthropic shapes. +- _build_passthrough_payload() honors a caller-supplied tool_choice and + defaults to "auto" when unset. + +No running server or GPU required. +""" + +import os +import sys + +_backend = os.path.join(os.path.dirname(__file__), "..") +sys.path.insert(0, _backend) + +import pytest +from pydantic import ValidationError + +from models.inference import ( + ChatCompletionRequest, + ChatMessage, +) +from core.inference.anthropic_compat import ( + anthropic_tool_choice_to_openai, +) +from routes.inference import _build_passthrough_payload + + +# ===================================================================== +# ChatMessage — tool role, tool_calls, optional content +# ===================================================================== + + +class TestChatMessageToolRoles: + def test_tool_role_with_tool_call_id(self): + msg = ChatMessage( + role = "tool", + tool_call_id = "call_abc123", + content = '{"temperature": 72}', + ) + assert msg.role == "tool" + assert msg.tool_call_id == "call_abc123" + assert msg.content == '{"temperature": 72}' + + def test_tool_role_with_name(self): + msg = ChatMessage( + role = "tool", + tool_call_id = "call_abc123", + name = "get_weather", + content = '{"temperature": 72}', + ) + assert msg.name == "get_weather" + + def test_assistant_with_tool_calls_no_content(self): + msg = ChatMessage( + role = "assistant", + content = None, + tool_calls = [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "Paris"}', + }, + } + ], + ) + assert msg.role == "assistant" + assert msg.content is None + assert msg.tool_calls is not None + assert len(msg.tool_calls) == 1 + assert msg.tool_calls[0]["function"]["name"] == "get_weather" + + def test_assistant_with_content_and_tool_calls(self): + msg = ChatMessage( + role = "assistant", + content = "Let me check the weather.", + tool_calls = [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": "{}"}, + } + ], + ) + assert msg.content == "Let me check the weather." + assert msg.tool_calls[0]["id"] == "call_1" + + def test_plain_user_message_still_works(self): + msg = ChatMessage(role = "user", content = "Hello") + assert msg.role == "user" + assert msg.tool_call_id is None + assert msg.tool_calls is None + assert msg.name is None + + def test_invalid_role_rejected(self): + with pytest.raises(ValidationError): + ChatMessage(role = "function", content = "x") + + def test_content_absent_defaults_to_none(self): + msg = ChatMessage(role = "assistant") + assert msg.content is None + + +# ===================================================================== +# ChatCompletionRequest — standard OpenAI tool fields +# ===================================================================== + + +class TestChatCompletionRequestToolFields: + def _make(self, **kwargs): + base = {"messages": [{"role": "user", "content": "Hi"}]} + base.update(kwargs) + return ChatCompletionRequest(**base) + + def test_tools_parses(self): + req = self._make( + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Return the weather in a city", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ], + ) + assert req.tools is not None + assert len(req.tools) == 1 + assert req.tools[0]["function"]["name"] == "get_weather" + + def test_tool_choice_string_auto(self): + assert self._make(tool_choice = "auto").tool_choice == "auto" + + def test_tool_choice_string_required(self): + assert self._make(tool_choice = "required").tool_choice == "required" + + def test_tool_choice_string_none(self): + assert self._make(tool_choice = "none").tool_choice == "none" + + def test_tool_choice_named_function(self): + tc = {"type": "function", "function": {"name": "get_weather"}} + assert self._make(tool_choice = tc).tool_choice == tc + + def test_stop_string(self): + assert self._make(stop = "\nUser:").stop == "\nUser:" + + def test_stop_list(self): + assert self._make(stop = ["\nUser:", "\nAssistant:"]).stop == [ + "\nUser:", + "\nAssistant:", + ] + + def test_tools_default_none(self): + req = self._make() + assert req.tools is None + assert req.tool_choice is None + assert req.stop is None + + def test_extra_fields_accepted(self): + # `frequency_penalty`, `seed`, `response_format` are not yet + # explicitly declared but must survive Pydantic parsing now that + # extra="allow" is set. + req = self._make( + frequency_penalty = 0.5, + seed = 42, + response_format = {"type": "json_object"}, + ) + # Extras land in model_extra + assert req.model_extra is not None + assert req.model_extra.get("frequency_penalty") == 0.5 + assert req.model_extra.get("seed") == 42 + assert req.model_extra.get("response_format") == {"type": "json_object"} + + def test_unsloth_extensions_still_work(self): + req = self._make( + enable_tools = True, + enabled_tools = ["web_search", "python"], + session_id = "abc", + ) + assert req.enable_tools is True + assert req.enabled_tools == ["web_search", "python"] + assert req.session_id == "abc" + + def test_multiturn_tool_loop_messages(self): + req = ChatCompletionRequest( + messages = [ + {"role": "user", "content": "What's the weather in Paris?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "Paris"}', + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": '{"temperature": 14, "unit": "celsius"}', + }, + ], + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object"}, + }, + } + ], + ) + assert len(req.messages) == 3 + assert req.messages[1].role == "assistant" + assert req.messages[1].content is None + assert req.messages[1].tool_calls[0]["id"] == "call_1" + assert req.messages[2].role == "tool" + assert req.messages[2].tool_call_id == "call_1" + + +# ===================================================================== +# anthropic_tool_choice_to_openai — pure translation helper +# ===================================================================== + + +class TestAnthropicToolChoiceToOpenAI: + def test_auto(self): + assert anthropic_tool_choice_to_openai({"type": "auto"}) == "auto" + + def test_any_becomes_required(self): + assert anthropic_tool_choice_to_openai({"type": "any"}) == "required" + + def test_none(self): + assert anthropic_tool_choice_to_openai({"type": "none"}) == "none" + + def test_tool_named(self): + result = anthropic_tool_choice_to_openai( + {"type": "tool", "name": "get_weather"} + ) + assert result == { + "type": "function", + "function": {"name": "get_weather"}, + } + + def test_tool_missing_name_returns_none(self): + assert anthropic_tool_choice_to_openai({"type": "tool"}) is None + + def test_none_input_returns_none(self): + assert anthropic_tool_choice_to_openai(None) is None + + def test_unrecognized_shape_returns_none(self): + assert anthropic_tool_choice_to_openai({"type": "wibble"}) is None + assert anthropic_tool_choice_to_openai("auto") is None + assert anthropic_tool_choice_to_openai(42) is None + + +# ===================================================================== +# _build_passthrough_payload — tool_choice propagation +# ===================================================================== + + +class TestBuildPassthroughPayloadToolChoice: + def _args(self): + return dict( + openai_messages = [{"role": "user", "content": "Hi"}], + openai_tools = [ + { + "type": "function", + "function": {"name": "f", "parameters": {"type": "object"}}, + } + ], + temperature = 0.6, + top_p = 0.95, + top_k = 20, + max_tokens = 128, + stream = False, + ) + + def test_default_tool_choice_is_auto(self): + body = _build_passthrough_payload(**self._args()) + assert body["tool_choice"] == "auto" + + def test_override_tool_choice_required(self): + body = _build_passthrough_payload(**self._args(), tool_choice = "required") + assert body["tool_choice"] == "required" + + def test_override_tool_choice_none(self): + body = _build_passthrough_payload(**self._args(), tool_choice = "none") + assert body["tool_choice"] == "none" + + def test_override_tool_choice_named_function(self): + tc = {"type": "function", "function": {"name": "f"}} + body = _build_passthrough_payload(**self._args(), tool_choice = tc) + assert body["tool_choice"] == tc + + def test_stream_adds_include_usage(self): + args = self._args() + args["stream"] = True + body = _build_passthrough_payload(**args) + assert body.get("stream_options") == {"include_usage": True} + + def test_repetition_penalty_renamed(self): + body = _build_passthrough_payload(**self._args(), repetition_penalty = 1.1) + assert body.get("repeat_penalty") == 1.1 + assert "repetition_penalty" not in body diff --git a/studio/backend/tests/test_studio_api.py b/studio/backend/tests/test_studio_api.py index 9cc17c89fb..521c99e126 100644 --- a/studio/backend/tests/test_studio_api.py +++ b/studio/backend/tests/test_studio_api.py @@ -11,11 +11,16 @@ authentication and the CLI's ``--help`` output: 1. curl -- basic chat completions (non-streaming) 2. curl -- streaming chat completions 3. Python OpenAI SDK -- streaming completions - 4. curl -- with tools (web_search + python) - 5. Anthropic Messages API -- basic non-streaming - 6. Anthropic Messages API -- streaming SSE - 7. Anthropic Python SDK -- non-streaming - 8. Anthropic Messages API -- streaming with tools + 4. curl -- Studio server-side tools (enable_tools=true) + 5. curl -- Standard OpenAI function calling (non-streaming) + 6. curl -- Standard OpenAI function calling (streaming) + 7. curl -- Standard OpenAI function calling (multi-turn tool loop) + 8. OpenAI Python SDK -- Standard function calling + 9. Anthropic Messages API -- basic non-streaming + 10. Anthropic Messages API -- streaming SSE + 11. Anthropic Python SDK -- non-streaming + 12. Anthropic Messages API -- streaming with tools + 13. Anthropic Messages API -- tool_choice={"type":"any"} honored Training, export, fine-tuning, and chat-UI concerns are out of scope — see the unit suites elsewhere under ``studio/backend/tests/`` for those. @@ -266,6 +271,250 @@ def test_curl_with_tools(base_url: str, api_key: str): print(f" PASS curl with tools: {len(chunks)} chunks, {len(full)} chars content") +# ── Standard OpenAI function-calling pass-through tests ───────────── +# +# Regression coverage for unslothai/unsloth#4999: Studio's +# /v1/chat/completions used to silently strip standard OpenAI `tools` +# and `tool_choice` fields, so clients (opencode, Claude Code, Cursor, +# Continue, ...) could never get structured tool_calls back. These +# tests exercise the client-side pass-through path that forwards those +# fields to llama-server verbatim. +# +# They require a tool-capable GGUF (``supports_tools=True`` — e.g. +# Qwen3, Qwen2.5-Coder, Llama-3.1-Instruct). The default test model +# ``unsloth/Qwen3-1.7B-GGUF`` advertises tool support via its chat +# template metadata. + +_WEATHER_TOOL = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Look up the current weather for a given city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The name of the city, e.g. 'Paris'.", + }, + }, + "required": ["city"], + }, + }, +} + + +def _collect_streamed_tool_calls(chunks: list[dict]) -> list[dict]: + """Reassemble OpenAI streaming delta.tool_calls into full tool calls. + + OpenAI streams partial tool calls across chunks — the first chunk for + a given index carries ``id`` + ``function.name``, and subsequent + chunks append fragments to ``function.arguments``. + """ + by_index: dict[int, dict] = {} + for c in chunks: + choices = c.get("choices") or [] + if not choices: + continue + delta = choices[0].get("delta") or {} + tool_calls = delta.get("tool_calls") or [] + for tc in tool_calls: + idx = tc.get("index", 0) + slot = by_index.setdefault( + idx, + { + "id": None, + "type": "function", + "function": {"name": None, "arguments": ""}, + }, + ) + if tc.get("id"): + slot["id"] = tc["id"] + fn = tc.get("function") or {} + if fn.get("name"): + slot["function"]["name"] = fn["name"] + if fn.get("arguments"): + slot["function"]["arguments"] += fn["arguments"] + return [by_index[i] for i in sorted(by_index)] + + +def _final_finish_reason(chunks: list[dict]) -> str | None: + for c in reversed(chunks): + choices = c.get("choices") or [] + if not choices: + continue + fr = choices[0].get("finish_reason") + if fr is not None: + return fr + return None + + +def test_openai_tools_nonstream(base_url: str, api_key: str): + """Standard OpenAI function calling, non-streaming, tool_choice='required'. + + Regression: before the fix, Studio silently stripped `tools` and the + model returned plain text with finish_reason='stop'. After the fix, + llama-server's response is forwarded verbatim so the client sees + finish_reason='tool_calls' with a structured tool_calls array and + non-zero usage.prompt_tokens. + """ + status, text = _http( + "POST", + f"{base_url}/v1/chat/completions", + body = { + "messages": [{"role": "user", "content": "What is the weather in Paris?"}], + "tools": [_WEATHER_TOOL], + "tool_choice": "required", + "stream": False, + }, + headers = {"Authorization": f"Bearer {api_key}"}, + timeout = 120, + ) + assert status == 200, f"Expected 200, got {status}: {text[:500]}" + data = json.loads(text) + assert "choices" in data, f"Missing 'choices': {text[:300]}" + choice = data["choices"][0] + assert ( + choice["finish_reason"] == "tool_calls" + ), f"Expected finish_reason='tool_calls', got {choice['finish_reason']!r}" + msg = choice["message"] + tool_calls = msg.get("tool_calls") or [] + assert len(tool_calls) >= 1, f"No tool_calls in response: {msg}" + first = tool_calls[0] + assert first["type"] == "function" + assert ( + first["function"]["name"] == "get_weather" + ), f"Wrong tool name: {first['function']['name']!r}" + # arguments must be valid JSON + parsed = json.loads(first["function"]["arguments"]) + assert "city" in parsed, f"Tool call missing required 'city' arg: {parsed}" + # Usage must be non-zero (was 0 before the fix) + usage = data.get("usage") or {} + assert ( + usage.get("prompt_tokens", 0) > 0 + ), f"Expected non-zero prompt_tokens; got {usage}" + assert data.get("id"), "Missing response id" + print( + f" PASS openai tools non-stream: " + f"tool={first['function']['name']}, args={parsed}, " + f"prompt_tokens={usage['prompt_tokens']}" + ) + + +def test_openai_tools_stream(base_url: str, api_key: str): + """Standard OpenAI function calling, streaming, tool_choice='required'.""" + status, chunks = _stream_http( + f"{base_url}/v1/chat/completions", + body = { + "messages": [{"role": "user", "content": "What is the weather in Tokyo?"}], + "tools": [_WEATHER_TOOL], + "tool_choice": "required", + "stream": True, + }, + headers = {"Authorization": f"Bearer {api_key}"}, + timeout = 120, + ) + assert status == 200, f"Expected 200, got {status}" + assert len(chunks) > 0, "No SSE chunks received" + assert _final_finish_reason(chunks) == "tool_calls", ( + f"Expected final finish_reason='tool_calls', got " + f"{_final_finish_reason(chunks)!r}" + ) + assembled = _collect_streamed_tool_calls(chunks) + assert len(assembled) >= 1, "No tool_calls reassembled from stream" + first = assembled[0] + assert first["function"]["name"] == "get_weather" + parsed = json.loads(first["function"]["arguments"]) + assert "city" in parsed + print( + f" PASS openai tools stream: {len(chunks)} chunks, " + f"tool={first['function']['name']}, args={parsed}" + ) + + +def test_openai_tools_multiturn(base_url: str, api_key: str): + """Multi-turn client-side tool loop: validates that role='tool' result + messages and assistant messages carrying tool_calls are accepted. + + Regression: before the fix, ChatMessage.role was restricted to + {system,user,assistant} and rejected role='tool' at the Pydantic + validation stage. This test sends a full round trip so the model + receives the simulated tool result and responds with final text. + """ + status, text = _http( + "POST", + f"{base_url}/v1/chat/completions", + body = { + "messages": [ + {"role": "user", "content": "What is the weather in Paris?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_test_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "Paris"}', + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_test_1", + "content": '{"temperature_c": 14, "condition": "cloudy"}', + }, + ], + "tools": [_WEATHER_TOOL], + "stream": False, + }, + headers = {"Authorization": f"Bearer {api_key}"}, + timeout = 120, + ) + assert status == 200, f"Expected 200, got {status}: {text[:500]}" + data = json.loads(text) + msg = data["choices"][0]["message"] + # The model should respond with text now that it has the tool result + content = msg.get("content") or "" + assert len(content) > 0 or msg.get( + "tool_calls" + ), f"Expected text or follow-up tool call, got empty message: {msg}" + print(f" PASS openai tools multiturn: {content[:80]!r}") + + +def test_openai_sdk_tool_calling(base_url: str, api_key: str): + """OpenAI Python SDK round trip — the real client shape opencode et al. use.""" + try: + from openai import OpenAI + except ImportError: + print(" SKIP openai SDK not installed") + return + + client = OpenAI(base_url = f"{base_url}/v1", api_key = api_key) + resp = client.chat.completions.create( + model = "current", + messages = [{"role": "user", "content": "What's the weather in Berlin?"}], + tools = [_WEATHER_TOOL], + tool_choice = "required", + stream = False, + ) + assert resp.choices[0].finish_reason == "tool_calls", ( + f"Expected finish_reason='tool_calls', got " + f"{resp.choices[0].finish_reason!r}" + ) + tool_calls = resp.choices[0].message.tool_calls + assert tool_calls and len(tool_calls) >= 1, "No tool_calls from SDK" + tc = tool_calls[0] + assert tc.function.name == "get_weather" + parsed = json.loads(tc.function.arguments) + assert "city" in parsed + print( + f" PASS openai SDK tool calling: " f"tool={tc.function.name}, args={parsed}" + ) + + def test_invalid_key_rejected(base_url: str): """Requests with a bad API key should be rejected.""" status, _text = _http( @@ -464,6 +713,73 @@ def test_anthropic_with_tools(base_url: str, api_key: str): ) +def test_anthropic_tool_choice_any(base_url: str, api_key: str): + """Anthropic Messages API: ``tool_choice: {"type": "any"}`` must be + honored (forwarded as OpenAI ``tool_choice: "required"`` to + llama-server). Regression for the secondary fix bundled with #4999 — + previously this field was accepted on the request model but silently + dropped with a warning log, so the model was free to answer from + memory instead of using the tool. + """ + status, events = _stream_anthropic_http( + f"{base_url}/v1/messages", + body = { + "model": "default", + "max_tokens": 256, + "messages": [ + # A question the model could easily answer from memory if + # tool_choice were not enforced. + { + "role": "user", + "content": "What is the weather in London right now?", + } + ], + "tools": [ + { + "name": "get_weather", + "description": "Look up current weather for a city.", + "input_schema": { + "type": "object", + "properties": { + "city": {"type": "string"}, + }, + "required": ["city"], + }, + } + ], + "tool_choice": {"type": "any"}, + "stream": True, + }, + headers = {"Authorization": f"Bearer {api_key}"}, + timeout = 120, + ) + assert status == 200, f"Expected 200, got {status}" + assert len(events) > 0, "No SSE events received" + + # With tool_choice=any, stop_reason must be tool_use (not end_turn) + stop_reason = None + for etype, data in events: + if etype == "message_delta": + stop_reason = data.get("delta", {}).get("stop_reason") or stop_reason + assert stop_reason == "tool_use", ( + f"Expected stop_reason='tool_use' with tool_choice=any, got " + f"{stop_reason!r} — tool_choice may not be forwarded to llama-server." + ) + + # And at least one tool_use content block must be emitted + tool_use_starts = [ + e + for e in events + if e[0] == "content_block_start" + and e[1].get("content_block", {}).get("type") == "tool_use" + ] + assert len(tool_use_starts) >= 1, "No tool_use content block emitted" + print( + f" PASS anthropic tool_choice=any honored: " + f"{len(tool_use_starts)} tool_use blocks, stop_reason={stop_reason}" + ) + + # ── Server lifecycle ───────────────────────────────────────────────── @@ -578,10 +894,10 @@ def main(): print(f" ERROR {fn.__name__}: {type(exc).__name__}: {exc}") # ── 1. Test --help (no server needed) ──────────────────────────── - print("\n[1/11] Testing --help output") + print("\n[1/16] Testing --help output") run_test(test_help_output) - # ── 2-11. Start server and run API tests ───────────────────────── + # ── 2-16. Start server and run API tests ───────────────────────── print( f"\nStarting server: {args.model} (variant={args.gguf_variant}) on port {PORT}..." ) @@ -591,39 +907,54 @@ def main(): base_url = f"http://{HOST}:{PORT}" print(f"Server ready. API Key: {api_key[:20]}...\n") - print("[2/11] Testing curl basic (non-streaming)") + print("[2/16] Testing curl basic (non-streaming)") run_test(test_curl_basic, base_url, api_key) - print("[3/11] Testing curl streaming") + print("[3/16] Testing curl streaming") run_test(test_curl_streaming, base_url, api_key) - print("[4/11] Testing OpenAI Python SDK (streaming)") + print("[4/16] Testing OpenAI Python SDK (streaming)") run_test(test_openai_sdk, base_url, api_key) - print("[5/11] Testing curl with tools") + print("[5/16] Testing curl with tools (server-side enable_tools)") run_test(test_curl_with_tools, base_url, api_key) - print("[6/11] Testing invalid API key rejection") + print("[6/16] Testing OpenAI standard tools (non-streaming)") + run_test(test_openai_tools_nonstream, base_url, api_key) + + print("[7/16] Testing OpenAI standard tools (streaming)") + run_test(test_openai_tools_stream, base_url, api_key) + + print("[8/16] Testing OpenAI standard tools (multi-turn)") + run_test(test_openai_tools_multiturn, base_url, api_key) + + print("[9/16] Testing OpenAI SDK tool calling") + run_test(test_openai_sdk_tool_calling, base_url, api_key) + + print("[10/16] Testing invalid API key rejection") run_test(test_invalid_key_rejected, base_url) - print("[7/11] Testing no API key rejection") + print("[11/16] Testing no API key rejection") run_test(test_no_key_rejected, base_url) - print("[8/11] Testing Anthropic basic (non-streaming)") + print("[12/16] Testing Anthropic basic (non-streaming)") run_test(test_anthropic_basic, base_url, api_key) - print("[9/11] Testing Anthropic streaming") + print("[13/16] Testing Anthropic streaming") run_test(test_anthropic_streaming, base_url, api_key) - print("[10/11] Testing Anthropic Python SDK") + print("[14/16] Testing Anthropic Python SDK") run_test(test_anthropic_sdk, base_url, api_key) - print("[11/11] Testing Anthropic with tools") + print("[15/16] Testing Anthropic with tools") run_test(test_anthropic_with_tools, base_url, api_key) + print("[16/16] Testing Anthropic tool_choice=any honored") + run_test(test_anthropic_tool_choice_any, base_url, api_key) + except RuntimeError as exc: print(f"\nFATAL: Server failed to start: {exc}") - failed += 11 # count remaining tests as failed + failed += 16 # count remaining tests as failed finally: if proc: print("\nStopping server...") From e259a3dad0c39f4c968e1e564c945f2be8d5bd30 Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Wed, 15 Apr 2026 17:59:36 +0400 Subject: [PATCH 02/12] fix(studio): surface httpx transport errors from OpenAI passthrough When the managed llama-server subprocess crashes mid-request, the async pass-through helpers in routes/inference.py used to return a bare 500 (non-streaming) or an "An internal error occurred" SSE chunk (streaming) because _friendly_error only recognized the sync path's "Lost connection to llama-server" substring -- httpx transport failures (ConnectError / ReadError / RemoteProtocolError / ReadTimeout) stringify differently and fell through to the generic case. - _friendly_error: map any httpx.RequestError subclass to the same "Lost connection to the model server" message the sync chat path emits. Placed before the substring heuristics so the streaming path automatically picks it up via its existing except Exception catch. - _openai_passthrough_non_streaming: wrap the httpx.AsyncClient.post in a try/except httpx.RequestError and re-raise as HTTPException 502 with the friendly detail. - tests/test_openai_tool_passthrough.py: new TestFriendlyErrorHttpx class pinning the mapping for ConnectError, ReadError, RemoteProtocolError, ReadTimeout, and confirming non-httpx paths (context-size heuristic, generic fallback) are unchanged. --- studio/backend/routes/inference.py | 22 +++++++- .../tests/test_openai_tool_passthrough.py | 54 ++++++++++++++++++- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 3b438c8612..0682739e7e 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -29,6 +29,14 @@ from utils.models import extract_model_size_b as _extract_model_size_b def _friendly_error(exc: Exception) -> str: """Extract a user-friendly message from known llama-server errors.""" + # httpx transport-layer failures reaching the managed llama-server — + # raised by the async pass-through helpers that talk to llama-server + # directly. Treat any RequestError subclass (ConnectError, ReadError, + # RemoteProtocolError, WriteError, PoolTimeout, ...) as "the upstream + # subprocess is unreachable", which for Studio always means the + # llama-server subprocess crashed or is still coming up. + if isinstance(exc, httpx.RequestError): + return "Lost connection to the model server. It may have crashed -- try reloading the model." msg = str(exc) m = _re.search( r"request \((\d+) tokens?\) exceeds the available context size \((\d+) tokens?\)", @@ -3199,8 +3207,18 @@ async def _openai_passthrough_non_streaming( target_url = f"{llama_backend.base_url}/v1/chat/completions" body = _build_openai_passthrough_body(payload) - async with httpx.AsyncClient() as client: - resp = await client.post(target_url, json = body, timeout = 600) + try: + async with httpx.AsyncClient() as client: + resp = await client.post(target_url, json = body, timeout = 600) + except httpx.RequestError as e: + # llama-server subprocess crashed / still starting / unreachable. + # Surface the same friendly message the sync chat path emits so + # operators don't see a bare 500 with no diagnostic. + logger.error("openai passthrough non-streaming: upstream unreachable: %s", e) + raise HTTPException( + status_code = 502, + detail = _friendly_error(e), + ) if resp.status_code != 200: raise HTTPException( diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py index e3f4d6069e..4254dd9e89 100644 --- a/studio/backend/tests/test_openai_tool_passthrough.py +++ b/studio/backend/tests/test_openai_tool_passthrough.py @@ -12,6 +12,8 @@ Covers: - anthropic_tool_choice_to_openai() covers all four Anthropic shapes. - _build_passthrough_payload() honors a caller-supplied tool_choice and defaults to "auto" when unset. +- _friendly_error() maps httpx transport errors to a "Lost connection" + message so passthrough failures are legible instead of bare 500s. No running server or GPU required. """ @@ -22,6 +24,7 @@ import sys _backend = os.path.join(os.path.dirname(__file__), "..") sys.path.insert(0, _backend) +import httpx import pytest from pydantic import ValidationError @@ -32,7 +35,7 @@ from models.inference import ( from core.inference.anthropic_compat import ( anthropic_tool_choice_to_openai, ) -from routes.inference import _build_passthrough_payload +from routes.inference import _build_passthrough_payload, _friendly_error # ===================================================================== @@ -324,3 +327,52 @@ class TestBuildPassthroughPayloadToolChoice: body = _build_passthrough_payload(**self._args(), repetition_penalty = 1.1) assert body.get("repeat_penalty") == 1.1 assert "repetition_penalty" not in body + + +# ===================================================================== +# _friendly_error — httpx transport failures +# ===================================================================== + + +class TestFriendlyErrorHttpx: + """The async pass-through helpers talk to llama-server via httpx. + When the subprocess is down, httpx raises RequestError subclasses + whose string form (``"All connection attempts failed"``, ``"[Errno 111] + Connection refused"``, ...) does NOT contain the substring + ``"Lost connection to llama-server"`` the sync path uses, so the + previous substring-only `_friendly_error` returned a useless generic + message. These tests pin the new isinstance-based mapping. + """ + + def _req(self): + return httpx.Request("POST", "http://127.0.0.1:65535/v1/chat/completions") + + def test_connect_error_mapped(self): + exc = httpx.ConnectError("All connection attempts failed", request = self._req()) + assert "Lost connection" in _friendly_error(exc) + + def test_read_error_mapped(self): + exc = httpx.ReadError("EOF", request = self._req()) + assert "Lost connection" in _friendly_error(exc) + + def test_remote_protocol_error_mapped(self): + exc = httpx.RemoteProtocolError("peer closed", request = self._req()) + assert "Lost connection" in _friendly_error(exc) + + def test_read_timeout_mapped(self): + exc = httpx.ReadTimeout("timed out", request = self._req()) + assert "Lost connection" in _friendly_error(exc) + + def test_non_httpx_unchanged(self): + # Non-httpx exceptions still fall through to the existing substring + # heuristics — a context-size message must still produce the + # "Message too long" path. + ctx_msg = ( + "request (4096 tokens) exceeds the available context size (2048 tokens)" + ) + assert "Message too long" in _friendly_error(ValueError(ctx_msg)) + + def test_generic_exception_returns_generic_message(self): + assert ( + _friendly_error(RuntimeError("unrelated")) == "An internal error occurred" + ) From 06144affdbcb6822598252261c63a1bc51693f38 Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Wed, 15 Apr 2026 18:30:23 +0400 Subject: [PATCH 03/12] fix(studio): close aiter_bytes/aiter_lines explicitly in passthroughs The httpcore asyncgen cleanup fix in 5cedd9a5 is incomplete on Python 3.13 + httpcore 1.0.x: it switched to manual client/response lifecycle but still used anonymous `async for raw_line in resp.aiter_lines():` patterns in all three streaming paths. Python's async for does NOT auto-close the iterator on break/return, so the aiter_lines / aiter_bytes async generator remains alive, reachable only from the surrounding coroutine frame. Once `_stream()` returns the frame is GC'd and the orphaned asyncgen is finalized on a LATER GC pass in a DIFFERENT asyncio task, where httpcore's HTTP11ConnectionByteStream.aclose() enters anyio.CancelScope.__exit__ with a mismatched task and prints "Exception ignored in: " / "async generator ignored GeneratorExit" / "Attempted to exit cancel scope in a different task" to the server log. User observed this on /v1/messages after successful (status 200) requests, with the traceback pointing at HTTP11ConnectionByteStream .__aiter__ / .aclose inside httpcore. Fix: save resp.aiter_lines() / resp.aiter_bytes() as a variable and explicitly `await iter.aclose()` in the finally block BEFORE resp.aclose() / client.aclose(). This closes the asyncgen inside the current task's event loop, so the internal httpcore byte stream is cleaned up before Python's asyncgen GC hook has anything orphaned to finalize. Each aclose is wrapped in try/except Exception so nested anyio cleanup noise can't bubble out. Applied to all three streaming passthrough paths: - _anthropic_passthrough_stream (/v1/messages client-side tool path) - _openai_passthrough_stream (/v1/chat/completions client-side tool path, new in this PR) - openai_completions (/v1/completions bytes proxy from PR #4956) --- studio/backend/routes/inference.py | 101 +++++++++++++++++++---------- 1 file changed, 67 insertions(+), 34 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 0682739e7e..dc94b2fa84 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1975,25 +1975,32 @@ async def openai_completions( if is_stream: async def _stream(): - # Manual httpx client/response lifecycle — see - # _anthropic_passthrough_stream for the full rationale. Briefly: - # `async with` inside an async generator causes - # "Attempted to exit cancel scope in a different task" / - # "async generator ignored GeneratorExit" on Python 3.13 + - # httpcore 1.0.x when the generator is orphaned and finalized - # by GC. Closing via a finally block that catches Exception - # (but not BaseException) suppresses the anyio cleanup noise - # while letting GeneratorExit propagate cleanly. + # Manual httpx client/response lifecycle AND explicit + # aiter_bytes() iterator close — see _anthropic_passthrough_stream + # for the full rationale. Saving `bytes_iter = resp.aiter_bytes()` + # and `await bytes_iter.aclose()` in the finally block is the + # part that matters for avoiding the Python 3.13 + httpcore + # 1.0.x "Exception ignored in: " / anyio + # cancel-scope trace: an anonymous async for leaves the + # iterator unclosed, so Python's asyncgen GC finalizer runs + # cleanup on a later pass in a different asyncio task. client = httpx.AsyncClient(timeout = 600) resp = None + bytes_iter = None try: req = client.build_request("POST", target_url, json = body) resp = await client.send(req, stream = True) - async for chunk in resp.aiter_bytes(): + bytes_iter = resp.aiter_bytes() + async for chunk in bytes_iter: yield chunk except Exception as e: logger.error("openai_completions stream error: %s", e) finally: + if bytes_iter is not None: + try: + await bytes_iter.aclose() + except Exception: + pass if resp is not None: try: await resp.aclose() @@ -2852,33 +2859,42 @@ async def _anthropic_passthrough_stream( for line in emitter.start(message_id, model_name): yield line - # Manage the httpx client and response MANUALLY — no `async with`. + # Manage the httpx client, response, AND the aiter_lines() async + # generator MANUALLY — no `async with`, no anonymous iterator. # - # On Python 3.13 + httpcore 1.0.x, an orphaned async generator (e.g. - # when the client disconnects mid-stream and Starlette drops the - # StreamingResponse iterator without explicitly calling aclose()) - # is finalized by Python's asyncgen GC hook in a DIFFERENT asyncio - # task than the one that originally entered the httpx context - # managers. When `async with` exits run in the wrong task, httpcore's - # internal `HTTP11ConnectionByteStream.aclose()` hits - # `anyio.CancelScope.__exit__` with a mismatched task and raises - # RuntimeError("Attempted to exit cancel scope in a different task"), - # which escapes as "Exception ignored in:" because it happens during - # GC finalization outside any user-owned try/except. + # On Python 3.13 + httpcore 1.0.x, `async for raw_line in + # resp.aiter_lines():` creates an anonymous async generator. When + # the loop exits via `break` (or the generator is orphaned when a + # client disconnects mid-stream), Python's `async for` protocol + # does NOT auto-close the iterator the way a sync `for` loop + # would. The iterator remains reachable only from the current + # coroutine frame; once `_stream()` returns, the frame is GC'd + # and the iterator becomes unreachable. Python's asyncgen + # finalizer hook then runs its aclose() on a LATER GC pass in a + # DIFFERENT asyncio task, where httpcore's + # `HTTP11ConnectionByteStream.aclose()` enters + # `anyio.CancelScope.__exit__` with a mismatched task and prints + # `RuntimeError: Attempted to exit cancel scope in a different + # task` / `RuntimeError: async generator ignored GeneratorExit` + # as "Exception ignored in:" unraisable warnings. # - # The fix: do not use `async with` for the client/response. Close - # them in a finally block wrapped in `try: ... except Exception: pass`. - # This narrowly suppresses RuntimeError / other Exception subclasses - # from the anyio cleanup noise while letting GeneratorExit (a - # BaseException, not Exception) propagate through cleanly so the - # generator terminates as Python expects. + # The fix: save `resp.aiter_lines()` as `lines_iter`, and in the + # finally block explicitly `await lines_iter.aclose()` BEFORE + # `resp.aclose()` / `client.aclose()`. This closes the iterator + # inside our own task's event loop, so the internal httpcore + # byte-stream is cleaned up before Python's asyncgen finalizer + # has anything orphaned to finalize. Each aclose is wrapped in + # `try: ... except Exception: pass` so anyio cleanup noise from + # nested aclose paths can't bubble out. client = httpx.AsyncClient(timeout = 600) resp = None + lines_iter = None try: req = client.build_request("POST", target_url, json = body) resp = await client.send(req, stream = True) - async for raw_line in resp.aiter_lines(): + lines_iter = resp.aiter_lines() + async for raw_line in lines_iter: if await request.is_disconnected(): cancel_event.set() break @@ -2896,6 +2912,11 @@ async def _anthropic_passthrough_stream( except Exception as e: logger.error("anthropic_messages passthrough stream error: %s", e) finally: + if lines_iter is not None: + try: + await lines_iter.aclose() + except Exception: + pass if resp is not None: try: await resp.aclose() @@ -3120,12 +3141,18 @@ async def _openai_passthrough_stream( async def _stream(): # Same httpx lifecycle pattern as _anthropic_passthrough_stream: - # avoid `async with` on the client/response to sidestep the Python - # 3.13 + httpcore 1.0.x anyio cancel-scope bug when the async - # generator is garbage-collected from a different task than the - # one that originally entered the context managers. + # avoid `async with` on the client/response AND explicitly save + # resp.aiter_lines() so we can close it ourselves in the finally + # block. See the long comment there for the full rationale on + # why the anonymous `async for raw_line in resp.aiter_lines():` + # pattern leaks an unclosed async generator that Python's + # asyncgen GC hook then finalizes in a different asyncio task, + # producing "Exception ignored in:" / "async generator ignored + # GeneratorExit" / anyio cancel-scope traces on Python 3.13 + + # httpcore 1.0.x. client = httpx.AsyncClient(timeout = 600) resp = None + lines_iter = None try: req = client.build_request("POST", target_url, json = body) resp = await client.send(req, stream = True) @@ -3147,7 +3174,8 @@ async def _openai_passthrough_stream( yield f"data: {json.dumps(err)}\n\n" return - async for raw_line in resp.aiter_lines(): + lines_iter = resp.aiter_lines() + async for raw_line in lines_iter: if await request.is_disconnected(): cancel_event.set() break @@ -3171,6 +3199,11 @@ async def _openai_passthrough_stream( } yield f"data: {json.dumps(err)}\n\n" finally: + if lines_iter is not None: + try: + await lines_iter.aclose() + except Exception: + pass if resp is not None: try: await resp.aclose() From 9bd7d1ec2022c0c2ee3b97e2c0ab44b02d989354 Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Thu, 16 Apr 2026 20:10:52 +0400 Subject: [PATCH 04/12] fix(studio): default ChatCompletionRequest.stream to false per OpenAI spec OpenAI's /v1/chat/completions spec defaults `stream` to false, so clients that omit the field (naive curl, minimal integrations) expect a single JSON response back. Studio was defaulting to true, silently switching those clients into SSE and breaking any parser that didn't also handle streaming. ResponsesRequest and AnthropicMessagesRequest already default to false correctly; only ChatCompletionRequest was wrong. Studio's own frontend always sets `stream` explicitly on every chat-adapter / chat-api / runtime-provider call site, so the flip has no UI impact. SDK users (OpenAI Python/JS SDK, opencode, Claude Code, Cursor, Continue) also always pass `stream` explicitly, so they're unaffected. The only clients feeling the change are raw-curl users who were relying on the wrong default -- those get the correct OpenAI behavior now. Added a regression test pinning the default so it can't silently flip back. --- studio/backend/models/inference.py | 8 +++++++- studio/backend/tests/test_openai_tool_passthrough.py | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index 44ca03d6b9..f918a73656 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -382,7 +382,13 @@ class ChatCompletionRequest(BaseModel): description = "Model identifier (informational; the active model is used)", ) messages: list[ChatMessage] = Field(..., description = "Conversation messages") - stream: bool = Field(True, description = "Whether to stream the response via SSE") + stream: bool = Field( + False, + description = ( + "Whether to stream the response via SSE. Default matches OpenAI's " + "spec (`false`); opt into streaming by sending `stream: true`." + ), + ) temperature: float = Field(0.6, ge = 0.0, le = 2.0) top_p: float = Field(0.95, ge = 0.0, le = 1.0) max_tokens: Optional[int] = Field( diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py index 4254dd9e89..1c0dec1264 100644 --- a/studio/backend/tests/test_openai_tool_passthrough.py +++ b/studio/backend/tests/test_openai_tool_passthrough.py @@ -200,6 +200,14 @@ class TestChatCompletionRequestToolFields: assert req.enabled_tools == ["web_search", "python"] assert req.session_id == "abc" + def test_stream_defaults_false_matching_openai_spec(self): + # OpenAI's /v1/chat/completions spec defaults `stream` to false. + # Studio previously defaulted to true, which broke naive curl + # clients that omit `stream` (they expect a JSON blob, got SSE). + # Pin the corrected default so it can't silently regress. + req = self._make() + assert req.stream is False + def test_multiturn_tool_loop_messages(self): req = ChatCompletionRequest( messages = [ From 4ef0453cc9e6d43e5d8fdc902aa5ec8b3b4a2970 Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Thu, 16 Apr 2026 21:09:02 +0400 Subject: [PATCH 05/12] fix(studio): reject images in OpenAI tool passthrough for text-only GGUFs The new tool passthrough branch runs before _extract_content_parts, skipping the existing not is_vision guard. Requests combining tools with an image on a text-only tool-capable GGUF were forwarded to llama-server, producing opaque upstream errors instead of the pre-existing clear 400. Restore the guard inline at the dispatch point, checking both legacy image_base64 and inline image_url parts. --- studio/backend/routes/inference.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index dc94b2fa84..96b9f827c0 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1147,6 +1147,23 @@ async def openai_chat_completions( and not payload.enable_tools and ((payload.tools and len(payload.tools) > 0) or _has_tool_messages) ): + # Preserve the vision guard that would otherwise run in the + # non-passthrough path below: text-only tool-capable GGUFs + # should return a clear 400 here rather than forwarding the + # image to llama-server and surfacing an opaque upstream error. + if not llama_backend.is_vision and ( + payload.image_base64 + or any( + isinstance(m.content, list) + and any(isinstance(p, ImageContentPart) for p in m.content) + for m in payload.messages + ) + ): + raise HTTPException( + status_code = 400, + detail = "Image provided but current GGUF model does not support vision.", + ) + cancel_event = threading.Event() completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}" if payload.stream: From a7a7805db6c273e6e4f191c9855dd04e8c73ab5b Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Thu, 16 Apr 2026 21:15:51 +0400 Subject: [PATCH 06/12] fix(studio): require tool_call_id on role=tool chat messages Enforce the OpenAI spec rule that role="tool" messages must carry a tool_call_id. Without it, upstream backends cannot associate a tool result with the assistant's prior tool_calls entry and the request fails in non-obvious ways through the passthrough path. Reject at the request boundary with a 422 instead. --- studio/backend/models/inference.py | 15 ++++++++++++++- .../tests/test_openai_tool_passthrough.py | 17 +++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index f918a73656..10706235e1 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -11,7 +11,7 @@ import time import uuid from typing import Annotated, Any, Dict, Literal, Optional, List, Union -from pydantic import BaseModel, Discriminator, Field, Tag +from pydantic import BaseModel, Discriminator, Field, Tag, model_validator class LoadRequest(BaseModel): @@ -363,6 +363,19 @@ class ChatMessage(BaseModel): description = "OpenAI tool-result messages: name of the tool whose result this is.", ) + @model_validator(mode = "after") + def _require_tool_call_id_for_tool_role(self) -> "ChatMessage": + # OpenAI's spec requires `tool_call_id` on role="tool" messages so + # the upstream backend can associate the result with the assistant's + # prior `tool_calls` entry. Reject malformed tool-result messages at + # the request boundary instead of forwarding them to llama-server + # through the passthrough path, where the failure mode is opaque. + if self.role == "tool" and not self.tool_call_id: + raise ValueError( + 'role="tool" messages require "tool_call_id" per the OpenAI spec.' + ) + return self + class ChatCompletionRequest(BaseModel): """ diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py index 1c0dec1264..cd50ab91f8 100644 --- a/studio/backend/tests/test_openai_tool_passthrough.py +++ b/studio/backend/tests/test_openai_tool_passthrough.py @@ -114,6 +114,23 @@ class TestChatMessageToolRoles: msg = ChatMessage(role = "assistant") assert msg.content is None + def test_tool_role_missing_tool_call_id_rejected(self): + # Per OpenAI spec, role="tool" messages must carry tool_call_id so + # upstream backends can associate the result with its prior call. + # Pin the boundary-level rejection so a malformed tool-result + # message never reaches the passthrough path. + with pytest.raises(ValidationError) as exc_info: + ChatMessage(role = "tool", content = '{"temperature": 72}') + assert "tool_call_id" in str(exc_info.value) + + def test_tool_role_empty_tool_call_id_rejected(self): + with pytest.raises(ValidationError): + ChatMessage( + role = "tool", + tool_call_id = "", + content = '{"temperature": 72}', + ) + # ===================================================================== # ChatCompletionRequest — standard OpenAI tool fields From 1ac464bd282e141da878c5aa5fbe69ea7382630e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 16 Apr 2026 17:29:44 +0000 Subject: [PATCH 07/12] Fix review findings for PR #19 --- studio/backend/models/inference.py | 19 ++++++++++- studio/backend/routes/inference.py | 51 +++++++++++++++++++++++------- 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index f918a73656..66b599962b 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -11,7 +11,7 @@ import time import uuid from typing import Annotated, Any, Dict, Literal, Optional, List, Union -from pydantic import BaseModel, Discriminator, Field, Tag +from pydantic import BaseModel, Discriminator, Field, Tag, model_validator class LoadRequest(BaseModel): @@ -363,6 +363,23 @@ class ChatMessage(BaseModel): description = "OpenAI tool-result messages: name of the tool whose result this is.", ) + @model_validator(mode = "after") + def _validate_role_shape(self): + if self.role == "assistant": + if self.content is None and not self.tool_calls: + raise ValueError( + "assistant messages require content or tool_calls" + ) + elif self.role == "tool": + if self.content is None: + raise ValueError("tool messages require content") + if not self.tool_call_id: + raise ValueError("tool messages require tool_call_id") + else: + if self.content is None: + raise ValueError(f"{self.role} messages require content") + return self + class ChatCompletionRequest(BaseModel): """ diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index dc94b2fa84..9e7e93cbbb 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1141,27 +1141,47 @@ async def openai_chat_completions( # carry `tool_calls` (content=None) — both of which are valid in # multi-turn client-side tool loops. _has_tool_messages = any(m.role == "tool" or m.tool_calls for m in payload.messages) - if ( + _has_inline_image = any( + isinstance(m.content, list) + and any(getattr(p, "type", None) == "image_url" for p in m.content) + for m in payload.messages + ) + _openai_tool_passthrough = ( using_gguf and llama_backend.supports_tools and not payload.enable_tools - and ((payload.tools and len(payload.tools) > 0) or _has_tool_messages) - ): + and (bool(payload.tools) or _has_tool_messages) + ) + if _openai_tool_passthrough: + if (payload.image_base64 or _has_inline_image) and not llama_backend.is_vision: + raise HTTPException( + status_code = 400, + detail = "Image provided but current GGUF model does not support vision.", + ) cancel_event = threading.Event() - completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}" if payload.stream: return await _openai_passthrough_stream( request, cancel_event, llama_backend, payload, - model_name, - completion_id, ) return await _openai_passthrough_non_streaming( llama_backend, payload, - model_name, + ) + + _has_assistant_tool_only = any( + m.role == "assistant" and m.content is None and m.tool_calls + for m in payload.messages + ) + if _has_assistant_tool_only: + raise HTTPException( + status_code = 400, + detail = ( + "Assistant messages with only `tool_calls` (content=None) are " + "only supported on the GGUF llama-server tool passthrough path." + ), ) # ── Parse messages (handles multimodal content parts) ───── @@ -2795,13 +2815,14 @@ def _build_passthrough_payload( ): body = { "messages": openai_messages, - "tools": openai_tools, - "tool_choice": tool_choice, "temperature": temperature, "top_p": top_p, "top_k": top_k, "stream": stream, } + if openai_tools is not None: + body["tools"] = openai_tools + body["tool_choice"] = tool_choice if stream: body["stream_options"] = {"include_usage": True} if max_tokens is not None: @@ -3057,6 +3078,15 @@ def _openai_messages_for_passthrough(payload) -> list[dict]: if not payload.image_base64: return messages + for msg in messages: + if msg.get("role") != "user": + continue + content = msg.get("content") + if isinstance(content, list) and any( + isinstance(p, dict) and p.get("type") == "image_url" for p in content + ): + return messages + try: import base64 as _b64 from io import BytesIO as _BytesIO @@ -3125,8 +3155,6 @@ async def _openai_passthrough_stream( cancel_event, llama_backend, payload, - model_name, - completion_id, ): """Streaming client-side pass-through for /v1/chat/completions. @@ -3228,7 +3256,6 @@ async def _openai_passthrough_stream( async def _openai_passthrough_non_streaming( llama_backend, payload, - model_name, ): """Non-streaming client-side pass-through for /v1/chat/completions. From 541365ee1a472f40737d3286c7277c5d10ba285d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 16 Apr 2026 18:07:52 +0000 Subject: [PATCH 08/12] Fix review findings for PR #5061 --- studio/backend/routes/inference.py | 62 +++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 14 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 9e7e93cbbb..0c32e932cd 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1171,16 +1171,17 @@ async def openai_chat_completions( payload, ) - _has_assistant_tool_only = any( - m.role == "assistant" and m.content is None and m.tool_calls + _has_unsupported_tool_shape = any( + (m.role == "assistant" and m.content is None and m.tool_calls) + or m.role == "tool" for m in payload.messages ) - if _has_assistant_tool_only: + if _has_unsupported_tool_shape: raise HTTPException( status_code = 400, detail = ( - "Assistant messages with only `tool_calls` (content=None) are " - "only supported on the GGUF llama-server tool passthrough path." + "Messages with role='tool' or assistant tool_calls-only turns " + "are only supported on the GGUF llama-server tool passthrough path." ), ) @@ -2413,6 +2414,11 @@ async def anthropic_messages( # matches the prior hardcoded behavior. openai_tool_choice = anthropic_tool_choice_to_openai(payload.tool_choice) if openai_tool_choice is None: + if payload.tool_choice is not None: + logger.warning( + "anthropic_messages.tool_choice_unrecognized", + tool_choice = payload.tool_choice, + ) openai_tool_choice = "auto" cancel_event = threading.Event() @@ -2423,6 +2429,14 @@ async def anthropic_messages( # 2. tools=[...] only → client-side pass-through (standard Anthropic behavior) # 3. neither → plain chat server_tools = payload.enable_tools and llama_backend.supports_tools + if server_tools and payload.tool_choice is not None: + raise HTTPException( + status_code = 400, + detail = ( + "tool_choice is not honored when enable_tools=true. " + "Use client-side tools (omit enable_tools) to control tool_choice." + ), + ) client_tools = ( not server_tools and payload.tools @@ -2799,6 +2813,11 @@ async def _anthropic_plain_non_streaming(run_gen, message_id, model_name): # ===================================================================== +def _llama_auth_headers(llama_backend): + api_key = getattr(llama_backend, "_api_key", None) + return {"Authorization": f"Bearer {api_key}"} if api_key else None + + def _build_passthrough_payload( openai_messages, openai_tools, @@ -2907,7 +2926,7 @@ async def _anthropic_passthrough_stream( # has anything orphaned to finalize. Each aclose is wrapped in # `try: ... except Exception: pass` so anyio cleanup noise from # nested aclose paths can't bubble out. - client = httpx.AsyncClient(timeout = 600) + client = httpx.AsyncClient(timeout = 600, headers = _llama_auth_headers(llama_backend)) resp = None lines_iter = None try: @@ -2995,8 +3014,15 @@ async def _anthropic_passthrough_non_streaming( tool_choice = tool_choice, ) - async with httpx.AsyncClient() as client: - resp = await client.post(target_url, json = body, timeout = 600) + try: + async with httpx.AsyncClient(headers = _llama_auth_headers(llama_backend)) as client: + resp = await client.post(target_url, json = body, timeout = 600) + except httpx.RequestError as e: + logger.error("anthropic passthrough non-streaming: upstream unreachable: %s", e) + raise HTTPException( + status_code = 502, + detail = _friendly_error(e), + ) if resp.status_code != 200: raise HTTPException( @@ -3078,10 +3104,11 @@ def _openai_messages_for_passthrough(payload) -> list[dict]: if not payload.image_base64: return messages - for msg in messages: - if msg.get("role") != "user": - continue - content = msg.get("content") + last_user = next( + (m for m in reversed(messages) if m.get("role") == "user"), None + ) + if last_user is not None: + content = last_user.get("content") if isinstance(content, list) and any( isinstance(p, dict) and p.get("type") == "image_url" for p in content ): @@ -3178,7 +3205,7 @@ async def _openai_passthrough_stream( # producing "Exception ignored in:" / "async generator ignored # GeneratorExit" / anyio cancel-scope traces on Python 3.13 + # httpcore 1.0.x. - client = httpx.AsyncClient(timeout = 600) + client = httpx.AsyncClient(timeout = 600, headers = _llama_auth_headers(llama_backend)) resp = None lines_iter = None try: @@ -3200,6 +3227,7 @@ async def _openai_passthrough_stream( }, } yield f"data: {json.dumps(err)}\n\n" + yield "data: [DONE]\n\n" return lines_iter = resp.aiter_lines() @@ -3226,6 +3254,7 @@ async def _openai_passthrough_stream( }, } yield f"data: {json.dumps(err)}\n\n" + yield "data: [DONE]\n\n" finally: if lines_iter is not None: try: @@ -3268,7 +3297,7 @@ async def _openai_passthrough_non_streaming( body = _build_openai_passthrough_body(payload) try: - async with httpx.AsyncClient() as client: + async with httpx.AsyncClient(headers = _llama_auth_headers(llama_backend)) as client: resp = await client.post(target_url, json = body, timeout = 600) except httpx.RequestError as e: # llama-server subprocess crashed / still starting / unreachable. @@ -3281,6 +3310,11 @@ async def _openai_passthrough_non_streaming( ) if resp.status_code != 200: + logger.error( + "openai passthrough non-streaming upstream error: status=%s body=%s", + resp.status_code, + resp.text[:500], + ) raise HTTPException( status_code = resp.status_code, detail = f"llama-server error: {resp.text[:500]}", From 91ab31e736a54dbbba5b07b2973d7df10af11439 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 16 Apr 2026 18:15:23 +0000 Subject: [PATCH 09/12] Add review tests for PR #5061 --- .../test_pr5061_r2_anthropic_nonstream_502.py | 79 +++++++++++ ...st_pr5061_r2_anthropic_tool_choice_warn.py | 49 +++++++ tests/test_pr5061_r2_auth_headers.py | 40 ++++++ ..._pr5061_r2_enable_tools_tool_choice_400.py | 75 +++++++++++ tests/test_pr5061_r2_image_dedup_last_user.py | 74 +++++++++++ .../test_pr5061_r2_nonstream_success_shape.py | 92 +++++++++++++ .../test_pr5061_r2_nonstream_upstream_log.py | 74 +++++++++++ .../test_pr5061_r2_role_tool_nonpass_guard.py | 123 ++++++++++++++++++ tests/test_pr5061_r2_sse_done_on_error.py | 117 +++++++++++++++++ .../test_pr5061_r2_verbatim_stream_chunks.py | 122 +++++++++++++++++ 10 files changed, 845 insertions(+) create mode 100644 tests/test_pr5061_r2_anthropic_nonstream_502.py create mode 100644 tests/test_pr5061_r2_anthropic_tool_choice_warn.py create mode 100644 tests/test_pr5061_r2_auth_headers.py create mode 100644 tests/test_pr5061_r2_enable_tools_tool_choice_400.py create mode 100644 tests/test_pr5061_r2_image_dedup_last_user.py create mode 100644 tests/test_pr5061_r2_nonstream_success_shape.py create mode 100644 tests/test_pr5061_r2_nonstream_upstream_log.py create mode 100644 tests/test_pr5061_r2_role_tool_nonpass_guard.py create mode 100644 tests/test_pr5061_r2_sse_done_on_error.py create mode 100644 tests/test_pr5061_r2_verbatim_stream_chunks.py diff --git a/tests/test_pr5061_r2_anthropic_nonstream_502.py b/tests/test_pr5061_r2_anthropic_nonstream_502.py new file mode 100644 index 0000000000..f40887e6dc --- /dev/null +++ b/tests/test_pr5061_r2_anthropic_nonstream_502.py @@ -0,0 +1,79 @@ +import asyncio +import os, sys +from unittest.mock import patch + +_backend = os.path.join(os.path.dirname(__file__), "..", "studio", "backend") +sys.path.insert(0, _backend) + +import httpx +import pytest +from fastapi import HTTPException + + +class _Llama: + base_url = "http://127.0.0.1:0" + _api_key = None + + +def test_httpx_connect_error_mapped_to_502(): + from routes import inference as inf_mod + + class _BadClient: + def __init__(self, *a, **kw): + pass + async def __aenter__(self): + return self + async def __aexit__(self, *a): + return False + async def post(self, *a, **kw): + raise httpx.ConnectError("refused", request=httpx.Request("POST", "http://x")) + + with patch.object(inf_mod.httpx, "AsyncClient", _BadClient): + with pytest.raises(HTTPException) as ei: + asyncio.run( + inf_mod._anthropic_passthrough_non_streaming( + _Llama(), + openai_messages=[{"role": "user", "content": "q"}], + openai_tools=[{"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}}], + temperature=0.6, + top_p=0.95, + top_k=20, + max_tokens=None, + message_id="msg_1", + model_name="stub", + tool_choice="auto", + ) + ) + assert ei.value.status_code == 502 + assert "Lost connection" in ei.value.detail + + +def test_httpx_read_error_mapped_to_502(): + from routes import inference as inf_mod + + class _BadClient: + def __init__(self, *a, **kw): + pass + async def __aenter__(self): + return self + async def __aexit__(self, *a): + return False + async def post(self, *a, **kw): + raise httpx.ReadError("eof", request=httpx.Request("POST", "http://x")) + + with patch.object(inf_mod.httpx, "AsyncClient", _BadClient): + with pytest.raises(HTTPException) as ei: + asyncio.run( + inf_mod._anthropic_passthrough_non_streaming( + _Llama(), + openai_messages=[{"role": "user", "content": "q"}], + openai_tools=[{"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}}], + temperature=0.6, + top_p=0.95, + top_k=20, + max_tokens=None, + message_id="msg_1", + model_name="stub", + ) + ) + assert ei.value.status_code == 502 diff --git a/tests/test_pr5061_r2_anthropic_tool_choice_warn.py b/tests/test_pr5061_r2_anthropic_tool_choice_warn.py new file mode 100644 index 0000000000..955063d675 --- /dev/null +++ b/tests/test_pr5061_r2_anthropic_tool_choice_warn.py @@ -0,0 +1,49 @@ +import os, sys +from unittest.mock import patch, MagicMock + +_backend = os.path.join(os.path.dirname(__file__), "..", "studio", "backend") +sys.path.insert(0, _backend) + +from routes import inference as inf_mod + + +def _simulate_coerce(tool_choice): + """Mimic the exact snippet that lives at the top of anthropic_messages().""" + translated = inf_mod.anthropic_tool_choice_to_openai(tool_choice) + if translated is None: + if tool_choice is not None: + inf_mod.logger.warning( + "anthropic_messages.tool_choice_unrecognized", + tool_choice=tool_choice, + ) + translated = "auto" + return translated + + +def test_unrecognized_dict_warns_and_falls_back_to_auto(): + with patch.object(inf_mod.logger, "warning") as mock_warn: + result = _simulate_coerce({"type": "wibble"}) + assert result == "auto" + mock_warn.assert_called_once() + assert mock_warn.call_args.args[0] == "anthropic_messages.tool_choice_unrecognized" + + +def test_non_dict_input_warns_and_falls_back(): + with patch.object(inf_mod.logger, "warning") as mock_warn: + result = _simulate_coerce("auto_string") + assert result == "auto" + mock_warn.assert_called_once() + + +def test_none_input_no_warning(): + with patch.object(inf_mod.logger, "warning") as mock_warn: + result = _simulate_coerce(None) + assert result == "auto" + mock_warn.assert_not_called() + + +def test_recognized_input_no_warning(): + with patch.object(inf_mod.logger, "warning") as mock_warn: + result = _simulate_coerce({"type": "any"}) + assert result == "required" + mock_warn.assert_not_called() diff --git a/tests/test_pr5061_r2_auth_headers.py b/tests/test_pr5061_r2_auth_headers.py new file mode 100644 index 0000000000..8ba2939400 --- /dev/null +++ b/tests/test_pr5061_r2_auth_headers.py @@ -0,0 +1,40 @@ +import os, sys + +_backend = os.path.join(os.path.dirname(__file__), "..", "studio", "backend") +sys.path.insert(0, _backend) + +from routes.inference import _llama_auth_headers + + +class _WithKey: + _api_key = "k_secret_123" + + +class _NoKey: + _api_key = None + + +class _Missing: + pass + + +class _EmptyKey: + _api_key = "" + + +def test_returns_bearer_header_when_api_key_set(): + headers = _llama_auth_headers(_WithKey()) + assert headers == {"Authorization": "Bearer k_secret_123"} + + +def test_returns_none_when_api_key_none(): + assert _llama_auth_headers(_NoKey()) is None + + +def test_returns_none_when_api_key_attribute_missing(): + assert _llama_auth_headers(_Missing()) is None + + +def test_returns_none_when_api_key_empty_string(): + # Empty string is falsy; the helper should treat it as absent. + assert _llama_auth_headers(_EmptyKey()) is None diff --git a/tests/test_pr5061_r2_enable_tools_tool_choice_400.py b/tests/test_pr5061_r2_enable_tools_tool_choice_400.py new file mode 100644 index 0000000000..d1247beb89 --- /dev/null +++ b/tests/test_pr5061_r2_enable_tools_tool_choice_400.py @@ -0,0 +1,75 @@ +import asyncio +import os, sys +from unittest.mock import patch + +_backend = os.path.join(os.path.dirname(__file__), "..", "studio", "backend") +sys.path.insert(0, _backend) + +import pytest +from fastapi import HTTPException + + +class _Llama: + is_loaded = True + supports_tools = True + is_vision = False + _is_audio = False + model_identifier = "stub" + base_url = "http://127.0.0.1:0" + _api_key = None + + +class _Req: + async def is_disconnected(self): + return False + + +def _build_payload(enable_tools, tool_choice): + from models.inference import AnthropicMessagesRequest + return AnthropicMessagesRequest( + model="default", + max_tokens=64, + messages=[{"role": "user", "content": "q"}], + tool_choice=tool_choice, + enable_tools=enable_tools, + ) + + +def test_enable_tools_true_with_tool_choice_raises_400(): + from routes import inference as inf_mod + payload = _build_payload(enable_tools=True, tool_choice={"type": "any"}) + with patch.object(inf_mod, "get_llama_cpp_backend", return_value=_Llama()): + with pytest.raises(HTTPException) as ei: + asyncio.run(inf_mod.anthropic_messages(payload, _Req(), current_subject="u")) + assert ei.value.status_code == 400 + assert "tool_choice" in ei.value.detail + assert "enable_tools" in ei.value.detail + + +def test_guard_logic_unit(): + # Re-implement the guard predicate exactly and pin it. + def would_raise(enable_tools, supports_tools, tool_choice): + server_tools = enable_tools and supports_tools + return bool(server_tools and tool_choice is not None) + + assert would_raise(True, True, {"type": "any"}) is True + assert would_raise(True, True, None) is False + assert would_raise(False, True, {"type": "any"}) is False + assert would_raise(True, False, {"type": "any"}) is False + + +def test_enable_tools_false_with_tool_choice_does_not_raise_the_guard(): + from routes import inference as inf_mod + payload = _build_payload(enable_tools=False, tool_choice={"type": "any"}) + with patch.object(inf_mod, "get_llama_cpp_backend", return_value=_Llama()): + try: + asyncio.run(inf_mod.anthropic_messages(payload, _Req(), current_subject="u")) + except HTTPException as e: + assert not ( + e.status_code == 400 + and "tool_choice is not honored" in (e.detail or "") + ) + except Exception: + # Any other failure downstream is fine; we only pin that the + # enable_tools+tool_choice guard does NOT fire when enable_tools is False. + pass diff --git a/tests/test_pr5061_r2_image_dedup_last_user.py b/tests/test_pr5061_r2_image_dedup_last_user.py new file mode 100644 index 0000000000..60f9e0ff6e --- /dev/null +++ b/tests/test_pr5061_r2_image_dedup_last_user.py @@ -0,0 +1,74 @@ +import os, sys +_backend = os.path.join(os.path.dirname(__file__), "..", "studio", "backend") +sys.path.insert(0, _backend) + +from models.inference import ChatMessage +from routes.inference import _openai_messages_for_passthrough + + +_TINY = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" + + +class _P: + def __init__(self, messages, image_base64=None): + self.messages = messages + self.image_base64 = image_base64 + + +def _img_url_part(b64=_TINY): + return {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}} + + +def test_prior_user_image_does_not_block_new_base64(): + p = _P( + messages=[ + ChatMessage(role="user", content=[{"type": "text", "text": "q1"}, _img_url_part()]), + ChatMessage(role="assistant", content="a1"), + ChatMessage(role="user", content="q2 no inline"), + ], + image_base64=_TINY, + ) + out = _openai_messages_for_passthrough(p) + # last user must receive spliced image + assert isinstance(out[-1]["content"], list) + assert any(x.get("type") == "image_url" for x in out[-1]["content"]) + + +def test_last_user_with_inline_image_skips_splice(): + p = _P( + messages=[ + ChatMessage(role="user", content="earlier"), + ChatMessage(role="user", content=[{"type": "text", "text": "last"}, _img_url_part()]), + ], + image_base64=_TINY, + ) + out = _openai_messages_for_passthrough(p) + last_parts = out[-1]["content"] + assert sum(1 for x in last_parts if x.get("type") == "image_url") == 1 + + +def test_no_user_messages_appends_trailing_user(): + p = _P( + messages=[ChatMessage(role="system", content="sys")], + image_base64=_TINY, + ) + out = _openai_messages_for_passthrough(p) + assert out[-1]["role"] == "user" + assert any(x.get("type") == "image_url" for x in out[-1]["content"]) + + +def test_three_user_turns_only_last_receives_splice(): + p = _P( + messages=[ + ChatMessage(role="user", content="u1"), + ChatMessage(role="assistant", content="a1"), + ChatMessage(role="user", content="u2"), + ChatMessage(role="assistant", content="a2"), + ChatMessage(role="user", content="u3"), + ], + image_base64=_TINY, + ) + out = _openai_messages_for_passthrough(p) + assert isinstance(out[0]["content"], str) + assert isinstance(out[2]["content"], str) + assert isinstance(out[4]["content"], list) diff --git a/tests/test_pr5061_r2_nonstream_success_shape.py b/tests/test_pr5061_r2_nonstream_success_shape.py new file mode 100644 index 0000000000..143c510188 --- /dev/null +++ b/tests/test_pr5061_r2_nonstream_success_shape.py @@ -0,0 +1,92 @@ +import asyncio +import os, sys +from unittest.mock import patch, MagicMock + +_backend = os.path.join(os.path.dirname(__file__), "..", "studio", "backend") +sys.path.insert(0, _backend) + +from models.inference import ChatCompletionRequest + + +class _Llama: + base_url = "http://127.0.0.1:0" + _api_key = None + + +def _payload(): + return ChatCompletionRequest( + messages=[{"role": "user", "content": "q"}], + tools=[{"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}}], + ) + + +def _build_mock_client(status, payload_json): + class _Client: + def __init__(self, *a, **kw): + pass + async def __aenter__(self): + return self + async def __aexit__(self, *a): + return False + async def post(self, *a, **kw): + m = MagicMock() + m.status_code = status + m.text = "" + m.json = lambda: payload_json + return m + return _Client + + +def test_verbatim_json_body_returned(): + from routes import inference as inf_mod + + native = { + "id": "chatcmpl-foo", + "object": "chat.completion", + "model": "qwen-native", + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + } + ], + }, + } + ], + "usage": {"prompt_tokens": 42, "completion_tokens": 7, "total_tokens": 49}, + } + + with patch.object(inf_mod.httpx, "AsyncClient", _build_mock_client(200, native)): + resp = asyncio.run(inf_mod._openai_passthrough_non_streaming(_Llama(), _payload())) + + import json + body = json.loads(resp.body.decode("utf-8")) + assert body == native # verbatim + assert body["choices"][0]["finish_reason"] == "tool_calls" + assert body["usage"]["prompt_tokens"] == 42 + + +def test_preserves_native_id_and_model_fields(): + from routes import inference as inf_mod + + native = { + "id": "chatcmpl-native-xyz", + "model": "llama-native", + "choices": [{"finish_reason": "stop", "message": {"role": "assistant", "content": "ok"}}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + with patch.object(inf_mod.httpx, "AsyncClient", _build_mock_client(200, native)): + resp = asyncio.run(inf_mod._openai_passthrough_non_streaming(_Llama(), _payload())) + + import json + body = json.loads(resp.body.decode("utf-8")) + assert body["id"] == "chatcmpl-native-xyz" + assert body["model"] == "llama-native" diff --git a/tests/test_pr5061_r2_nonstream_upstream_log.py b/tests/test_pr5061_r2_nonstream_upstream_log.py new file mode 100644 index 0000000000..ad4f077d06 --- /dev/null +++ b/tests/test_pr5061_r2_nonstream_upstream_log.py @@ -0,0 +1,74 @@ +import asyncio +import os, sys +from unittest.mock import patch, MagicMock + +_backend = os.path.join(os.path.dirname(__file__), "..", "studio", "backend") +sys.path.insert(0, _backend) + +import pytest +from fastapi import HTTPException + +from models.inference import ChatCompletionRequest + + +class _Llama: + base_url = "http://127.0.0.1:0" + _api_key = None + + +def _payload(): + return ChatCompletionRequest( + messages=[{"role": "user", "content": "q"}], + tools=[{"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}}], + ) + + +def _mk_client(status, text): + class _Client: + def __init__(self, *a, **kw): + pass + async def __aenter__(self): + return self + async def __aexit__(self, *a): + return False + async def post(self, *a, **kw): + m = MagicMock() + m.status_code = status + m.text = text + m.json = lambda: {} + return m + return _Client + + +def test_nonstream_non_200_calls_logger_error_with_status_and_body(): + from routes import inference as inf_mod + + with patch.object(inf_mod.httpx, "AsyncClient", _mk_client(503, "backend overloaded detail")), \ + patch.object(inf_mod.logger, "error") as mock_error: + with pytest.raises(HTTPException) as ei: + asyncio.run(inf_mod._openai_passthrough_non_streaming(_Llama(), _payload())) + + assert ei.value.status_code == 503 + mock_error.assert_called() + # Check any error call mentions upstream status and body + found = False + for call in mock_error.call_args_list: + msg = call.args[0] if call.args else "" + combined = f"{msg} {call.args} {call.kwargs}" + if "upstream error" in combined and "503" in combined and "backend overloaded" in combined: + found = True + break + assert found, f"expected upstream error log with status and body, got {mock_error.call_args_list}" + + +def test_nonstream_200_does_not_call_logger_error(): + from routes import inference as inf_mod + + with patch.object(inf_mod.httpx, "AsyncClient", _mk_client(200, "{}")), \ + patch.object(inf_mod.logger, "error") as mock_error: + asyncio.run(inf_mod._openai_passthrough_non_streaming(_Llama(), _payload())) + + # no upstream-error log on 200 + for call in mock_error.call_args_list: + msg = call.args[0] if call.args else "" + assert "upstream error" not in msg diff --git a/tests/test_pr5061_r2_role_tool_nonpass_guard.py b/tests/test_pr5061_r2_role_tool_nonpass_guard.py new file mode 100644 index 0000000000..943896e13c --- /dev/null +++ b/tests/test_pr5061_r2_role_tool_nonpass_guard.py @@ -0,0 +1,123 @@ +import asyncio +import os, sys +from unittest.mock import patch + +_backend = os.path.join(os.path.dirname(__file__), "..", "studio", "backend") +sys.path.insert(0, _backend) + +import pytest +from fastapi import HTTPException + +from models.inference import ChatCompletionRequest + + +class _Llama: + def __init__(self, is_loaded=True, supports_tools=True, is_vision=False): + self.is_loaded = is_loaded + self.supports_tools = supports_tools + self.is_vision = is_vision + self._is_audio = False + self.model_identifier = "stub" + self.base_url = "http://127.0.0.1:0" + self._api_key = None + + +class _Inf: + active_model_name = "hf" + models = {"hf": {}} + + +class _Req: + async def is_disconnected(self): + return False + + +async def _marker_stream(*a, **kw): + return ("passthrough_stream", None) + + +async def _marker_nonstream(*a, **kw): + return ("passthrough_nonstream", None) + + +async def _call(payload, llama): + from routes import inference as inf_mod + with patch.object(inf_mod, "get_llama_cpp_backend", return_value=llama), \ + patch.object(inf_mod, "get_inference_backend", return_value=_Inf()), \ + patch.object(inf_mod, "_openai_passthrough_stream", new=_marker_stream), \ + patch.object(inf_mod, "_openai_passthrough_non_streaming", new=_marker_nonstream): + return await inf_mod.openai_chat_completions(payload, _Req(), current_subject="u") + + +def test_role_tool_on_non_gguf_rejected(): + payload = ChatCompletionRequest( + messages=[ + {"role": "user", "content": "q"}, + {"role": "tool", "tool_call_id": "c1", "content": "r"}, + ], + ) + llama = _Llama(is_loaded=False) + with pytest.raises(HTTPException) as ei: + asyncio.run(_call(payload, llama)) + assert ei.value.status_code == 400 + assert "role='tool'" in ei.value.detail or "tool" in ei.value.detail.lower() + + +def test_role_tool_on_gguf_without_tool_support_rejected(): + payload = ChatCompletionRequest( + messages=[ + {"role": "user", "content": "q"}, + {"role": "tool", "tool_call_id": "c1", "content": "r"}, + ], + ) + llama = _Llama(supports_tools=False) + with pytest.raises(HTTPException) as ei: + asyncio.run(_call(payload, llama)) + assert ei.value.status_code == 400 + + +def test_role_tool_with_enable_tools_true_rejected(): + payload = ChatCompletionRequest( + messages=[ + {"role": "user", "content": "q"}, + {"role": "tool", "tool_call_id": "c1", "content": "r"}, + ], + enable_tools=True, + ) + llama = _Llama() + with pytest.raises(HTTPException) as ei: + asyncio.run(_call(payload, llama)) + assert ei.value.status_code == 400 + + +def test_assistant_tool_only_still_rejected_on_non_passthrough(): + payload = ChatCompletionRequest( + messages=[ + {"role": "user", "content": "q"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "c1", "type": "function", "function": {"name": "f", "arguments": "{}"}} + ], + }, + ], + ) + llama = _Llama(supports_tools=False) + with pytest.raises(HTTPException) as ei: + asyncio.run(_call(payload, llama)) + assert ei.value.status_code == 400 + + +def test_no_tool_messages_passes_through(): + payload = ChatCompletionRequest( + messages=[{"role": "user", "content": "plain"}], + ) + llama = _Llama(supports_tools=False) + # Should not raise the tool-shape guard; may raise later for non-GGUF/inf path, + # but specifically this guard must not fire. + try: + asyncio.run(_call(payload, llama)) + except HTTPException as e: + assert "role='tool'" not in (e.detail or "") + assert "tool_calls-only" not in (e.detail or "") diff --git a/tests/test_pr5061_r2_sse_done_on_error.py b/tests/test_pr5061_r2_sse_done_on_error.py new file mode 100644 index 0000000000..cc15d0a6b2 --- /dev/null +++ b/tests/test_pr5061_r2_sse_done_on_error.py @@ -0,0 +1,117 @@ +import asyncio +import os, sys, threading +from unittest.mock import patch, MagicMock, AsyncMock + +_backend = os.path.join(os.path.dirname(__file__), "..", "studio", "backend") +sys.path.insert(0, _backend) + +import httpx + +from models.inference import ChatCompletionRequest + + +class _Llama: + base_url = "http://127.0.0.1:0" + _api_key = None + + +class _Req: + async def is_disconnected(self): + return False + + +def _collect(stream_response): + async def _run(): + out = [] + async for chunk in stream_response.body_iterator: + out.append(chunk if isinstance(chunk, str) else chunk.decode("utf-8")) + return out + return asyncio.run(_run()) + + +def _make_payload(): + return ChatCompletionRequest( + messages=[{"role": "user", "content": "q"}], + tools=[{"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}}], + stream=True, + ) + + +def test_done_emitted_after_non_200_error(): + from routes import inference as inf_mod + + class _Resp: + status_code = 500 + async def aread(self): + return b"server oops" + async def aclose(self): + pass + + async def _send(req, stream): + return _Resp() + + with patch.object(inf_mod.httpx, "AsyncClient") as mock_cls: + inst = MagicMock() + inst.build_request = MagicMock(return_value=MagicMock()) + inst.send = _send + inst.aclose = AsyncMock() + mock_cls.return_value = inst + + resp = asyncio.run( + inf_mod._openai_passthrough_stream( + _Req(), threading.Event(), _Llama(), _make_payload() + ) + ) + chunks = _collect(resp) + + assert any('"error"' in c for c in chunks) + assert any(c.strip() == "data: [DONE]" for c in chunks) + + +def test_done_emitted_after_exception(): + from routes import inference as inf_mod + + async def _send(req, stream): + raise httpx.ConnectError("boom", request=httpx.Request("POST", "http://x")) + + with patch.object(inf_mod.httpx, "AsyncClient") as mock_cls: + inst = MagicMock() + inst.build_request = MagicMock(return_value=MagicMock()) + inst.send = _send + inst.aclose = AsyncMock() + mock_cls.return_value = inst + + resp = asyncio.run( + inf_mod._openai_passthrough_stream( + _Req(), threading.Event(), _Llama(), _make_payload() + ) + ) + chunks = _collect(resp) + + assert any('"error"' in c for c in chunks) + assert any(c.strip() == "data: [DONE]" for c in chunks) + + +def test_done_comes_after_error_chunk(): + from routes import inference as inf_mod + + async def _send(req, stream): + raise httpx.ReadError("reset", request=httpx.Request("POST", "http://x")) + + with patch.object(inf_mod.httpx, "AsyncClient") as mock_cls: + inst = MagicMock() + inst.build_request = MagicMock(return_value=MagicMock()) + inst.send = _send + inst.aclose = AsyncMock() + mock_cls.return_value = inst + + resp = asyncio.run( + inf_mod._openai_passthrough_stream( + _Req(), threading.Event(), _Llama(), _make_payload() + ) + ) + chunks = _collect(resp) + + err_idx = next(i for i, c in enumerate(chunks) if '"error"' in c) + done_idx = next(i for i, c in enumerate(chunks) if c.strip() == "data: [DONE]") + assert done_idx > err_idx diff --git a/tests/test_pr5061_r2_verbatim_stream_chunks.py b/tests/test_pr5061_r2_verbatim_stream_chunks.py new file mode 100644 index 0000000000..0d694ba557 --- /dev/null +++ b/tests/test_pr5061_r2_verbatim_stream_chunks.py @@ -0,0 +1,122 @@ +import asyncio +import os, sys, threading +from unittest.mock import patch, MagicMock + +_backend = os.path.join(os.path.dirname(__file__), "..", "studio", "backend") +sys.path.insert(0, _backend) + +from models.inference import ChatCompletionRequest + + +class _Llama: + base_url = "http://127.0.0.1:0" + _api_key = None + + +class _Req: + async def is_disconnected(self): + return False + + +def _payload(): + return ChatCompletionRequest( + messages=[{"role": "user", "content": "q"}], + tools=[{"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}}], + stream=True, + ) + + +class _FakeResp: + def __init__(self, lines): + self.status_code = 200 + self._lines = list(lines) + + def aiter_lines(self): + parent = self + + class _It: + def __init__(self): + self._idx = 0 + + def __aiter__(self): + return self + + async def __anext__(self): + if self._idx >= len(parent._lines): + raise StopAsyncIteration + v = parent._lines[self._idx] + self._idx += 1 + return v + + async def aclose(self): + pass + + return _It() + + async def aread(self): + return b"" + + async def aclose(self): + pass + + +def _run_stream(fake_lines): + from routes import inference as inf_mod + + async def _send(req, stream): + return _FakeResp(fake_lines) + + async def _aclose_noop(): + return None + + with patch.object(inf_mod.httpx, "AsyncClient") as mock_cls: + inst = MagicMock() + inst.build_request = MagicMock(return_value=MagicMock()) + inst.send = _send + inst.aclose = _aclose_noop + mock_cls.return_value = inst + + resp = asyncio.run( + inf_mod._openai_passthrough_stream( + _Req(), threading.Event(), _Llama(), _payload() + ) + ) + + async def _collect(): + return [ + c if isinstance(c, str) else c.decode("utf-8") + async for c in resp.body_iterator + ] + + return asyncio.run(_collect()) + + +def test_passthrough_relays_data_lines_verbatim(): + chunks = _run_stream([ + 'data: {"id":"abc","choices":[{"delta":{"content":"hi"}}]}', + 'data: [DONE]', + ]) + assert any('"id":"abc"' in c for c in chunks) + assert any("data: [DONE]" in c for c in chunks) + + +def test_passthrough_ignores_blank_and_non_data_lines(): + chunks = _run_stream([ + "", + ": heartbeat", + 'data: {"x":1}', + 'data: [DONE]', + ]) + # Only data: lines propagate. + for c in chunks: + assert c.startswith("data: ") or c == "" + assert any('"x":1' in c for c in chunks) + + +def test_passthrough_breaks_on_done(): + chunks = _run_stream([ + 'data: {"a":1}', + 'data: [DONE]', + 'data: {"should_not_appear":true}', + ]) + assert not any("should_not_appear" in c for c in chunks) From 750f5aa1327033643a96a39b2445c59fe544a26c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 Apr 2026 18:20:12 +0000 Subject: [PATCH 10/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/models/inference.py | 4 +- studio/backend/routes/inference.py | 20 ++++--- .../tests/test_openai_tool_passthrough.py | 6 +- .../test_pr5061_r2_anthropic_nonstream_502.py | 56 ++++++++++++------- ...st_pr5061_r2_anthropic_tool_choice_warn.py | 2 +- ..._pr5061_r2_enable_tools_tool_choice_400.py | 29 ++++++---- tests/test_pr5061_r2_image_dedup_last_user.py | 45 ++++++++------- .../test_pr5061_r2_nonstream_success_shape.py | 32 +++++++++-- .../test_pr5061_r2_nonstream_upstream_log.py | 33 ++++++++--- .../test_pr5061_r2_role_tool_nonpass_guard.py | 45 +++++++++------ tests/test_pr5061_r2_sse_done_on_error.py | 24 +++++--- .../test_pr5061_r2_verbatim_stream_chunks.py | 49 +++++++++------- 12 files changed, 226 insertions(+), 119 deletions(-) diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index df57791eb9..86d15def7e 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -367,9 +367,7 @@ class ChatMessage(BaseModel): def _validate_role_shape(self): if self.role == "assistant": if self.content is None and not self.tool_calls: - raise ValueError( - "assistant messages require content or tool_calls" - ) + raise ValueError("assistant messages require content or tool_calls") elif self.role == "tool": if self.content is None: raise ValueError("tool messages require content") diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 0c32e932cd..350e9786e8 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -2926,7 +2926,9 @@ async def _anthropic_passthrough_stream( # has anything orphaned to finalize. Each aclose is wrapped in # `try: ... except Exception: pass` so anyio cleanup noise from # nested aclose paths can't bubble out. - client = httpx.AsyncClient(timeout = 600, headers = _llama_auth_headers(llama_backend)) + client = httpx.AsyncClient( + timeout = 600, headers = _llama_auth_headers(llama_backend) + ) resp = None lines_iter = None try: @@ -3015,7 +3017,9 @@ async def _anthropic_passthrough_non_streaming( ) try: - async with httpx.AsyncClient(headers = _llama_auth_headers(llama_backend)) as client: + async with httpx.AsyncClient( + headers = _llama_auth_headers(llama_backend) + ) as client: resp = await client.post(target_url, json = body, timeout = 600) except httpx.RequestError as e: logger.error("anthropic passthrough non-streaming: upstream unreachable: %s", e) @@ -3104,9 +3108,7 @@ def _openai_messages_for_passthrough(payload) -> list[dict]: if not payload.image_base64: return messages - last_user = next( - (m for m in reversed(messages) if m.get("role") == "user"), None - ) + last_user = next((m for m in reversed(messages) if m.get("role") == "user"), None) if last_user is not None: content = last_user.get("content") if isinstance(content, list) and any( @@ -3205,7 +3207,9 @@ async def _openai_passthrough_stream( # producing "Exception ignored in:" / "async generator ignored # GeneratorExit" / anyio cancel-scope traces on Python 3.13 + # httpcore 1.0.x. - client = httpx.AsyncClient(timeout = 600, headers = _llama_auth_headers(llama_backend)) + client = httpx.AsyncClient( + timeout = 600, headers = _llama_auth_headers(llama_backend) + ) resp = None lines_iter = None try: @@ -3297,7 +3301,9 @@ async def _openai_passthrough_non_streaming( body = _build_openai_passthrough_body(payload) try: - async with httpx.AsyncClient(headers = _llama_auth_headers(llama_backend)) as client: + async with httpx.AsyncClient( + headers = _llama_auth_headers(llama_backend) + ) as client: resp = await client.post(target_url, json = body, timeout = 600) except httpx.RequestError as e: # llama-server subprocess crashed / still starting / unreachable. diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py index 056b1a62a2..d84da19634 100644 --- a/studio/backend/tests/test_openai_tool_passthrough.py +++ b/studio/backend/tests/test_openai_tool_passthrough.py @@ -114,7 +114,11 @@ class TestChatMessageToolRoles: msg = ChatMessage( role = "assistant", tool_calls = [ - {"id": "c1", "type": "function", "function": {"name": "f", "arguments": "{}"}} + { + "id": "c1", + "type": "function", + "function": {"name": "f", "arguments": "{}"}, + } ], ) assert msg.content is None diff --git a/tests/test_pr5061_r2_anthropic_nonstream_502.py b/tests/test_pr5061_r2_anthropic_nonstream_502.py index f40887e6dc..92b6856a76 100644 --- a/tests/test_pr5061_r2_anthropic_nonstream_502.py +++ b/tests/test_pr5061_r2_anthropic_nonstream_502.py @@ -21,27 +21,37 @@ def test_httpx_connect_error_mapped_to_502(): class _BadClient: def __init__(self, *a, **kw): pass + async def __aenter__(self): return self + async def __aexit__(self, *a): return False + async def post(self, *a, **kw): - raise httpx.ConnectError("refused", request=httpx.Request("POST", "http://x")) + raise httpx.ConnectError( + "refused", request = httpx.Request("POST", "http://x") + ) with patch.object(inf_mod.httpx, "AsyncClient", _BadClient): with pytest.raises(HTTPException) as ei: asyncio.run( inf_mod._anthropic_passthrough_non_streaming( _Llama(), - openai_messages=[{"role": "user", "content": "q"}], - openai_tools=[{"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}}], - temperature=0.6, - top_p=0.95, - top_k=20, - max_tokens=None, - message_id="msg_1", - model_name="stub", - tool_choice="auto", + openai_messages = [{"role": "user", "content": "q"}], + openai_tools = [ + { + "type": "function", + "function": {"name": "f", "parameters": {"type": "object"}}, + } + ], + temperature = 0.6, + top_p = 0.95, + top_k = 20, + max_tokens = None, + message_id = "msg_1", + model_name = "stub", + tool_choice = "auto", ) ) assert ei.value.status_code == 502 @@ -54,26 +64,34 @@ def test_httpx_read_error_mapped_to_502(): class _BadClient: def __init__(self, *a, **kw): pass + async def __aenter__(self): return self + async def __aexit__(self, *a): return False + async def post(self, *a, **kw): - raise httpx.ReadError("eof", request=httpx.Request("POST", "http://x")) + raise httpx.ReadError("eof", request = httpx.Request("POST", "http://x")) with patch.object(inf_mod.httpx, "AsyncClient", _BadClient): with pytest.raises(HTTPException) as ei: asyncio.run( inf_mod._anthropic_passthrough_non_streaming( _Llama(), - openai_messages=[{"role": "user", "content": "q"}], - openai_tools=[{"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}}], - temperature=0.6, - top_p=0.95, - top_k=20, - max_tokens=None, - message_id="msg_1", - model_name="stub", + openai_messages = [{"role": "user", "content": "q"}], + openai_tools = [ + { + "type": "function", + "function": {"name": "f", "parameters": {"type": "object"}}, + } + ], + temperature = 0.6, + top_p = 0.95, + top_k = 20, + max_tokens = None, + message_id = "msg_1", + model_name = "stub", ) ) assert ei.value.status_code == 502 diff --git a/tests/test_pr5061_r2_anthropic_tool_choice_warn.py b/tests/test_pr5061_r2_anthropic_tool_choice_warn.py index 955063d675..5e79ab38eb 100644 --- a/tests/test_pr5061_r2_anthropic_tool_choice_warn.py +++ b/tests/test_pr5061_r2_anthropic_tool_choice_warn.py @@ -14,7 +14,7 @@ def _simulate_coerce(tool_choice): if tool_choice is not None: inf_mod.logger.warning( "anthropic_messages.tool_choice_unrecognized", - tool_choice=tool_choice, + tool_choice = tool_choice, ) translated = "auto" return translated diff --git a/tests/test_pr5061_r2_enable_tools_tool_choice_400.py b/tests/test_pr5061_r2_enable_tools_tool_choice_400.py index d1247beb89..9e1b1013c2 100644 --- a/tests/test_pr5061_r2_enable_tools_tool_choice_400.py +++ b/tests/test_pr5061_r2_enable_tools_tool_choice_400.py @@ -26,21 +26,25 @@ class _Req: def _build_payload(enable_tools, tool_choice): from models.inference import AnthropicMessagesRequest + return AnthropicMessagesRequest( - model="default", - max_tokens=64, - messages=[{"role": "user", "content": "q"}], - tool_choice=tool_choice, - enable_tools=enable_tools, + model = "default", + max_tokens = 64, + messages = [{"role": "user", "content": "q"}], + tool_choice = tool_choice, + enable_tools = enable_tools, ) def test_enable_tools_true_with_tool_choice_raises_400(): from routes import inference as inf_mod - payload = _build_payload(enable_tools=True, tool_choice={"type": "any"}) - with patch.object(inf_mod, "get_llama_cpp_backend", return_value=_Llama()): + + payload = _build_payload(enable_tools = True, tool_choice = {"type": "any"}) + with patch.object(inf_mod, "get_llama_cpp_backend", return_value = _Llama()): with pytest.raises(HTTPException) as ei: - asyncio.run(inf_mod.anthropic_messages(payload, _Req(), current_subject="u")) + asyncio.run( + inf_mod.anthropic_messages(payload, _Req(), current_subject = "u") + ) assert ei.value.status_code == 400 assert "tool_choice" in ei.value.detail assert "enable_tools" in ei.value.detail @@ -60,10 +64,13 @@ def test_guard_logic_unit(): def test_enable_tools_false_with_tool_choice_does_not_raise_the_guard(): from routes import inference as inf_mod - payload = _build_payload(enable_tools=False, tool_choice={"type": "any"}) - with patch.object(inf_mod, "get_llama_cpp_backend", return_value=_Llama()): + + payload = _build_payload(enable_tools = False, tool_choice = {"type": "any"}) + with patch.object(inf_mod, "get_llama_cpp_backend", return_value = _Llama()): try: - asyncio.run(inf_mod.anthropic_messages(payload, _Req(), current_subject="u")) + asyncio.run( + inf_mod.anthropic_messages(payload, _Req(), current_subject = "u") + ) except HTTPException as e: assert not ( e.status_code == 400 diff --git a/tests/test_pr5061_r2_image_dedup_last_user.py b/tests/test_pr5061_r2_image_dedup_last_user.py index 60f9e0ff6e..e5ebf6852d 100644 --- a/tests/test_pr5061_r2_image_dedup_last_user.py +++ b/tests/test_pr5061_r2_image_dedup_last_user.py @@ -1,4 +1,5 @@ import os, sys + _backend = os.path.join(os.path.dirname(__file__), "..", "studio", "backend") sys.path.insert(0, _backend) @@ -10,23 +11,25 @@ _TINY = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjC class _P: - def __init__(self, messages, image_base64=None): + def __init__(self, messages, image_base64 = None): self.messages = messages self.image_base64 = image_base64 -def _img_url_part(b64=_TINY): +def _img_url_part(b64 = _TINY): return {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}} def test_prior_user_image_does_not_block_new_base64(): p = _P( - messages=[ - ChatMessage(role="user", content=[{"type": "text", "text": "q1"}, _img_url_part()]), - ChatMessage(role="assistant", content="a1"), - ChatMessage(role="user", content="q2 no inline"), + messages = [ + ChatMessage( + role = "user", content = [{"type": "text", "text": "q1"}, _img_url_part()] + ), + ChatMessage(role = "assistant", content = "a1"), + ChatMessage(role = "user", content = "q2 no inline"), ], - image_base64=_TINY, + image_base64 = _TINY, ) out = _openai_messages_for_passthrough(p) # last user must receive spliced image @@ -36,11 +39,13 @@ def test_prior_user_image_does_not_block_new_base64(): def test_last_user_with_inline_image_skips_splice(): p = _P( - messages=[ - ChatMessage(role="user", content="earlier"), - ChatMessage(role="user", content=[{"type": "text", "text": "last"}, _img_url_part()]), + messages = [ + ChatMessage(role = "user", content = "earlier"), + ChatMessage( + role = "user", content = [{"type": "text", "text": "last"}, _img_url_part()] + ), ], - image_base64=_TINY, + image_base64 = _TINY, ) out = _openai_messages_for_passthrough(p) last_parts = out[-1]["content"] @@ -49,8 +54,8 @@ def test_last_user_with_inline_image_skips_splice(): def test_no_user_messages_appends_trailing_user(): p = _P( - messages=[ChatMessage(role="system", content="sys")], - image_base64=_TINY, + messages = [ChatMessage(role = "system", content = "sys")], + image_base64 = _TINY, ) out = _openai_messages_for_passthrough(p) assert out[-1]["role"] == "user" @@ -59,14 +64,14 @@ def test_no_user_messages_appends_trailing_user(): def test_three_user_turns_only_last_receives_splice(): p = _P( - messages=[ - ChatMessage(role="user", content="u1"), - ChatMessage(role="assistant", content="a1"), - ChatMessage(role="user", content="u2"), - ChatMessage(role="assistant", content="a2"), - ChatMessage(role="user", content="u3"), + messages = [ + ChatMessage(role = "user", content = "u1"), + ChatMessage(role = "assistant", content = "a1"), + ChatMessage(role = "user", content = "u2"), + ChatMessage(role = "assistant", content = "a2"), + ChatMessage(role = "user", content = "u3"), ], - image_base64=_TINY, + image_base64 = _TINY, ) out = _openai_messages_for_passthrough(p) assert isinstance(out[0]["content"], str) diff --git a/tests/test_pr5061_r2_nonstream_success_shape.py b/tests/test_pr5061_r2_nonstream_success_shape.py index 143c510188..d03b7f9124 100644 --- a/tests/test_pr5061_r2_nonstream_success_shape.py +++ b/tests/test_pr5061_r2_nonstream_success_shape.py @@ -15,8 +15,13 @@ class _Llama: def _payload(): return ChatCompletionRequest( - messages=[{"role": "user", "content": "q"}], - tools=[{"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}}], + messages = [{"role": "user", "content": "q"}], + tools = [ + { + "type": "function", + "function": {"name": "f", "parameters": {"type": "object"}}, + } + ], ) @@ -24,16 +29,20 @@ def _build_mock_client(status, payload_json): class _Client: def __init__(self, *a, **kw): pass + async def __aenter__(self): return self + async def __aexit__(self, *a): return False + async def post(self, *a, **kw): m = MagicMock() m.status_code = status m.text = "" m.json = lambda: payload_json return m + return _Client @@ -55,7 +64,10 @@ def test_verbatim_json_body_returned(): { "id": "call_1", "type": "function", - "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + "function": { + "name": "get_weather", + "arguments": '{"city": "Paris"}', + }, } ], }, @@ -65,9 +77,12 @@ def test_verbatim_json_body_returned(): } with patch.object(inf_mod.httpx, "AsyncClient", _build_mock_client(200, native)): - resp = asyncio.run(inf_mod._openai_passthrough_non_streaming(_Llama(), _payload())) + resp = asyncio.run( + inf_mod._openai_passthrough_non_streaming(_Llama(), _payload()) + ) import json + body = json.loads(resp.body.decode("utf-8")) assert body == native # verbatim assert body["choices"][0]["finish_reason"] == "tool_calls" @@ -80,13 +95,18 @@ def test_preserves_native_id_and_model_fields(): native = { "id": "chatcmpl-native-xyz", "model": "llama-native", - "choices": [{"finish_reason": "stop", "message": {"role": "assistant", "content": "ok"}}], + "choices": [ + {"finish_reason": "stop", "message": {"role": "assistant", "content": "ok"}} + ], "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, } with patch.object(inf_mod.httpx, "AsyncClient", _build_mock_client(200, native)): - resp = asyncio.run(inf_mod._openai_passthrough_non_streaming(_Llama(), _payload())) + resp = asyncio.run( + inf_mod._openai_passthrough_non_streaming(_Llama(), _payload()) + ) import json + body = json.loads(resp.body.decode("utf-8")) assert body["id"] == "chatcmpl-native-xyz" assert body["model"] == "llama-native" diff --git a/tests/test_pr5061_r2_nonstream_upstream_log.py b/tests/test_pr5061_r2_nonstream_upstream_log.py index ad4f077d06..4f7f661da0 100644 --- a/tests/test_pr5061_r2_nonstream_upstream_log.py +++ b/tests/test_pr5061_r2_nonstream_upstream_log.py @@ -18,8 +18,13 @@ class _Llama: def _payload(): return ChatCompletionRequest( - messages=[{"role": "user", "content": "q"}], - tools=[{"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}}], + messages = [{"role": "user", "content": "q"}], + tools = [ + { + "type": "function", + "function": {"name": "f", "parameters": {"type": "object"}}, + } + ], ) @@ -27,24 +32,32 @@ def _mk_client(status, text): class _Client: def __init__(self, *a, **kw): pass + async def __aenter__(self): return self + async def __aexit__(self, *a): return False + async def post(self, *a, **kw): m = MagicMock() m.status_code = status m.text = text m.json = lambda: {} return m + return _Client def test_nonstream_non_200_calls_logger_error_with_status_and_body(): from routes import inference as inf_mod - with patch.object(inf_mod.httpx, "AsyncClient", _mk_client(503, "backend overloaded detail")), \ - patch.object(inf_mod.logger, "error") as mock_error: + with ( + patch.object( + inf_mod.httpx, "AsyncClient", _mk_client(503, "backend overloaded detail") + ), + patch.object(inf_mod.logger, "error") as mock_error, + ): with pytest.raises(HTTPException) as ei: asyncio.run(inf_mod._openai_passthrough_non_streaming(_Llama(), _payload())) @@ -55,7 +68,11 @@ def test_nonstream_non_200_calls_logger_error_with_status_and_body(): for call in mock_error.call_args_list: msg = call.args[0] if call.args else "" combined = f"{msg} {call.args} {call.kwargs}" - if "upstream error" in combined and "503" in combined and "backend overloaded" in combined: + if ( + "upstream error" in combined + and "503" in combined + and "backend overloaded" in combined + ): found = True break assert found, f"expected upstream error log with status and body, got {mock_error.call_args_list}" @@ -64,8 +81,10 @@ def test_nonstream_non_200_calls_logger_error_with_status_and_body(): def test_nonstream_200_does_not_call_logger_error(): from routes import inference as inf_mod - with patch.object(inf_mod.httpx, "AsyncClient", _mk_client(200, "{}")), \ - patch.object(inf_mod.logger, "error") as mock_error: + with ( + patch.object(inf_mod.httpx, "AsyncClient", _mk_client(200, "{}")), + patch.object(inf_mod.logger, "error") as mock_error, + ): asyncio.run(inf_mod._openai_passthrough_non_streaming(_Llama(), _payload())) # no upstream-error log on 200 diff --git a/tests/test_pr5061_r2_role_tool_nonpass_guard.py b/tests/test_pr5061_r2_role_tool_nonpass_guard.py index 943896e13c..1278bb9275 100644 --- a/tests/test_pr5061_r2_role_tool_nonpass_guard.py +++ b/tests/test_pr5061_r2_role_tool_nonpass_guard.py @@ -12,7 +12,7 @@ from models.inference import ChatCompletionRequest class _Llama: - def __init__(self, is_loaded=True, supports_tools=True, is_vision=False): + def __init__(self, is_loaded = True, supports_tools = True, is_vision = False): self.is_loaded = is_loaded self.supports_tools = supports_tools self.is_vision = is_vision @@ -42,21 +42,28 @@ async def _marker_nonstream(*a, **kw): async def _call(payload, llama): from routes import inference as inf_mod - with patch.object(inf_mod, "get_llama_cpp_backend", return_value=llama), \ - patch.object(inf_mod, "get_inference_backend", return_value=_Inf()), \ - patch.object(inf_mod, "_openai_passthrough_stream", new=_marker_stream), \ - patch.object(inf_mod, "_openai_passthrough_non_streaming", new=_marker_nonstream): - return await inf_mod.openai_chat_completions(payload, _Req(), current_subject="u") + + with ( + patch.object(inf_mod, "get_llama_cpp_backend", return_value = llama), + patch.object(inf_mod, "get_inference_backend", return_value = _Inf()), + patch.object(inf_mod, "_openai_passthrough_stream", new = _marker_stream), + patch.object( + inf_mod, "_openai_passthrough_non_streaming", new = _marker_nonstream + ), + ): + return await inf_mod.openai_chat_completions( + payload, _Req(), current_subject = "u" + ) def test_role_tool_on_non_gguf_rejected(): payload = ChatCompletionRequest( - messages=[ + messages = [ {"role": "user", "content": "q"}, {"role": "tool", "tool_call_id": "c1", "content": "r"}, ], ) - llama = _Llama(is_loaded=False) + llama = _Llama(is_loaded = False) with pytest.raises(HTTPException) as ei: asyncio.run(_call(payload, llama)) assert ei.value.status_code == 400 @@ -65,12 +72,12 @@ def test_role_tool_on_non_gguf_rejected(): def test_role_tool_on_gguf_without_tool_support_rejected(): payload = ChatCompletionRequest( - messages=[ + messages = [ {"role": "user", "content": "q"}, {"role": "tool", "tool_call_id": "c1", "content": "r"}, ], ) - llama = _Llama(supports_tools=False) + llama = _Llama(supports_tools = False) with pytest.raises(HTTPException) as ei: asyncio.run(_call(payload, llama)) assert ei.value.status_code == 400 @@ -78,11 +85,11 @@ def test_role_tool_on_gguf_without_tool_support_rejected(): def test_role_tool_with_enable_tools_true_rejected(): payload = ChatCompletionRequest( - messages=[ + messages = [ {"role": "user", "content": "q"}, {"role": "tool", "tool_call_id": "c1", "content": "r"}, ], - enable_tools=True, + enable_tools = True, ) llama = _Llama() with pytest.raises(HTTPException) as ei: @@ -92,18 +99,22 @@ def test_role_tool_with_enable_tools_true_rejected(): def test_assistant_tool_only_still_rejected_on_non_passthrough(): payload = ChatCompletionRequest( - messages=[ + messages = [ {"role": "user", "content": "q"}, { "role": "assistant", "content": None, "tool_calls": [ - {"id": "c1", "type": "function", "function": {"name": "f", "arguments": "{}"}} + { + "id": "c1", + "type": "function", + "function": {"name": "f", "arguments": "{}"}, + } ], }, ], ) - llama = _Llama(supports_tools=False) + llama = _Llama(supports_tools = False) with pytest.raises(HTTPException) as ei: asyncio.run(_call(payload, llama)) assert ei.value.status_code == 400 @@ -111,9 +122,9 @@ def test_assistant_tool_only_still_rejected_on_non_passthrough(): def test_no_tool_messages_passes_through(): payload = ChatCompletionRequest( - messages=[{"role": "user", "content": "plain"}], + messages = [{"role": "user", "content": "plain"}], ) - llama = _Llama(supports_tools=False) + llama = _Llama(supports_tools = False) # Should not raise the tool-shape guard; may raise later for non-GGUF/inf path, # but specifically this guard must not fire. try: diff --git a/tests/test_pr5061_r2_sse_done_on_error.py b/tests/test_pr5061_r2_sse_done_on_error.py index cc15d0a6b2..65144b2481 100644 --- a/tests/test_pr5061_r2_sse_done_on_error.py +++ b/tests/test_pr5061_r2_sse_done_on_error.py @@ -26,14 +26,20 @@ def _collect(stream_response): async for chunk in stream_response.body_iterator: out.append(chunk if isinstance(chunk, str) else chunk.decode("utf-8")) return out + return asyncio.run(_run()) def _make_payload(): return ChatCompletionRequest( - messages=[{"role": "user", "content": "q"}], - tools=[{"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}}], - stream=True, + messages = [{"role": "user", "content": "q"}], + tools = [ + { + "type": "function", + "function": {"name": "f", "parameters": {"type": "object"}}, + } + ], + stream = True, ) @@ -42,8 +48,10 @@ def test_done_emitted_after_non_200_error(): class _Resp: status_code = 500 + async def aread(self): return b"server oops" + async def aclose(self): pass @@ -52,7 +60,7 @@ def test_done_emitted_after_non_200_error(): with patch.object(inf_mod.httpx, "AsyncClient") as mock_cls: inst = MagicMock() - inst.build_request = MagicMock(return_value=MagicMock()) + inst.build_request = MagicMock(return_value = MagicMock()) inst.send = _send inst.aclose = AsyncMock() mock_cls.return_value = inst @@ -72,11 +80,11 @@ def test_done_emitted_after_exception(): from routes import inference as inf_mod async def _send(req, stream): - raise httpx.ConnectError("boom", request=httpx.Request("POST", "http://x")) + raise httpx.ConnectError("boom", request = httpx.Request("POST", "http://x")) with patch.object(inf_mod.httpx, "AsyncClient") as mock_cls: inst = MagicMock() - inst.build_request = MagicMock(return_value=MagicMock()) + inst.build_request = MagicMock(return_value = MagicMock()) inst.send = _send inst.aclose = AsyncMock() mock_cls.return_value = inst @@ -96,11 +104,11 @@ def test_done_comes_after_error_chunk(): from routes import inference as inf_mod async def _send(req, stream): - raise httpx.ReadError("reset", request=httpx.Request("POST", "http://x")) + raise httpx.ReadError("reset", request = httpx.Request("POST", "http://x")) with patch.object(inf_mod.httpx, "AsyncClient") as mock_cls: inst = MagicMock() - inst.build_request = MagicMock(return_value=MagicMock()) + inst.build_request = MagicMock(return_value = MagicMock()) inst.send = _send inst.aclose = AsyncMock() mock_cls.return_value = inst diff --git a/tests/test_pr5061_r2_verbatim_stream_chunks.py b/tests/test_pr5061_r2_verbatim_stream_chunks.py index 0d694ba557..5ff5cd0768 100644 --- a/tests/test_pr5061_r2_verbatim_stream_chunks.py +++ b/tests/test_pr5061_r2_verbatim_stream_chunks.py @@ -20,9 +20,14 @@ class _Req: def _payload(): return ChatCompletionRequest( - messages=[{"role": "user", "content": "q"}], - tools=[{"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}}], - stream=True, + messages = [{"role": "user", "content": "q"}], + tools = [ + { + "type": "function", + "function": {"name": "f", "parameters": {"type": "object"}}, + } + ], + stream = True, ) @@ -71,7 +76,7 @@ def _run_stream(fake_lines): with patch.object(inf_mod.httpx, "AsyncClient") as mock_cls: inst = MagicMock() - inst.build_request = MagicMock(return_value=MagicMock()) + inst.build_request = MagicMock(return_value = MagicMock()) inst.send = _send inst.aclose = _aclose_noop mock_cls.return_value = inst @@ -92,21 +97,25 @@ def _run_stream(fake_lines): def test_passthrough_relays_data_lines_verbatim(): - chunks = _run_stream([ - 'data: {"id":"abc","choices":[{"delta":{"content":"hi"}}]}', - 'data: [DONE]', - ]) + chunks = _run_stream( + [ + 'data: {"id":"abc","choices":[{"delta":{"content":"hi"}}]}', + "data: [DONE]", + ] + ) assert any('"id":"abc"' in c for c in chunks) assert any("data: [DONE]" in c for c in chunks) def test_passthrough_ignores_blank_and_non_data_lines(): - chunks = _run_stream([ - "", - ": heartbeat", - 'data: {"x":1}', - 'data: [DONE]', - ]) + chunks = _run_stream( + [ + "", + ": heartbeat", + 'data: {"x":1}', + "data: [DONE]", + ] + ) # Only data: lines propagate. for c in chunks: assert c.startswith("data: ") or c == "" @@ -114,9 +123,11 @@ def test_passthrough_ignores_blank_and_non_data_lines(): def test_passthrough_breaks_on_done(): - chunks = _run_stream([ - 'data: {"a":1}', - 'data: [DONE]', - 'data: {"should_not_appear":true}', - ]) + chunks = _run_stream( + [ + 'data: {"a":1}', + "data: [DONE]", + 'data: {"should_not_appear":true}', + ] + ) assert not any("should_not_appear" in c for c in chunks) From b735c3a006971eb861c05cb9409d792edcd7778b Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 16 Apr 2026 18:39:10 +0000 Subject: [PATCH 11/12] test(studio): consolidate PR #5061 review tests into the passthrough suite The 10 test_pr5061_r2_*.py files at repo root tests/ had three issues: wrong directory (studio tests live at studio/backend/tests/), PR number baked into filenames, and two files (anthropic_nonstream_502, anthropic_tool_choice_warn) duplicated coverage already in Roland's TestFriendlyErrorHttpx / TestAnthropicToolChoiceToOpenAI. A third file (nonstream_upstream_log) pinned logger-call shape, which is brittle diagnostic noise. Dropped the three low-value files and merged the rest into studio/backend/tests/test_openai_tool_passthrough.py as generic classes: - TestLlamaAuthHeaders - TestOpenAIMessagesForPassthrough - TestOpenAIChatCompletionsToolGuards - TestAnthropicEnableToolsToolChoiceConflict - TestOpenAIPassthroughNonStreaming (verbatim body + httpx 502 mapping) - TestOpenAIPassthroughStreamVerbatim - TestOpenAIPassthroughStreamErrorTermination Test count: 65 tests across 8 Test* classes. Net -316 lines. --- .../tests/test_openai_tool_passthrough.py | 640 +++++++++++++++++- .../test_pr5061_r2_anthropic_nonstream_502.py | 97 --- ...st_pr5061_r2_anthropic_tool_choice_warn.py | 49 -- tests/test_pr5061_r2_auth_headers.py | 40 -- ..._pr5061_r2_enable_tools_tool_choice_400.py | 82 --- tests/test_pr5061_r2_image_dedup_last_user.py | 79 --- .../test_pr5061_r2_nonstream_success_shape.py | 112 --- .../test_pr5061_r2_nonstream_upstream_log.py | 93 --- .../test_pr5061_r2_role_tool_nonpass_guard.py | 134 ---- tests/test_pr5061_r2_sse_done_on_error.py | 125 ---- .../test_pr5061_r2_verbatim_stream_chunks.py | 133 ---- 11 files changed, 634 insertions(+), 950 deletions(-) delete mode 100644 tests/test_pr5061_r2_anthropic_nonstream_502.py delete mode 100644 tests/test_pr5061_r2_anthropic_tool_choice_warn.py delete mode 100644 tests/test_pr5061_r2_auth_headers.py delete mode 100644 tests/test_pr5061_r2_enable_tools_tool_choice_400.py delete mode 100644 tests/test_pr5061_r2_image_dedup_last_user.py delete mode 100644 tests/test_pr5061_r2_nonstream_success_shape.py delete mode 100644 tests/test_pr5061_r2_nonstream_upstream_log.py delete mode 100644 tests/test_pr5061_r2_role_tool_nonpass_guard.py delete mode 100644 tests/test_pr5061_r2_sse_done_on_error.py delete mode 100644 tests/test_pr5061_r2_verbatim_stream_chunks.py diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py index d84da19634..17053205b3 100644 --- a/studio/backend/tests/test_openai_tool_passthrough.py +++ b/studio/backend/tests/test_openai_tool_passthrough.py @@ -14,28 +14,47 @@ Covers: defaults to "auto" when unset. - _friendly_error() maps httpx transport errors to a "Lost connection" message so passthrough failures are legible instead of bare 500s. +- _llama_auth_headers() returns a Bearer header only when an API key is set. +- _openai_messages_for_passthrough() splices legacy image_base64 into the + last user message and skips the splice when one is already inline. +- openai_chat_completions() rejects role="tool" / tool_calls-only messages + when the request does not take the passthrough path. +- anthropic_messages() rejects the enable_tools + tool_choice combination. +- _openai_passthrough_non_streaming() wraps httpx transport errors as 502 + and returns the upstream JSON body verbatim on success. +- _openai_passthrough_stream() relays data: lines verbatim, breaks on + [DONE], and always emits [DONE] after an upstream error. No running server or GPU required. """ -import os -import sys - -_backend = os.path.join(os.path.dirname(__file__), "..") -sys.path.insert(0, _backend) +import asyncio +import threading +from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +from fastapi import HTTPException from pydantic import ValidationError +# conftest.py adds the backend root to sys.path so these flat imports resolve. from models.inference import ( + AnthropicMessagesRequest, ChatCompletionRequest, ChatMessage, ) from core.inference.anthropic_compat import ( anthropic_tool_choice_to_openai, ) -from routes.inference import _build_passthrough_payload, _friendly_error +from routes import inference as inference_module +from routes.inference import ( + _build_passthrough_payload, + _friendly_error, + _llama_auth_headers, + _openai_messages_for_passthrough, + _openai_passthrough_non_streaming, + _openai_passthrough_stream, +) # ===================================================================== @@ -410,3 +429,612 @@ class TestFriendlyErrorHttpx: assert ( _friendly_error(RuntimeError("unrelated")) == "An internal error occurred" ) + + +# ===================================================================== +# _llama_auth_headers — Bearer header only when API key is set +# ===================================================================== + + +class TestLlamaAuthHeaders: + def test_returns_bearer_header_when_api_key_set(self): + class _Backend: + _api_key = "k_secret_123" + assert _llama_auth_headers(_Backend()) == { + "Authorization": "Bearer k_secret_123", + } + + def test_returns_none_when_api_key_none(self): + class _Backend: + _api_key = None + assert _llama_auth_headers(_Backend()) is None + + def test_returns_none_when_api_key_attribute_missing(self): + class _Backend: + pass + assert _llama_auth_headers(_Backend()) is None + + def test_returns_none_when_api_key_empty_string(self): + class _Backend: + _api_key = "" + assert _llama_auth_headers(_Backend()) is None + + +# ===================================================================== +# _openai_messages_for_passthrough — legacy image_base64 splice +# ===================================================================== + + +_TINY_PNG_B64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYA" + "AjCB0C8AAAAASUVORK5CYII=" +) + + +def _inline_image_part(b64 = _TINY_PNG_B64): + return { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{b64}"}, + } + + +class _PayloadWithImage: + """Minimal stand-in for ChatCompletionRequest; only attributes read by + _openai_messages_for_passthrough matter here.""" + + def __init__(self, messages, image_base64 = None): + self.messages = messages + self.image_base64 = image_base64 + + +class TestOpenAIMessagesForPassthrough: + def test_prior_user_image_does_not_block_new_base64(self): + payload = _PayloadWithImage( + messages = [ + ChatMessage( + role = "user", + content = [{"type": "text", "text": "q1"}, _inline_image_part()], + ), + ChatMessage(role = "assistant", content = "a1"), + ChatMessage(role = "user", content = "q2 no inline"), + ], + image_base64 = _TINY_PNG_B64, + ) + out = _openai_messages_for_passthrough(payload) + assert isinstance(out[-1]["content"], list) + assert any(part.get("type") == "image_url" for part in out[-1]["content"]) + + def test_last_user_with_inline_image_skips_splice(self): + payload = _PayloadWithImage( + messages = [ + ChatMessage(role = "user", content = "earlier"), + ChatMessage( + role = "user", + content = [ + {"type": "text", "text": "last"}, + _inline_image_part(), + ], + ), + ], + image_base64 = _TINY_PNG_B64, + ) + out = _openai_messages_for_passthrough(payload) + last_parts = out[-1]["content"] + assert sum(1 for part in last_parts if part.get("type") == "image_url") == 1 + + def test_no_user_messages_appends_trailing_user(self): + payload = _PayloadWithImage( + messages = [ChatMessage(role = "system", content = "sys")], + image_base64 = _TINY_PNG_B64, + ) + out = _openai_messages_for_passthrough(payload) + assert out[-1]["role"] == "user" + assert any(part.get("type") == "image_url" for part in out[-1]["content"]) + + def test_only_last_of_multiple_user_turns_receives_splice(self): + payload = _PayloadWithImage( + messages = [ + ChatMessage(role = "user", content = "u1"), + ChatMessage(role = "assistant", content = "a1"), + ChatMessage(role = "user", content = "u2"), + ChatMessage(role = "assistant", content = "a2"), + ChatMessage(role = "user", content = "u3"), + ], + image_base64 = _TINY_PNG_B64, + ) + out = _openai_messages_for_passthrough(payload) + assert isinstance(out[0]["content"], str) + assert isinstance(out[2]["content"], str) + assert isinstance(out[4]["content"], list) + + +# ===================================================================== +# openai_chat_completions — tool-shape guards on non-passthrough paths +# ===================================================================== + + +class _FakeLlamaBackend: + def __init__(self, is_loaded = True, supports_tools = True, is_vision = False): + self.is_loaded = is_loaded + self.supports_tools = supports_tools + self.is_vision = is_vision + self._is_audio = False + self.model_identifier = "stub" + self.base_url = "http://127.0.0.1:0" + self._api_key = None + + +class _FakeInferenceBackend: + active_model_name = "hf" + models = {"hf": {}} + + +class _FakeFastAPIRequest: + async def is_disconnected(self): + return False + + +async def _marker_stream(*args, **kwargs): + return ("passthrough_stream", None) + + +async def _marker_nonstream(*args, **kwargs): + return ("passthrough_nonstream", None) + + +async def _call_openai_chat_completions(payload, llama): + with ( + patch.object(inference_module, "get_llama_cpp_backend", return_value = llama), + patch.object( + inference_module, "get_inference_backend", return_value = _FakeInferenceBackend(), + ), + patch.object(inference_module, "_openai_passthrough_stream", new = _marker_stream), + patch.object( + inference_module, "_openai_passthrough_non_streaming", new = _marker_nonstream, + ), + ): + return await inference_module.openai_chat_completions( + payload, _FakeFastAPIRequest(), current_subject = "u", + ) + + +class TestOpenAIChatCompletionsToolGuards: + """When the request does NOT take the tool-passthrough path, messages + with role="tool" or assistant tool_calls-only must be rejected at the + route boundary rather than producing an opaque upstream error.""" + + def _tool_result_payload(self, **extra): + return ChatCompletionRequest( + messages = [ + {"role": "user", "content": "q"}, + {"role": "tool", "tool_call_id": "c1", "content": "r"}, + ], + **extra, + ) + + def test_role_tool_on_non_gguf_rejected(self): + payload = self._tool_result_payload() + with pytest.raises(HTTPException) as exc_info: + asyncio.run( + _call_openai_chat_completions( + payload, _FakeLlamaBackend(is_loaded = False), + ) + ) + assert exc_info.value.status_code == 400 + + def test_role_tool_on_gguf_without_tool_support_rejected(self): + payload = self._tool_result_payload() + with pytest.raises(HTTPException) as exc_info: + asyncio.run( + _call_openai_chat_completions( + payload, _FakeLlamaBackend(supports_tools = False), + ) + ) + assert exc_info.value.status_code == 400 + + def test_role_tool_with_enable_tools_true_rejected(self): + payload = self._tool_result_payload(enable_tools = True) + with pytest.raises(HTTPException) as exc_info: + asyncio.run(_call_openai_chat_completions(payload, _FakeLlamaBackend())) + assert exc_info.value.status_code == 400 + + def test_assistant_tool_calls_only_rejected_when_no_passthrough(self): + payload = ChatCompletionRequest( + messages = [ + {"role": "user", "content": "q"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "f", "arguments": "{}"}, + } + ], + }, + ], + ) + with pytest.raises(HTTPException) as exc_info: + asyncio.run( + _call_openai_chat_completions( + payload, _FakeLlamaBackend(supports_tools = False), + ) + ) + assert exc_info.value.status_code == 400 + + def test_plain_user_message_does_not_trip_tool_guard(self): + payload = ChatCompletionRequest( + messages = [{"role": "user", "content": "plain"}], + ) + try: + asyncio.run( + _call_openai_chat_completions( + payload, _FakeLlamaBackend(supports_tools = False), + ) + ) + except HTTPException as exc: + # Downstream paths may fail for unrelated reasons; only assert + # the tool-shape guard did NOT fire. + assert "role='tool'" not in (exc.detail or "") + assert "tool_calls-only" not in (exc.detail or "") + + +# ===================================================================== +# anthropic_messages — enable_tools + tool_choice conflict +# ===================================================================== + + +class TestAnthropicEnableToolsToolChoiceConflict: + """Server-side agentic loop (enable_tools=True) does not honor + tool_choice. Reject the combination at the route boundary with 400 + so callers don't silently see their tool_choice dropped.""" + + def _payload(self, *, enable_tools, tool_choice): + return AnthropicMessagesRequest( + model = "default", + max_tokens = 64, + messages = [{"role": "user", "content": "q"}], + tool_choice = tool_choice, + enable_tools = enable_tools, + ) + + def test_enable_tools_true_with_tool_choice_raises_400(self): + payload = self._payload(enable_tools = True, tool_choice = {"type": "any"}) + with patch.object( + inference_module, + "get_llama_cpp_backend", + return_value = _FakeLlamaBackend(), + ): + with pytest.raises(HTTPException) as exc_info: + asyncio.run( + inference_module.anthropic_messages( + payload, _FakeFastAPIRequest(), current_subject = "u", + ) + ) + assert exc_info.value.status_code == 400 + assert "tool_choice" in exc_info.value.detail + assert "enable_tools" in exc_info.value.detail + + def test_enable_tools_false_with_tool_choice_skips_guard(self): + payload = self._payload(enable_tools = False, tool_choice = {"type": "any"}) + with patch.object( + inference_module, + "get_llama_cpp_backend", + return_value = _FakeLlamaBackend(), + ): + try: + asyncio.run( + inference_module.anthropic_messages( + payload, _FakeFastAPIRequest(), current_subject = "u", + ) + ) + except HTTPException as exc: + assert not ( + exc.status_code == 400 + and "tool_choice is not honored" in (exc.detail or "") + ) + except Exception: + # Any other failure downstream is fine; we only pin that + # the enable_tools+tool_choice guard does NOT fire. + pass + + +# ===================================================================== +# _openai_passthrough_non_streaming — verbatim body + httpx 502 mapping +# ===================================================================== + + +class _FakeLlamaBase: + base_url = "http://127.0.0.1:0" + _api_key = None + + +def _openai_tools_payload(stream = False): + return ChatCompletionRequest( + messages = [{"role": "user", "content": "q"}], + tools = [ + { + "type": "function", + "function": {"name": "f", "parameters": {"type": "object"}}, + } + ], + stream = stream, + ) + + +def _mock_async_client_post(status_code, *, json_body = None, text_body = ""): + class _Client: + def __init__(self, *args, **kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + return False + + async def post(self, *args, **kwargs): + resp = MagicMock() + resp.status_code = status_code + resp.text = text_body + resp.json = lambda: (json_body if json_body is not None else {}) + return resp + + return _Client + + +def _mock_async_client_raise(exc_factory): + class _Client: + def __init__(self, *args, **kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + return False + + async def post(self, *args, **kwargs): + raise exc_factory() + + return _Client + + +class TestOpenAIPassthroughNonStreaming: + def test_verbatim_json_body_returned_on_success(self): + native = { + "id": "chatcmpl-foo", + "object": "chat.completion", + "model": "qwen-native", + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "Paris"}', + }, + } + ], + }, + } + ], + "usage": {"prompt_tokens": 42, "completion_tokens": 7, "total_tokens": 49}, + } + client_cls = _mock_async_client_post(200, json_body = native) + with patch.object(inference_module.httpx, "AsyncClient", client_cls): + resp = asyncio.run( + _openai_passthrough_non_streaming(_FakeLlamaBase(), _openai_tools_payload()) + ) + import json + body = json.loads(resp.body.decode("utf-8")) + assert body == native + assert body["choices"][0]["finish_reason"] == "tool_calls" + + def test_preserves_native_id_and_model(self): + native = { + "id": "chatcmpl-native-xyz", + "model": "llama-native", + "choices": [ + { + "finish_reason": "stop", + "message": {"role": "assistant", "content": "ok"}, + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + client_cls = _mock_async_client_post(200, json_body = native) + with patch.object(inference_module.httpx, "AsyncClient", client_cls): + resp = asyncio.run( + _openai_passthrough_non_streaming(_FakeLlamaBase(), _openai_tools_payload()) + ) + import json + body = json.loads(resp.body.decode("utf-8")) + assert body["id"] == "chatcmpl-native-xyz" + assert body["model"] == "llama-native" + + def test_httpx_connect_error_mapped_to_502(self): + client_cls = _mock_async_client_raise( + lambda: httpx.ConnectError( + "refused", request = httpx.Request("POST", "http://x"), + ) + ) + with patch.object(inference_module.httpx, "AsyncClient", client_cls): + with pytest.raises(HTTPException) as exc_info: + asyncio.run( + _openai_passthrough_non_streaming( + _FakeLlamaBase(), _openai_tools_payload(), + ) + ) + assert exc_info.value.status_code == 502 + assert "Lost connection" in exc_info.value.detail + + def test_httpx_read_error_mapped_to_502(self): + client_cls = _mock_async_client_raise( + lambda: httpx.ReadError("eof", request = httpx.Request("POST", "http://x")), + ) + with patch.object(inference_module.httpx, "AsyncClient", client_cls): + with pytest.raises(HTTPException) as exc_info: + asyncio.run( + _openai_passthrough_non_streaming( + _FakeLlamaBase(), _openai_tools_payload(), + ) + ) + assert exc_info.value.status_code == 502 + + +# ===================================================================== +# _openai_passthrough_stream — verbatim data: lines, [DONE] on error +# ===================================================================== + + +class _FakeStreamResponse: + """Stand-in for httpx.Response that yields a fixed list of SSE lines.""" + + def __init__(self, lines, status_code = 200): + self.status_code = status_code + self._lines = list(lines) + + def aiter_lines(self): + parent = self + + class _Iter: + def __init__(self): + self._idx = 0 + + def __aiter__(self): + return self + + async def __anext__(self): + if self._idx >= len(parent._lines): + raise StopAsyncIteration + v = parent._lines[self._idx] + self._idx += 1 + return v + + async def aclose(self): + pass + + return _Iter() + + async def aread(self): + return b"" + + async def aclose(self): + pass + + +def _run_passthrough_stream(*, send_returns = None, send_raises = None): + """Drive _openai_passthrough_stream with a fake httpx client and return + the list of emitted SSE chunks (decoded to str).""" + + async def _send(req, stream): + if send_raises is not None: + raise send_raises() + return send_returns + + with patch.object(inference_module.httpx, "AsyncClient") as mock_cls: + instance = MagicMock() + instance.build_request = MagicMock(return_value = MagicMock()) + instance.send = _send + instance.aclose = AsyncMock() + mock_cls.return_value = instance + + resp = asyncio.run( + _openai_passthrough_stream( + _FakeFastAPIRequest(), + threading.Event(), + _FakeLlamaBase(), + _openai_tools_payload(stream = True), + ) + ) + + async def _collect(): + return [ + chunk if isinstance(chunk, str) else chunk.decode("utf-8") + async for chunk in resp.body_iterator + ] + + return asyncio.run(_collect()) + + +class TestOpenAIPassthroughStreamVerbatim: + def test_relays_data_lines_verbatim(self): + chunks = _run_passthrough_stream( + send_returns = _FakeStreamResponse([ + 'data: {"id":"abc","choices":[{"delta":{"content":"hi"}}]}', + "data: [DONE]", + ]), + ) + assert any('"id":"abc"' in c for c in chunks) + assert any("data: [DONE]" in c for c in chunks) + + def test_ignores_blank_and_non_data_lines(self): + chunks = _run_passthrough_stream( + send_returns = _FakeStreamResponse([ + "", + ": heartbeat", + 'data: {"x":1}', + "data: [DONE]", + ]), + ) + for chunk in chunks: + assert chunk.startswith("data: ") or chunk == "" + assert any('"x":1' in c for c in chunks) + + def test_breaks_on_done(self): + chunks = _run_passthrough_stream( + send_returns = _FakeStreamResponse([ + 'data: {"a":1}', + "data: [DONE]", + 'data: {"should_not_appear":true}', + ]), + ) + assert not any("should_not_appear" in c for c in chunks) + + +class TestOpenAIPassthroughStreamErrorTermination: + """On upstream failure (non-200 or transport exception), the stream + must emit an SSE error chunk followed by `data: [DONE]` so clients + that wait for [DONE] don't hang.""" + + def test_done_emitted_after_non_200_error(self): + class _Resp: + status_code = 500 + + async def aread(self): + return b"server oops" + + async def aclose(self): + pass + + chunks = _run_passthrough_stream(send_returns = _Resp()) + assert any('"error"' in c for c in chunks) + assert any(c.strip() == "data: [DONE]" for c in chunks) + + def test_done_emitted_after_transport_exception(self): + chunks = _run_passthrough_stream( + send_raises = lambda: httpx.ConnectError( + "boom", request = httpx.Request("POST", "http://x"), + ), + ) + assert any('"error"' in c for c in chunks) + assert any(c.strip() == "data: [DONE]" for c in chunks) + + def test_done_comes_after_error_chunk(self): + chunks = _run_passthrough_stream( + send_raises = lambda: httpx.ReadError( + "reset", request = httpx.Request("POST", "http://x"), + ), + ) + err_idx = next(i for i, c in enumerate(chunks) if '"error"' in c) + done_idx = next(i for i, c in enumerate(chunks) if c.strip() == "data: [DONE]") + assert done_idx > err_idx diff --git a/tests/test_pr5061_r2_anthropic_nonstream_502.py b/tests/test_pr5061_r2_anthropic_nonstream_502.py deleted file mode 100644 index 92b6856a76..0000000000 --- a/tests/test_pr5061_r2_anthropic_nonstream_502.py +++ /dev/null @@ -1,97 +0,0 @@ -import asyncio -import os, sys -from unittest.mock import patch - -_backend = os.path.join(os.path.dirname(__file__), "..", "studio", "backend") -sys.path.insert(0, _backend) - -import httpx -import pytest -from fastapi import HTTPException - - -class _Llama: - base_url = "http://127.0.0.1:0" - _api_key = None - - -def test_httpx_connect_error_mapped_to_502(): - from routes import inference as inf_mod - - class _BadClient: - def __init__(self, *a, **kw): - pass - - async def __aenter__(self): - return self - - async def __aexit__(self, *a): - return False - - async def post(self, *a, **kw): - raise httpx.ConnectError( - "refused", request = httpx.Request("POST", "http://x") - ) - - with patch.object(inf_mod.httpx, "AsyncClient", _BadClient): - with pytest.raises(HTTPException) as ei: - asyncio.run( - inf_mod._anthropic_passthrough_non_streaming( - _Llama(), - openai_messages = [{"role": "user", "content": "q"}], - openai_tools = [ - { - "type": "function", - "function": {"name": "f", "parameters": {"type": "object"}}, - } - ], - temperature = 0.6, - top_p = 0.95, - top_k = 20, - max_tokens = None, - message_id = "msg_1", - model_name = "stub", - tool_choice = "auto", - ) - ) - assert ei.value.status_code == 502 - assert "Lost connection" in ei.value.detail - - -def test_httpx_read_error_mapped_to_502(): - from routes import inference as inf_mod - - class _BadClient: - def __init__(self, *a, **kw): - pass - - async def __aenter__(self): - return self - - async def __aexit__(self, *a): - return False - - async def post(self, *a, **kw): - raise httpx.ReadError("eof", request = httpx.Request("POST", "http://x")) - - with patch.object(inf_mod.httpx, "AsyncClient", _BadClient): - with pytest.raises(HTTPException) as ei: - asyncio.run( - inf_mod._anthropic_passthrough_non_streaming( - _Llama(), - openai_messages = [{"role": "user", "content": "q"}], - openai_tools = [ - { - "type": "function", - "function": {"name": "f", "parameters": {"type": "object"}}, - } - ], - temperature = 0.6, - top_p = 0.95, - top_k = 20, - max_tokens = None, - message_id = "msg_1", - model_name = "stub", - ) - ) - assert ei.value.status_code == 502 diff --git a/tests/test_pr5061_r2_anthropic_tool_choice_warn.py b/tests/test_pr5061_r2_anthropic_tool_choice_warn.py deleted file mode 100644 index 5e79ab38eb..0000000000 --- a/tests/test_pr5061_r2_anthropic_tool_choice_warn.py +++ /dev/null @@ -1,49 +0,0 @@ -import os, sys -from unittest.mock import patch, MagicMock - -_backend = os.path.join(os.path.dirname(__file__), "..", "studio", "backend") -sys.path.insert(0, _backend) - -from routes import inference as inf_mod - - -def _simulate_coerce(tool_choice): - """Mimic the exact snippet that lives at the top of anthropic_messages().""" - translated = inf_mod.anthropic_tool_choice_to_openai(tool_choice) - if translated is None: - if tool_choice is not None: - inf_mod.logger.warning( - "anthropic_messages.tool_choice_unrecognized", - tool_choice = tool_choice, - ) - translated = "auto" - return translated - - -def test_unrecognized_dict_warns_and_falls_back_to_auto(): - with patch.object(inf_mod.logger, "warning") as mock_warn: - result = _simulate_coerce({"type": "wibble"}) - assert result == "auto" - mock_warn.assert_called_once() - assert mock_warn.call_args.args[0] == "anthropic_messages.tool_choice_unrecognized" - - -def test_non_dict_input_warns_and_falls_back(): - with patch.object(inf_mod.logger, "warning") as mock_warn: - result = _simulate_coerce("auto_string") - assert result == "auto" - mock_warn.assert_called_once() - - -def test_none_input_no_warning(): - with patch.object(inf_mod.logger, "warning") as mock_warn: - result = _simulate_coerce(None) - assert result == "auto" - mock_warn.assert_not_called() - - -def test_recognized_input_no_warning(): - with patch.object(inf_mod.logger, "warning") as mock_warn: - result = _simulate_coerce({"type": "any"}) - assert result == "required" - mock_warn.assert_not_called() diff --git a/tests/test_pr5061_r2_auth_headers.py b/tests/test_pr5061_r2_auth_headers.py deleted file mode 100644 index 8ba2939400..0000000000 --- a/tests/test_pr5061_r2_auth_headers.py +++ /dev/null @@ -1,40 +0,0 @@ -import os, sys - -_backend = os.path.join(os.path.dirname(__file__), "..", "studio", "backend") -sys.path.insert(0, _backend) - -from routes.inference import _llama_auth_headers - - -class _WithKey: - _api_key = "k_secret_123" - - -class _NoKey: - _api_key = None - - -class _Missing: - pass - - -class _EmptyKey: - _api_key = "" - - -def test_returns_bearer_header_when_api_key_set(): - headers = _llama_auth_headers(_WithKey()) - assert headers == {"Authorization": "Bearer k_secret_123"} - - -def test_returns_none_when_api_key_none(): - assert _llama_auth_headers(_NoKey()) is None - - -def test_returns_none_when_api_key_attribute_missing(): - assert _llama_auth_headers(_Missing()) is None - - -def test_returns_none_when_api_key_empty_string(): - # Empty string is falsy; the helper should treat it as absent. - assert _llama_auth_headers(_EmptyKey()) is None diff --git a/tests/test_pr5061_r2_enable_tools_tool_choice_400.py b/tests/test_pr5061_r2_enable_tools_tool_choice_400.py deleted file mode 100644 index 9e1b1013c2..0000000000 --- a/tests/test_pr5061_r2_enable_tools_tool_choice_400.py +++ /dev/null @@ -1,82 +0,0 @@ -import asyncio -import os, sys -from unittest.mock import patch - -_backend = os.path.join(os.path.dirname(__file__), "..", "studio", "backend") -sys.path.insert(0, _backend) - -import pytest -from fastapi import HTTPException - - -class _Llama: - is_loaded = True - supports_tools = True - is_vision = False - _is_audio = False - model_identifier = "stub" - base_url = "http://127.0.0.1:0" - _api_key = None - - -class _Req: - async def is_disconnected(self): - return False - - -def _build_payload(enable_tools, tool_choice): - from models.inference import AnthropicMessagesRequest - - return AnthropicMessagesRequest( - model = "default", - max_tokens = 64, - messages = [{"role": "user", "content": "q"}], - tool_choice = tool_choice, - enable_tools = enable_tools, - ) - - -def test_enable_tools_true_with_tool_choice_raises_400(): - from routes import inference as inf_mod - - payload = _build_payload(enable_tools = True, tool_choice = {"type": "any"}) - with patch.object(inf_mod, "get_llama_cpp_backend", return_value = _Llama()): - with pytest.raises(HTTPException) as ei: - asyncio.run( - inf_mod.anthropic_messages(payload, _Req(), current_subject = "u") - ) - assert ei.value.status_code == 400 - assert "tool_choice" in ei.value.detail - assert "enable_tools" in ei.value.detail - - -def test_guard_logic_unit(): - # Re-implement the guard predicate exactly and pin it. - def would_raise(enable_tools, supports_tools, tool_choice): - server_tools = enable_tools and supports_tools - return bool(server_tools and tool_choice is not None) - - assert would_raise(True, True, {"type": "any"}) is True - assert would_raise(True, True, None) is False - assert would_raise(False, True, {"type": "any"}) is False - assert would_raise(True, False, {"type": "any"}) is False - - -def test_enable_tools_false_with_tool_choice_does_not_raise_the_guard(): - from routes import inference as inf_mod - - payload = _build_payload(enable_tools = False, tool_choice = {"type": "any"}) - with patch.object(inf_mod, "get_llama_cpp_backend", return_value = _Llama()): - try: - asyncio.run( - inf_mod.anthropic_messages(payload, _Req(), current_subject = "u") - ) - except HTTPException as e: - assert not ( - e.status_code == 400 - and "tool_choice is not honored" in (e.detail or "") - ) - except Exception: - # Any other failure downstream is fine; we only pin that the - # enable_tools+tool_choice guard does NOT fire when enable_tools is False. - pass diff --git a/tests/test_pr5061_r2_image_dedup_last_user.py b/tests/test_pr5061_r2_image_dedup_last_user.py deleted file mode 100644 index e5ebf6852d..0000000000 --- a/tests/test_pr5061_r2_image_dedup_last_user.py +++ /dev/null @@ -1,79 +0,0 @@ -import os, sys - -_backend = os.path.join(os.path.dirname(__file__), "..", "studio", "backend") -sys.path.insert(0, _backend) - -from models.inference import ChatMessage -from routes.inference import _openai_messages_for_passthrough - - -_TINY = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" - - -class _P: - def __init__(self, messages, image_base64 = None): - self.messages = messages - self.image_base64 = image_base64 - - -def _img_url_part(b64 = _TINY): - return {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}} - - -def test_prior_user_image_does_not_block_new_base64(): - p = _P( - messages = [ - ChatMessage( - role = "user", content = [{"type": "text", "text": "q1"}, _img_url_part()] - ), - ChatMessage(role = "assistant", content = "a1"), - ChatMessage(role = "user", content = "q2 no inline"), - ], - image_base64 = _TINY, - ) - out = _openai_messages_for_passthrough(p) - # last user must receive spliced image - assert isinstance(out[-1]["content"], list) - assert any(x.get("type") == "image_url" for x in out[-1]["content"]) - - -def test_last_user_with_inline_image_skips_splice(): - p = _P( - messages = [ - ChatMessage(role = "user", content = "earlier"), - ChatMessage( - role = "user", content = [{"type": "text", "text": "last"}, _img_url_part()] - ), - ], - image_base64 = _TINY, - ) - out = _openai_messages_for_passthrough(p) - last_parts = out[-1]["content"] - assert sum(1 for x in last_parts if x.get("type") == "image_url") == 1 - - -def test_no_user_messages_appends_trailing_user(): - p = _P( - messages = [ChatMessage(role = "system", content = "sys")], - image_base64 = _TINY, - ) - out = _openai_messages_for_passthrough(p) - assert out[-1]["role"] == "user" - assert any(x.get("type") == "image_url" for x in out[-1]["content"]) - - -def test_three_user_turns_only_last_receives_splice(): - p = _P( - messages = [ - ChatMessage(role = "user", content = "u1"), - ChatMessage(role = "assistant", content = "a1"), - ChatMessage(role = "user", content = "u2"), - ChatMessage(role = "assistant", content = "a2"), - ChatMessage(role = "user", content = "u3"), - ], - image_base64 = _TINY, - ) - out = _openai_messages_for_passthrough(p) - assert isinstance(out[0]["content"], str) - assert isinstance(out[2]["content"], str) - assert isinstance(out[4]["content"], list) diff --git a/tests/test_pr5061_r2_nonstream_success_shape.py b/tests/test_pr5061_r2_nonstream_success_shape.py deleted file mode 100644 index d03b7f9124..0000000000 --- a/tests/test_pr5061_r2_nonstream_success_shape.py +++ /dev/null @@ -1,112 +0,0 @@ -import asyncio -import os, sys -from unittest.mock import patch, MagicMock - -_backend = os.path.join(os.path.dirname(__file__), "..", "studio", "backend") -sys.path.insert(0, _backend) - -from models.inference import ChatCompletionRequest - - -class _Llama: - base_url = "http://127.0.0.1:0" - _api_key = None - - -def _payload(): - return ChatCompletionRequest( - messages = [{"role": "user", "content": "q"}], - tools = [ - { - "type": "function", - "function": {"name": "f", "parameters": {"type": "object"}}, - } - ], - ) - - -def _build_mock_client(status, payload_json): - class _Client: - def __init__(self, *a, **kw): - pass - - async def __aenter__(self): - return self - - async def __aexit__(self, *a): - return False - - async def post(self, *a, **kw): - m = MagicMock() - m.status_code = status - m.text = "" - m.json = lambda: payload_json - return m - - return _Client - - -def test_verbatim_json_body_returned(): - from routes import inference as inf_mod - - native = { - "id": "chatcmpl-foo", - "object": "chat.completion", - "model": "qwen-native", - "choices": [ - { - "index": 0, - "finish_reason": "tool_calls", - "message": { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": '{"city": "Paris"}', - }, - } - ], - }, - } - ], - "usage": {"prompt_tokens": 42, "completion_tokens": 7, "total_tokens": 49}, - } - - with patch.object(inf_mod.httpx, "AsyncClient", _build_mock_client(200, native)): - resp = asyncio.run( - inf_mod._openai_passthrough_non_streaming(_Llama(), _payload()) - ) - - import json - - body = json.loads(resp.body.decode("utf-8")) - assert body == native # verbatim - assert body["choices"][0]["finish_reason"] == "tool_calls" - assert body["usage"]["prompt_tokens"] == 42 - - -def test_preserves_native_id_and_model_fields(): - from routes import inference as inf_mod - - native = { - "id": "chatcmpl-native-xyz", - "model": "llama-native", - "choices": [ - {"finish_reason": "stop", "message": {"role": "assistant", "content": "ok"}} - ], - "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, - } - with patch.object(inf_mod.httpx, "AsyncClient", _build_mock_client(200, native)): - resp = asyncio.run( - inf_mod._openai_passthrough_non_streaming(_Llama(), _payload()) - ) - - import json - - body = json.loads(resp.body.decode("utf-8")) - assert body["id"] == "chatcmpl-native-xyz" - assert body["model"] == "llama-native" diff --git a/tests/test_pr5061_r2_nonstream_upstream_log.py b/tests/test_pr5061_r2_nonstream_upstream_log.py deleted file mode 100644 index 4f7f661da0..0000000000 --- a/tests/test_pr5061_r2_nonstream_upstream_log.py +++ /dev/null @@ -1,93 +0,0 @@ -import asyncio -import os, sys -from unittest.mock import patch, MagicMock - -_backend = os.path.join(os.path.dirname(__file__), "..", "studio", "backend") -sys.path.insert(0, _backend) - -import pytest -from fastapi import HTTPException - -from models.inference import ChatCompletionRequest - - -class _Llama: - base_url = "http://127.0.0.1:0" - _api_key = None - - -def _payload(): - return ChatCompletionRequest( - messages = [{"role": "user", "content": "q"}], - tools = [ - { - "type": "function", - "function": {"name": "f", "parameters": {"type": "object"}}, - } - ], - ) - - -def _mk_client(status, text): - class _Client: - def __init__(self, *a, **kw): - pass - - async def __aenter__(self): - return self - - async def __aexit__(self, *a): - return False - - async def post(self, *a, **kw): - m = MagicMock() - m.status_code = status - m.text = text - m.json = lambda: {} - return m - - return _Client - - -def test_nonstream_non_200_calls_logger_error_with_status_and_body(): - from routes import inference as inf_mod - - with ( - patch.object( - inf_mod.httpx, "AsyncClient", _mk_client(503, "backend overloaded detail") - ), - patch.object(inf_mod.logger, "error") as mock_error, - ): - with pytest.raises(HTTPException) as ei: - asyncio.run(inf_mod._openai_passthrough_non_streaming(_Llama(), _payload())) - - assert ei.value.status_code == 503 - mock_error.assert_called() - # Check any error call mentions upstream status and body - found = False - for call in mock_error.call_args_list: - msg = call.args[0] if call.args else "" - combined = f"{msg} {call.args} {call.kwargs}" - if ( - "upstream error" in combined - and "503" in combined - and "backend overloaded" in combined - ): - found = True - break - assert found, f"expected upstream error log with status and body, got {mock_error.call_args_list}" - - -def test_nonstream_200_does_not_call_logger_error(): - from routes import inference as inf_mod - - with ( - patch.object(inf_mod.httpx, "AsyncClient", _mk_client(200, "{}")), - patch.object(inf_mod.logger, "error") as mock_error, - ): - asyncio.run(inf_mod._openai_passthrough_non_streaming(_Llama(), _payload())) - - # no upstream-error log on 200 - for call in mock_error.call_args_list: - msg = call.args[0] if call.args else "" - assert "upstream error" not in msg diff --git a/tests/test_pr5061_r2_role_tool_nonpass_guard.py b/tests/test_pr5061_r2_role_tool_nonpass_guard.py deleted file mode 100644 index 1278bb9275..0000000000 --- a/tests/test_pr5061_r2_role_tool_nonpass_guard.py +++ /dev/null @@ -1,134 +0,0 @@ -import asyncio -import os, sys -from unittest.mock import patch - -_backend = os.path.join(os.path.dirname(__file__), "..", "studio", "backend") -sys.path.insert(0, _backend) - -import pytest -from fastapi import HTTPException - -from models.inference import ChatCompletionRequest - - -class _Llama: - def __init__(self, is_loaded = True, supports_tools = True, is_vision = False): - self.is_loaded = is_loaded - self.supports_tools = supports_tools - self.is_vision = is_vision - self._is_audio = False - self.model_identifier = "stub" - self.base_url = "http://127.0.0.1:0" - self._api_key = None - - -class _Inf: - active_model_name = "hf" - models = {"hf": {}} - - -class _Req: - async def is_disconnected(self): - return False - - -async def _marker_stream(*a, **kw): - return ("passthrough_stream", None) - - -async def _marker_nonstream(*a, **kw): - return ("passthrough_nonstream", None) - - -async def _call(payload, llama): - from routes import inference as inf_mod - - with ( - patch.object(inf_mod, "get_llama_cpp_backend", return_value = llama), - patch.object(inf_mod, "get_inference_backend", return_value = _Inf()), - patch.object(inf_mod, "_openai_passthrough_stream", new = _marker_stream), - patch.object( - inf_mod, "_openai_passthrough_non_streaming", new = _marker_nonstream - ), - ): - return await inf_mod.openai_chat_completions( - payload, _Req(), current_subject = "u" - ) - - -def test_role_tool_on_non_gguf_rejected(): - payload = ChatCompletionRequest( - messages = [ - {"role": "user", "content": "q"}, - {"role": "tool", "tool_call_id": "c1", "content": "r"}, - ], - ) - llama = _Llama(is_loaded = False) - with pytest.raises(HTTPException) as ei: - asyncio.run(_call(payload, llama)) - assert ei.value.status_code == 400 - assert "role='tool'" in ei.value.detail or "tool" in ei.value.detail.lower() - - -def test_role_tool_on_gguf_without_tool_support_rejected(): - payload = ChatCompletionRequest( - messages = [ - {"role": "user", "content": "q"}, - {"role": "tool", "tool_call_id": "c1", "content": "r"}, - ], - ) - llama = _Llama(supports_tools = False) - with pytest.raises(HTTPException) as ei: - asyncio.run(_call(payload, llama)) - assert ei.value.status_code == 400 - - -def test_role_tool_with_enable_tools_true_rejected(): - payload = ChatCompletionRequest( - messages = [ - {"role": "user", "content": "q"}, - {"role": "tool", "tool_call_id": "c1", "content": "r"}, - ], - enable_tools = True, - ) - llama = _Llama() - with pytest.raises(HTTPException) as ei: - asyncio.run(_call(payload, llama)) - assert ei.value.status_code == 400 - - -def test_assistant_tool_only_still_rejected_on_non_passthrough(): - payload = ChatCompletionRequest( - messages = [ - {"role": "user", "content": "q"}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "c1", - "type": "function", - "function": {"name": "f", "arguments": "{}"}, - } - ], - }, - ], - ) - llama = _Llama(supports_tools = False) - with pytest.raises(HTTPException) as ei: - asyncio.run(_call(payload, llama)) - assert ei.value.status_code == 400 - - -def test_no_tool_messages_passes_through(): - payload = ChatCompletionRequest( - messages = [{"role": "user", "content": "plain"}], - ) - llama = _Llama(supports_tools = False) - # Should not raise the tool-shape guard; may raise later for non-GGUF/inf path, - # but specifically this guard must not fire. - try: - asyncio.run(_call(payload, llama)) - except HTTPException as e: - assert "role='tool'" not in (e.detail or "") - assert "tool_calls-only" not in (e.detail or "") diff --git a/tests/test_pr5061_r2_sse_done_on_error.py b/tests/test_pr5061_r2_sse_done_on_error.py deleted file mode 100644 index 65144b2481..0000000000 --- a/tests/test_pr5061_r2_sse_done_on_error.py +++ /dev/null @@ -1,125 +0,0 @@ -import asyncio -import os, sys, threading -from unittest.mock import patch, MagicMock, AsyncMock - -_backend = os.path.join(os.path.dirname(__file__), "..", "studio", "backend") -sys.path.insert(0, _backend) - -import httpx - -from models.inference import ChatCompletionRequest - - -class _Llama: - base_url = "http://127.0.0.1:0" - _api_key = None - - -class _Req: - async def is_disconnected(self): - return False - - -def _collect(stream_response): - async def _run(): - out = [] - async for chunk in stream_response.body_iterator: - out.append(chunk if isinstance(chunk, str) else chunk.decode("utf-8")) - return out - - return asyncio.run(_run()) - - -def _make_payload(): - return ChatCompletionRequest( - messages = [{"role": "user", "content": "q"}], - tools = [ - { - "type": "function", - "function": {"name": "f", "parameters": {"type": "object"}}, - } - ], - stream = True, - ) - - -def test_done_emitted_after_non_200_error(): - from routes import inference as inf_mod - - class _Resp: - status_code = 500 - - async def aread(self): - return b"server oops" - - async def aclose(self): - pass - - async def _send(req, stream): - return _Resp() - - with patch.object(inf_mod.httpx, "AsyncClient") as mock_cls: - inst = MagicMock() - inst.build_request = MagicMock(return_value = MagicMock()) - inst.send = _send - inst.aclose = AsyncMock() - mock_cls.return_value = inst - - resp = asyncio.run( - inf_mod._openai_passthrough_stream( - _Req(), threading.Event(), _Llama(), _make_payload() - ) - ) - chunks = _collect(resp) - - assert any('"error"' in c for c in chunks) - assert any(c.strip() == "data: [DONE]" for c in chunks) - - -def test_done_emitted_after_exception(): - from routes import inference as inf_mod - - async def _send(req, stream): - raise httpx.ConnectError("boom", request = httpx.Request("POST", "http://x")) - - with patch.object(inf_mod.httpx, "AsyncClient") as mock_cls: - inst = MagicMock() - inst.build_request = MagicMock(return_value = MagicMock()) - inst.send = _send - inst.aclose = AsyncMock() - mock_cls.return_value = inst - - resp = asyncio.run( - inf_mod._openai_passthrough_stream( - _Req(), threading.Event(), _Llama(), _make_payload() - ) - ) - chunks = _collect(resp) - - assert any('"error"' in c for c in chunks) - assert any(c.strip() == "data: [DONE]" for c in chunks) - - -def test_done_comes_after_error_chunk(): - from routes import inference as inf_mod - - async def _send(req, stream): - raise httpx.ReadError("reset", request = httpx.Request("POST", "http://x")) - - with patch.object(inf_mod.httpx, "AsyncClient") as mock_cls: - inst = MagicMock() - inst.build_request = MagicMock(return_value = MagicMock()) - inst.send = _send - inst.aclose = AsyncMock() - mock_cls.return_value = inst - - resp = asyncio.run( - inf_mod._openai_passthrough_stream( - _Req(), threading.Event(), _Llama(), _make_payload() - ) - ) - chunks = _collect(resp) - - err_idx = next(i for i, c in enumerate(chunks) if '"error"' in c) - done_idx = next(i for i, c in enumerate(chunks) if c.strip() == "data: [DONE]") - assert done_idx > err_idx diff --git a/tests/test_pr5061_r2_verbatim_stream_chunks.py b/tests/test_pr5061_r2_verbatim_stream_chunks.py deleted file mode 100644 index 5ff5cd0768..0000000000 --- a/tests/test_pr5061_r2_verbatim_stream_chunks.py +++ /dev/null @@ -1,133 +0,0 @@ -import asyncio -import os, sys, threading -from unittest.mock import patch, MagicMock - -_backend = os.path.join(os.path.dirname(__file__), "..", "studio", "backend") -sys.path.insert(0, _backend) - -from models.inference import ChatCompletionRequest - - -class _Llama: - base_url = "http://127.0.0.1:0" - _api_key = None - - -class _Req: - async def is_disconnected(self): - return False - - -def _payload(): - return ChatCompletionRequest( - messages = [{"role": "user", "content": "q"}], - tools = [ - { - "type": "function", - "function": {"name": "f", "parameters": {"type": "object"}}, - } - ], - stream = True, - ) - - -class _FakeResp: - def __init__(self, lines): - self.status_code = 200 - self._lines = list(lines) - - def aiter_lines(self): - parent = self - - class _It: - def __init__(self): - self._idx = 0 - - def __aiter__(self): - return self - - async def __anext__(self): - if self._idx >= len(parent._lines): - raise StopAsyncIteration - v = parent._lines[self._idx] - self._idx += 1 - return v - - async def aclose(self): - pass - - return _It() - - async def aread(self): - return b"" - - async def aclose(self): - pass - - -def _run_stream(fake_lines): - from routes import inference as inf_mod - - async def _send(req, stream): - return _FakeResp(fake_lines) - - async def _aclose_noop(): - return None - - with patch.object(inf_mod.httpx, "AsyncClient") as mock_cls: - inst = MagicMock() - inst.build_request = MagicMock(return_value = MagicMock()) - inst.send = _send - inst.aclose = _aclose_noop - mock_cls.return_value = inst - - resp = asyncio.run( - inf_mod._openai_passthrough_stream( - _Req(), threading.Event(), _Llama(), _payload() - ) - ) - - async def _collect(): - return [ - c if isinstance(c, str) else c.decode("utf-8") - async for c in resp.body_iterator - ] - - return asyncio.run(_collect()) - - -def test_passthrough_relays_data_lines_verbatim(): - chunks = _run_stream( - [ - 'data: {"id":"abc","choices":[{"delta":{"content":"hi"}}]}', - "data: [DONE]", - ] - ) - assert any('"id":"abc"' in c for c in chunks) - assert any("data: [DONE]" in c for c in chunks) - - -def test_passthrough_ignores_blank_and_non_data_lines(): - chunks = _run_stream( - [ - "", - ": heartbeat", - 'data: {"x":1}', - "data: [DONE]", - ] - ) - # Only data: lines propagate. - for c in chunks: - assert c.startswith("data: ") or c == "" - assert any('"x":1' in c for c in chunks) - - -def test_passthrough_breaks_on_done(): - chunks = _run_stream( - [ - 'data: {"a":1}', - "data: [DONE]", - 'data: {"should_not_appear":true}', - ] - ) - assert not any("should_not_appear" in c for c in chunks) From f7f322ff682df5b44983d159e9063098ebe1866e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 Apr 2026 18:40:57 +0000 Subject: [PATCH 12/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../tests/test_openai_tool_passthrough.py | 101 ++++++++++++------ 1 file changed, 69 insertions(+), 32 deletions(-) diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py index 17053205b3..e687aeb604 100644 --- a/studio/backend/tests/test_openai_tool_passthrough.py +++ b/studio/backend/tests/test_openai_tool_passthrough.py @@ -440,6 +440,7 @@ class TestLlamaAuthHeaders: def test_returns_bearer_header_when_api_key_set(self): class _Backend: _api_key = "k_secret_123" + assert _llama_auth_headers(_Backend()) == { "Authorization": "Bearer k_secret_123", } @@ -447,16 +448,19 @@ class TestLlamaAuthHeaders: def test_returns_none_when_api_key_none(self): class _Backend: _api_key = None + assert _llama_auth_headers(_Backend()) is None def test_returns_none_when_api_key_attribute_missing(self): class _Backend: pass + assert _llama_auth_headers(_Backend()) is None def test_returns_none_when_api_key_empty_string(self): class _Backend: _api_key = "" + assert _llama_auth_headers(_Backend()) is None @@ -586,15 +590,23 @@ async def _call_openai_chat_completions(payload, llama): with ( patch.object(inference_module, "get_llama_cpp_backend", return_value = llama), patch.object( - inference_module, "get_inference_backend", return_value = _FakeInferenceBackend(), + inference_module, + "get_inference_backend", + return_value = _FakeInferenceBackend(), ), - patch.object(inference_module, "_openai_passthrough_stream", new = _marker_stream), patch.object( - inference_module, "_openai_passthrough_non_streaming", new = _marker_nonstream, + inference_module, "_openai_passthrough_stream", new = _marker_stream + ), + patch.object( + inference_module, + "_openai_passthrough_non_streaming", + new = _marker_nonstream, ), ): return await inference_module.openai_chat_completions( - payload, _FakeFastAPIRequest(), current_subject = "u", + payload, + _FakeFastAPIRequest(), + current_subject = "u", ) @@ -617,7 +629,8 @@ class TestOpenAIChatCompletionsToolGuards: with pytest.raises(HTTPException) as exc_info: asyncio.run( _call_openai_chat_completions( - payload, _FakeLlamaBackend(is_loaded = False), + payload, + _FakeLlamaBackend(is_loaded = False), ) ) assert exc_info.value.status_code == 400 @@ -627,7 +640,8 @@ class TestOpenAIChatCompletionsToolGuards: with pytest.raises(HTTPException) as exc_info: asyncio.run( _call_openai_chat_completions( - payload, _FakeLlamaBackend(supports_tools = False), + payload, + _FakeLlamaBackend(supports_tools = False), ) ) assert exc_info.value.status_code == 400 @@ -658,7 +672,8 @@ class TestOpenAIChatCompletionsToolGuards: with pytest.raises(HTTPException) as exc_info: asyncio.run( _call_openai_chat_completions( - payload, _FakeLlamaBackend(supports_tools = False), + payload, + _FakeLlamaBackend(supports_tools = False), ) ) assert exc_info.value.status_code == 400 @@ -670,7 +685,8 @@ class TestOpenAIChatCompletionsToolGuards: try: asyncio.run( _call_openai_chat_completions( - payload, _FakeLlamaBackend(supports_tools = False), + payload, + _FakeLlamaBackend(supports_tools = False), ) ) except HTTPException as exc: @@ -709,7 +725,9 @@ class TestAnthropicEnableToolsToolChoiceConflict: with pytest.raises(HTTPException) as exc_info: asyncio.run( inference_module.anthropic_messages( - payload, _FakeFastAPIRequest(), current_subject = "u", + payload, + _FakeFastAPIRequest(), + current_subject = "u", ) ) assert exc_info.value.status_code == 400 @@ -726,7 +744,9 @@ class TestAnthropicEnableToolsToolChoiceConflict: try: asyncio.run( inference_module.anthropic_messages( - payload, _FakeFastAPIRequest(), current_subject = "u", + payload, + _FakeFastAPIRequest(), + current_subject = "u", ) ) except HTTPException as exc: @@ -832,9 +852,12 @@ class TestOpenAIPassthroughNonStreaming: client_cls = _mock_async_client_post(200, json_body = native) with patch.object(inference_module.httpx, "AsyncClient", client_cls): resp = asyncio.run( - _openai_passthrough_non_streaming(_FakeLlamaBase(), _openai_tools_payload()) + _openai_passthrough_non_streaming( + _FakeLlamaBase(), _openai_tools_payload() + ) ) import json + body = json.loads(resp.body.decode("utf-8")) assert body == native assert body["choices"][0]["finish_reason"] == "tool_calls" @@ -854,9 +877,12 @@ class TestOpenAIPassthroughNonStreaming: client_cls = _mock_async_client_post(200, json_body = native) with patch.object(inference_module.httpx, "AsyncClient", client_cls): resp = asyncio.run( - _openai_passthrough_non_streaming(_FakeLlamaBase(), _openai_tools_payload()) + _openai_passthrough_non_streaming( + _FakeLlamaBase(), _openai_tools_payload() + ) ) import json + body = json.loads(resp.body.decode("utf-8")) assert body["id"] == "chatcmpl-native-xyz" assert body["model"] == "llama-native" @@ -864,14 +890,16 @@ class TestOpenAIPassthroughNonStreaming: def test_httpx_connect_error_mapped_to_502(self): client_cls = _mock_async_client_raise( lambda: httpx.ConnectError( - "refused", request = httpx.Request("POST", "http://x"), + "refused", + request = httpx.Request("POST", "http://x"), ) ) with patch.object(inference_module.httpx, "AsyncClient", client_cls): with pytest.raises(HTTPException) as exc_info: asyncio.run( _openai_passthrough_non_streaming( - _FakeLlamaBase(), _openai_tools_payload(), + _FakeLlamaBase(), + _openai_tools_payload(), ) ) assert exc_info.value.status_code == 502 @@ -885,7 +913,8 @@ class TestOpenAIPassthroughNonStreaming: with pytest.raises(HTTPException) as exc_info: asyncio.run( _openai_passthrough_non_streaming( - _FakeLlamaBase(), _openai_tools_payload(), + _FakeLlamaBase(), + _openai_tools_payload(), ) ) assert exc_info.value.status_code == 502 @@ -969,22 +998,26 @@ def _run_passthrough_stream(*, send_returns = None, send_raises = None): class TestOpenAIPassthroughStreamVerbatim: def test_relays_data_lines_verbatim(self): chunks = _run_passthrough_stream( - send_returns = _FakeStreamResponse([ - 'data: {"id":"abc","choices":[{"delta":{"content":"hi"}}]}', - "data: [DONE]", - ]), + send_returns = _FakeStreamResponse( + [ + 'data: {"id":"abc","choices":[{"delta":{"content":"hi"}}]}', + "data: [DONE]", + ] + ), ) assert any('"id":"abc"' in c for c in chunks) assert any("data: [DONE]" in c for c in chunks) def test_ignores_blank_and_non_data_lines(self): chunks = _run_passthrough_stream( - send_returns = _FakeStreamResponse([ - "", - ": heartbeat", - 'data: {"x":1}', - "data: [DONE]", - ]), + send_returns = _FakeStreamResponse( + [ + "", + ": heartbeat", + 'data: {"x":1}', + "data: [DONE]", + ] + ), ) for chunk in chunks: assert chunk.startswith("data: ") or chunk == "" @@ -992,11 +1025,13 @@ class TestOpenAIPassthroughStreamVerbatim: def test_breaks_on_done(self): chunks = _run_passthrough_stream( - send_returns = _FakeStreamResponse([ - 'data: {"a":1}', - "data: [DONE]", - 'data: {"should_not_appear":true}', - ]), + send_returns = _FakeStreamResponse( + [ + 'data: {"a":1}', + "data: [DONE]", + 'data: {"should_not_appear":true}', + ] + ), ) assert not any("should_not_appear" in c for c in chunks) @@ -1023,7 +1058,8 @@ class TestOpenAIPassthroughStreamErrorTermination: def test_done_emitted_after_transport_exception(self): chunks = _run_passthrough_stream( send_raises = lambda: httpx.ConnectError( - "boom", request = httpx.Request("POST", "http://x"), + "boom", + request = httpx.Request("POST", "http://x"), ), ) assert any('"error"' in c for c in chunks) @@ -1032,7 +1068,8 @@ class TestOpenAIPassthroughStreamErrorTermination: def test_done_comes_after_error_chunk(self): chunks = _run_passthrough_stream( send_raises = lambda: httpx.ReadError( - "reset", request = httpx.Request("POST", "http://x"), + "reset", + request = httpx.Request("POST", "http://x"), ), ) err_idx = next(i for i, c in enumerate(chunks) if '"error"' in c)