diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index 04c63c2247..4e6a5937d8 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -416,7 +416,7 @@ class ImageUrl(BaseModel): """Image URL object — supports data URIs and remote URLs.""" url: str = Field(..., description = "data:image/png;base64,... or https://...") - detail: Optional[Literal["auto", "low", "high"]] = "auto" + detail: Optional[Literal["auto", "low", "high", "original"]] = "auto" class ImageContentPart(BaseModel): @@ -1125,7 +1125,7 @@ class ResponsesInputImagePart(BaseModel): type: Literal["input_image"] image_url: str = Field(..., description = "data:image/png;base64,... or https://...") - detail: Optional[Literal["auto", "low", "high"]] = "auto" + detail: Optional[Literal["auto", "low", "high", "original"]] = "auto" class ResponsesOutputTextPart(BaseModel): diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index e9353cd803..2f9ed35936 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -5309,15 +5309,72 @@ def _responses_message_text(content: Union[str, list]) -> str: return "\n".join(parts) -def _responses_tool_output_text(output: Union[str, list]) -> str: +def _responses_tool_output_content(output: Union[str, list]) -> Union[str, list]: """Return Chat Completions-safe content for a Responses tool result.""" if isinstance(output, str): return output if output.strip() else "(no output)" - if output: + if not output: + return "(no output)" + + text_parts: list[str] = [] + chat_parts: list = [] + has_multimodal = False + for part in output: + if not isinstance(part, dict): + return json.dumps(output) + part_type = part.get("type") + if part_type in ("input_text", "output_text", "text"): + text = part.get("text") + if text is None: + _raise_unsupported_openai_parameter( + "input", + "Responses function_call_output.output text parts require a text field.", + ) + text = str(text) + text_parts.append(text) + chat_parts.append(TextContentPart(type = "text", text = text)) + continue + if part_type == "input_image": + image_url = part.get("image_url") + if not isinstance(image_url, str) or not image_url: + if part.get("file_id"): + _raise_unsupported_openai_parameter( + "input", + "Responses function_call_output.output input_image parts with file_id are not supported by the local adapter. Use image_url instead.", + ) + _raise_unsupported_openai_parameter( + "input", + "Responses function_call_output.output input_image parts require an image_url string.", + ) + detail = part.get("detail", "auto") + if detail is None: + detail = "auto" + if detail not in ("auto", "low", "high", "original"): + _raise_unsupported_openai_parameter( + "input", + "Responses function_call_output.output input_image detail must be auto, low, high, or original.", + ) + chat_parts.append( + ImageContentPart( + type = "image_url", + image_url = ImageUrl(url = image_url, detail = detail), + ) + ) + has_multimodal = True + continue + if part_type == "input_file": + _raise_unsupported_openai_parameter( + "input", + "Responses function_call_output.output input_file parts are not supported by the local adapter.", + ) return json.dumps(output) - return "(no output)" + if has_multimodal: + return chat_parts + + text = "\n".join(text_parts) + return text if text.strip() else "(no output)" _RESPONSES_THINK_OPEN = "" @@ -5521,10 +5578,9 @@ def _normalise_responses_input(payload: ResponsesRequest) -> list[ChatMessage]: continue if isinstance(item, ResponsesFunctionCallOutputInputItem): - # Chat Completions `role="tool"` requires string content; serialize - # a Responses content-array output and keep empty outputs from - # tripping the stricter ChatMessage role validator. - output = _responses_tool_output_text(item.output) + # Flatten pure text arrays for broad template compatibility, and + # forward image URL outputs as real multimodal parts for vision models. + output = _responses_tool_output_content(item.output) messages.append( ChatMessage( role = "tool", diff --git a/studio/backend/tests/test_responses_tool_passthrough.py b/studio/backend/tests/test_responses_tool_passthrough.py index ae7ff729bd..89155c2daf 100644 --- a/studio/backend/tests/test_responses_tool_passthrough.py +++ b/studio/backend/tests/test_responses_tool_passthrough.py @@ -36,6 +36,7 @@ import json import httpx import pytest +from fastapi import HTTPException from fastapi.responses import JSONResponse from pydantic import ValidationError @@ -60,7 +61,7 @@ from routes.inference import ( _build_chat_request, _chat_tool_calls_to_responses_output, _normalise_responses_input, - _responses_tool_output_text, + _responses_tool_output_content, _responses_non_streaming, _responses_stream, _translate_responses_tool_choice_to_chat, @@ -435,20 +436,144 @@ class TestNormaliseResponsesInputWithTools: assert sum(1 for m in msgs if m.role == "system") == 1 assert "A" in msgs[0].content and "B" in msgs[0].content - def test_content_array_output_serialised_to_json_string(self): + def test_content_array_text_output_flattens_to_tool_text(self): payload = ResponsesRequest( input = [ { "type": "function_call_output", "call_id": "call_1", - "output": [{"type": "output_text", "text": "ok"}], + "output": [{"type": "input_text", "text": "ok"}], } ], ) msgs = _normalise_responses_input(payload) assert msgs[0].role == "tool" - # Content is serialised so llama-server sees a string. - assert json.loads(msgs[0].content) == [{"type": "output_text", "text": "ok"}] + assert msgs[0].content == "ok" + + def test_content_array_image_output_becomes_multimodal_tool_content(self): + payload = ResponsesRequest( + input = [ + { + "type": "function_call_output", + "call_id": "call_1", + "output": [ + {"type": "input_text", "text": "see image"}, + { + "type": "input_image", + "image_url": "data:image/png;base64,AAA", + "detail": "high", + }, + ], + } + ], + ) + msgs = _normalise_responses_input(payload) + assert msgs[0].role == "tool" + assert msgs[0].tool_call_id == "call_1" + assert msgs[0].model_dump(exclude_none = True)["content"] == [ + {"type": "text", "text": "see image"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,AAA", + "detail": "high", + }, + }, + ] + + chat_req = _build_chat_request(payload, msgs, stream = False) + assert chat_req.model_dump(exclude_none = True)["messages"][0]["content"] == [ + {"type": "text", "text": "see image"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,AAA", + "detail": "high", + }, + }, + ] + + def test_content_array_image_output_allows_original_detail(self): + payload = ResponsesRequest( + input = [ + { + "type": "function_call_output", + "call_id": "call_1", + "output": [ + { + "type": "input_image", + "image_url": "https://example.com/screenshot.png", + "detail": "original", + }, + ], + } + ], + ) + msgs = _normalise_responses_input(payload) + assert msgs[0].model_dump(exclude_none = True)["content"] == [ + { + "type": "image_url", + "image_url": { + "url": "https://example.com/screenshot.png", + "detail": "original", + }, + }, + ] + + def test_content_array_file_id_image_output_rejected_clearly(self): + payload = ResponsesRequest( + input = [ + { + "type": "function_call_output", + "call_id": "call_1", + "output": [ + {"type": "input_text", "text": "see image"}, + {"type": "input_image", "file_id": "file_abc"}, + ], + } + ], + ) + with pytest.raises(HTTPException) as exc: + _normalise_responses_input(payload) + assert exc.value.status_code == 400 + assert "file_id" in str(exc.value.detail) + + def test_content_array_file_output_rejected_clearly(self): + payload = ResponsesRequest( + input = [ + { + "type": "function_call_output", + "call_id": "call_1", + "output": [ + {"type": "input_text", "text": "see file"}, + { + "type": "input_file", + "file_data": "data:application/pdf;base64,AAA", + "filename": "report.pdf", + }, + ], + } + ], + ) + with pytest.raises(HTTPException) as exc: + _normalise_responses_input(payload) + assert exc.value.status_code == 400 + assert "input_file" in str(exc.value.detail) + + def test_content_array_malformed_image_output_rejected_clearly(self): + payload = ResponsesRequest( + input = [ + { + "type": "function_call_output", + "call_id": "call_1", + "output": [{"type": "input_image", "detail": "high"}], + } + ], + ) + with pytest.raises(HTTPException) as exc: + _normalise_responses_input(payload) + assert exc.value.status_code == 400 + assert "image_url" in str(exc.value.detail) def test_empty_function_call_output_gets_no_output_sentinel(self): payload = ResponsesRequest( @@ -541,8 +666,8 @@ class TestNormaliseResponsesInputWithTools: assert msgs[0].content == "(no output)" def test_tool_output_serializer_preserves_non_empty_text(self): - assert _responses_tool_output_text("done") == "done" - assert _responses_tool_output_text(" done ") == " done " + assert _responses_tool_output_content("done") == "done" + assert _responses_tool_output_content(" done ") == " done " # =====================================================================