ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix Responses tool output content arrays (#6287)

Browse source 
* Fix Responses tool output content arrays

* Fix/adjust Responses tool output content for PR #6287

* Fix/adjust original image detail for PR #6287

---------

Co-authored-by: Lee Jackson <130007945+Imagineer99@users.noreply.github.com>
Co-authored-by: wasimysaid <wasimysdev@gmail.com>
This commit is contained in:
alkinun 2026-06-13 12:23:34 +03:00 • committed by GitHub
commit 8e57c5ee34
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 197 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

4
studio/backend/models/inference.py
View file

@ -416,7 +416,7 @@ class ImageUrl(BaseModel):
"""Image URL object — supports data URIs and remote URLs."""
url: str = Field(..., description = "data:image/png;base64,... or https://...")
detail: Optional[Literal["auto", "low", "high"]] = "auto"
detail: Optional[Literal["auto", "low", "high", "original"]] = "auto"
class ImageContentPart(BaseModel):
@ -1125,7 +1125,7 @@ class ResponsesInputImagePart(BaseModel):
type: Literal["input_image"]
image_url: str = Field(..., description = "data:image/png;base64,... or https://...")
detail: Optional[Literal["auto", "low", "high"]] = "auto"
detail: Optional[Literal["auto", "low", "high", "original"]] = "auto"
class ResponsesOutputTextPart(BaseModel):

70
studio/backend/routes/inference.py
View file

@ -5309,15 +5309,72 @@ def _responses_message_text(content: Union[str, list]) -> str:
return "\n".join(parts)
def _responses_tool_output_text(output: Union[str, list]) -> str:
def _responses_tool_output_content(output: Union[str, list]) -> Union[str, list]:
"""Return Chat Completions-safe content for a Responses tool result."""
if isinstance(output, str):
return output if output.strip() else "(no output)"
if output:
if not output:
return "(no output)"
text_parts: list[str] = []
chat_parts: list = []
has_multimodal = False
for part in output:
if not isinstance(part, dict):
return json.dumps(output)
part_type = part.get("type")
if part_type in ("input_text", "output_text", "text"):
text = part.get("text")
if text is None:
_raise_unsupported_openai_parameter(
"input",
"Responses function_call_output.output text parts require a text field.",
)
text = str(text)
text_parts.append(text)
chat_parts.append(TextContentPart(type = "text", text = text))
continue
if part_type == "input_image":
image_url = part.get("image_url")
if not isinstance(image_url, str) or not image_url:
if part.get("file_id"):
_raise_unsupported_openai_parameter(
"input",
"Responses function_call_output.output input_image parts with file_id are not supported by the local adapter. Use image_url instead.",
)
_raise_unsupported_openai_parameter(
"input",
"Responses function_call_output.output input_image parts require an image_url string.",
)
detail = part.get("detail", "auto")
if detail is None:
detail = "auto"
if detail not in ("auto", "low", "high", "original"):
_raise_unsupported_openai_parameter(
"input",
"Responses function_call_output.output input_image detail must be auto, low, high, or original.",
)
chat_parts.append(
ImageContentPart(
type = "image_url",
image_url = ImageUrl(url = image_url, detail = detail),
)
)
has_multimodal = True
continue
if part_type == "input_file":
_raise_unsupported_openai_parameter(
"input",
"Responses function_call_output.output input_file parts are not supported by the local adapter.",
)
return json.dumps(output)
return "(no output)"
if has_multimodal:
return chat_parts
text = "\n".join(text_parts)
return text if text.strip() else "(no output)"
_RESPONSES_THINK_OPEN = "<think>"
@ -5521,10 +5578,9 @@ def _normalise_responses_input(payload: ResponsesRequest) -> list[ChatMessage]:
continue
if isinstance(item, ResponsesFunctionCallOutputInputItem):
# Chat Completions `role="tool"` requires string content; serialize
# a Responses content-array output and keep empty outputs from
# tripping the stricter ChatMessage role validator.
output = _responses_tool_output_text(item.output)
# Flatten pure text arrays for broad template compatibility, and
# forward image URL outputs as real multimodal parts for vision models.
output = _responses_tool_output_content(item.output)
messages.append(
ChatMessage(
role = "tool",

139
studio/backend/tests/test_responses_tool_passthrough.py
View file

@ -36,6 +36,7 @@ import json
import httpx
import pytest
from fastapi import HTTPException
from fastapi.responses import JSONResponse
from pydantic import ValidationError
@ -60,7 +61,7 @@ from routes.inference import (
_build_chat_request,
_chat_tool_calls_to_responses_output,
_normalise_responses_input,
_responses_tool_output_text,
_responses_tool_output_content,
_responses_non_streaming,
_responses_stream,
_translate_responses_tool_choice_to_chat,
@ -435,20 +436,144 @@ class TestNormaliseResponsesInputWithTools:
assert sum(1 for m in msgs if m.role == "system") == 1
assert "A" in msgs[0].content and "B" in msgs[0].content
def test_content_array_output_serialised_to_json_string(self):
def test_content_array_text_output_flattens_to_tool_text(self):
payload = ResponsesRequest(
input = [
{
"type": "function_call_output",
"call_id": "call_1",
"output": [{"type": "output_text", "text": "ok"}],
"output": [{"type": "input_text", "text": "ok"}],
}
],
)
msgs = _normalise_responses_input(payload)
assert msgs[0].role == "tool"
# Content is serialised so llama-server sees a string.
assert json.loads(msgs[0].content) == [{"type": "output_text", "text": "ok"}]
assert msgs[0].content == "ok"
def test_content_array_image_output_becomes_multimodal_tool_content(self):
payload = ResponsesRequest(
input = [
{
"type": "function_call_output",
"call_id": "call_1",
"output": [
{"type": "input_text", "text": "see image"},
{
"type": "input_image",
"image_url": "data:image/png;base64,AAA",
"detail": "high",
},
],
}
],
)
msgs = _normalise_responses_input(payload)
assert msgs[0].role == "tool"
assert msgs[0].tool_call_id == "call_1"
assert msgs[0].model_dump(exclude_none = True)["content"] == [
{"type": "text", "text": "see image"},
{
"type": "image_url",
"image_url": {
"url": "data:image/png;base64,AAA",
"detail": "high",
},
},
]
chat_req = _build_chat_request(payload, msgs, stream = False)
assert chat_req.model_dump(exclude_none = True)["messages"][0]["content"] == [
{"type": "text", "text": "see image"},
{
"type": "image_url",
"image_url": {
"url": "data:image/png;base64,AAA",
"detail": "high",
},
},
]
def test_content_array_image_output_allows_original_detail(self):
payload = ResponsesRequest(
input = [
{
"type": "function_call_output",
"call_id": "call_1",
"output": [
{
"type": "input_image",
"image_url": "https://example.com/screenshot.png",
"detail": "original",
},
],
}
],
)
msgs = _normalise_responses_input(payload)
assert msgs[0].model_dump(exclude_none = True)["content"] == [
{
"type": "image_url",
"image_url": {
"url": "https://example.com/screenshot.png",
"detail": "original",
},
},
]
def test_content_array_file_id_image_output_rejected_clearly(self):
payload = ResponsesRequest(
input = [
{
"type": "function_call_output",
"call_id": "call_1",
"output": [
{"type": "input_text", "text": "see image"},
{"type": "input_image", "file_id": "file_abc"},
],
}
],
)
with pytest.raises(HTTPException) as exc:
_normalise_responses_input(payload)
assert exc.value.status_code == 400
assert "file_id" in str(exc.value.detail)
def test_content_array_file_output_rejected_clearly(self):
payload = ResponsesRequest(
input = [
{
"type": "function_call_output",
"call_id": "call_1",
"output": [
{"type": "input_text", "text": "see file"},
{
"type": "input_file",
"file_data": "data:application/pdf;base64,AAA",
"filename": "report.pdf",
},
],
}
],
)
with pytest.raises(HTTPException) as exc:
_normalise_responses_input(payload)
assert exc.value.status_code == 400
assert "input_file" in str(exc.value.detail)
def test_content_array_malformed_image_output_rejected_clearly(self):
payload = ResponsesRequest(
input = [
{
"type": "function_call_output",
"call_id": "call_1",
"output": [{"type": "input_image", "detail": "high"}],
}
],
)
with pytest.raises(HTTPException) as exc:
_normalise_responses_input(payload)
assert exc.value.status_code == 400
assert "image_url" in str(exc.value.detail)
def test_empty_function_call_output_gets_no_output_sentinel(self):
payload = ResponsesRequest(
@ -541,8 +666,8 @@ class TestNormaliseResponsesInputWithTools:
assert msgs[0].content == "(no output)"
def test_tool_output_serializer_preserves_non_empty_text(self):
assert _responses_tool_output_text("done") == "done"
assert _responses_tool_output_text(" done ") == " done "
assert _responses_tool_output_content("done") == "done"
assert _responses_tool_output_content(" done ") == " done "
# =====================================================================