Studio: improve OpenAI- and Anthropic-compatible API spec compliance (#6010)
* Studio: fix OpenAI- and Anthropic-compatible API spec compliance * Studio: fix API spec-compliance gaps on passthrough and streaming paths * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: carry context_length_exceeded through the OpenAI passthrough error path * Studio: count tool-schema tokens in the Anthropic server-tool stream, and small stream-handling guards * Studio: guard message_delta usage against None and normalize developer role before proxying * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: honor max_completion_tokens on the external-provider proxy path * Studio: forward llama-server cached_tokens into OpenAI prompt_tokens_details * Studio: sanitize messages in count_tokens to match the /v1/messages prompt * Studio: report max_tokens for truncated tool calls and guard null usage in metadata events * Studio: drop the request-id middleware (headers aren't declared in either spec) * Studio: include the required request_id field in Anthropic error bodies * Studio: honor max_completion_tokens on the audio (TTS / audio-input) paths * Studio: add the _effective_max_tokens helper and route all max-token sites through it * Studio: align API compatibility edge cases * Studio: clarify multi-choice chat support * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: clarify logprobs chat support * Studio: opt the local chat UI into the streaming usage chunk so the context bar and tok/s repopulate * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: forward seed to llama-server, and fix Anthropic server-tool stop_reason, tool_result id correlation, and parallel-tool execution cap * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: align OpenAI chat completion spec edge cases * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: align backend API compatibility tests * Studio: honor tool caps and internal stream usage * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: coerce nullable stream usage counts * Studio: preserve system prompts with developer messages --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: wasimysaid <wasimysdev@gmail.com>
This commit is contained in:
oobabooga
GitHub
parent
a01960f20f
commit
57be5868f9
13 changed files with 2175 additions and 294 deletions
|
|
@ -10,9 +10,37 @@ Pure functions plus stateful stream emitters; no FastAPI, no I/O.
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
|
||||
def openai_finish_to_anthropic_stop(finish_reason, had_tool_calls = False) -> str:
|
||||
"""Map an OpenAI finish_reason to an Anthropic stop_reason.
|
||||
'length' -> 'max_tokens' (truncation wins even mid tool call, so a cut-off
|
||||
tool call isn't mislabeled tool_use); tool_calls / had_tool_calls -> 'tool_use';
|
||||
'stop_sequence' -> 'stop_sequence'; 'stop'/None/unknown -> 'end_turn'."""
|
||||
# Truncation takes precedence: a tool call cut off at max_tokens has possibly
|
||||
# incomplete arguments, so report max_tokens rather than telling the client to
|
||||
# run the tool.
|
||||
if finish_reason == "length":
|
||||
return "max_tokens"
|
||||
if finish_reason == "tool_calls" or had_tool_calls:
|
||||
return "tool_use"
|
||||
if finish_reason == "stop_sequence":
|
||||
return "stop_sequence"
|
||||
# "stop", None, and any unknown value collapse to end_turn.
|
||||
return "end_turn"
|
||||
|
||||
|
||||
def anthropic_tool_use_id(upstream_id = None) -> str:
|
||||
"""Return an Anthropic-style tool_use id (prefix 'toolu_'). Reuses an
|
||||
upstream id only if it already starts with 'toolu_'; otherwise mints a fresh
|
||||
'toolu_<24 hex>'."""
|
||||
if upstream_id and isinstance(upstream_id, str) and upstream_id.startswith("toolu_"):
|
||||
return upstream_id
|
||||
return f"toolu_{uuid.uuid4().hex[:24]}"
|
||||
|
||||
|
||||
def _anthropic_image_block_to_openai_part(block: dict) -> Optional[dict]:
|
||||
"""Translate one Anthropic ``image`` block to an OpenAI ``image_url`` part.
|
||||
|
||||
|
|
@ -204,6 +232,19 @@ def build_anthropic_sse_event(event_type: str, data: dict) -> str:
|
|||
return f"event: {event_type}\ndata: {json.dumps(data)}\n\n"
|
||||
|
||||
|
||||
def _message_delta_usage(usage: Optional[dict]) -> dict:
|
||||
"""Usage block for a message_delta event (cumulative token counts). Cache
|
||||
fields are always 0 — no prompt caching backend. ``usage`` may be None when a
|
||||
metadata event carried usage=None (e.g. only finish_reason set)."""
|
||||
usage = usage or {}
|
||||
return {
|
||||
"input_tokens": usage.get("prompt_tokens", 0),
|
||||
"cache_creation_input_tokens": 0,
|
||||
"cache_read_input_tokens": 0,
|
||||
"output_tokens": usage.get("completion_tokens", 0),
|
||||
}
|
||||
|
||||
|
||||
class AnthropicStreamEmitter:
|
||||
"""Converts generate_chat_completion_with_tools() events into Anthropic
|
||||
Messages SSE strings."""
|
||||
|
|
@ -212,11 +253,19 @@ class AnthropicStreamEmitter:
|
|||
self.block_index: int = 0
|
||||
self._text_block_open: bool = False
|
||||
self._open_tool_call_id: Optional[str] = None
|
||||
# The mapped Anthropic ``toolu_*`` id published in content_block_start,
|
||||
# reused for the paired tool_result so consumers can correlate them.
|
||||
self._open_tool_use_id: Optional[str] = None
|
||||
self._open_tool_args_sent: bool = False
|
||||
self._prev_text: str = ""
|
||||
self._usage: dict = {}
|
||||
|
||||
def start(self, message_id: str, model: str) -> list[str]:
|
||||
def start(
|
||||
self,
|
||||
message_id: str,
|
||||
model: str,
|
||||
input_tokens: int = 0,
|
||||
) -> list[str]:
|
||||
"""Emit message_start and open the first text content block."""
|
||||
events = []
|
||||
events.append(
|
||||
|
|
@ -232,7 +281,12 @@ class AnthropicStreamEmitter:
|
|||
"model": model,
|
||||
"stop_reason": None,
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 0, "output_tokens": 0},
|
||||
"usage": {
|
||||
"input_tokens": input_tokens,
|
||||
"output_tokens": 0,
|
||||
"cache_creation_input_tokens": 0,
|
||||
"cache_read_input_tokens": 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
|
@ -255,22 +309,28 @@ class AnthropicStreamEmitter:
|
|||
# status events — no Anthropic equivalent
|
||||
return []
|
||||
|
||||
def finish(self, stop_reason: str = "end_turn") -> list[str]:
|
||||
def finish(
|
||||
self,
|
||||
stop_reason: str = "end_turn",
|
||||
stop_sequence = None,
|
||||
) -> list[str]:
|
||||
"""Close any open block and emit message_delta + message_stop."""
|
||||
events = []
|
||||
if self._text_block_open or self._open_tool_call_id is not None:
|
||||
events.append(self._close_block())
|
||||
self._open_tool_call_id = None
|
||||
self._open_tool_use_id = None
|
||||
self._open_tool_args_sent = False
|
||||
events.append(
|
||||
build_anthropic_sse_event(
|
||||
"message_delta",
|
||||
{
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": stop_reason, "stop_sequence": None},
|
||||
"usage": {
|
||||
"output_tokens": self._usage.get("completion_tokens", 0),
|
||||
"delta": {
|
||||
"stop_reason": stop_reason,
|
||||
"stop_sequence": stop_sequence,
|
||||
},
|
||||
"usage": _message_delta_usage(self._usage),
|
||||
},
|
||||
)
|
||||
)
|
||||
|
|
@ -319,11 +379,13 @@ class AnthropicStreamEmitter:
|
|||
elif self._open_tool_call_id is not None:
|
||||
events.append(self._close_block())
|
||||
self._open_tool_call_id = None
|
||||
self._open_tool_use_id = None
|
||||
self._open_tool_args_sent = False
|
||||
|
||||
# Open a tool_use block.
|
||||
self.block_index += 1
|
||||
self._open_tool_call_id = tool_call_id
|
||||
self._open_tool_use_id = anthropic_tool_use_id(tool_call_id)
|
||||
self._open_tool_args_sent = False
|
||||
events.append(
|
||||
build_anthropic_sse_event(
|
||||
|
|
@ -333,7 +395,7 @@ class AnthropicStreamEmitter:
|
|||
"index": self.block_index,
|
||||
"content_block": {
|
||||
"type": "tool_use",
|
||||
"id": tool_call_id,
|
||||
"id": self._open_tool_use_id,
|
||||
"name": event.get("tool_name", ""),
|
||||
"input": {},
|
||||
},
|
||||
|
|
@ -368,7 +430,11 @@ class AnthropicStreamEmitter:
|
|||
# Close the tool_use block.
|
||||
if self._open_tool_call_id is not None or self._text_block_open:
|
||||
events.append(self._close_block())
|
||||
# Reuse the id published in content_block_start; fall back to mapping
|
||||
# the raw id only if no tool_start preceded this end.
|
||||
tool_use_id = self._open_tool_use_id or anthropic_tool_use_id(event.get("tool_call_id", ""))
|
||||
self._open_tool_call_id = None
|
||||
self._open_tool_use_id = None
|
||||
self._open_tool_args_sent = False
|
||||
# Emit custom tool_result event (non-standard, ignored by SDKs)
|
||||
events.append(
|
||||
|
|
@ -376,7 +442,7 @@ class AnthropicStreamEmitter:
|
|||
"tool_result",
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": event.get("tool_call_id", ""),
|
||||
"tool_use_id": tool_use_id,
|
||||
"content": event.get("result", ""),
|
||||
},
|
||||
)
|
||||
|
|
@ -427,8 +493,14 @@ class AnthropicPassthroughEmitter:
|
|||
self._tool_call_states: dict = {} # delta index -> {block_index, id, name}
|
||||
self._usage: dict = {}
|
||||
self._stop_reason: str = "end_turn"
|
||||
self._stop_sequence: Optional[str] = None
|
||||
|
||||
def start(self, message_id: str, model: str) -> list[str]:
|
||||
def start(
|
||||
self,
|
||||
message_id: str,
|
||||
model: str,
|
||||
input_tokens: int = 0,
|
||||
) -> list[str]:
|
||||
return [
|
||||
build_anthropic_sse_event(
|
||||
"message_start",
|
||||
|
|
@ -442,7 +514,12 @@ class AnthropicPassthroughEmitter:
|
|||
"model": model,
|
||||
"stop_reason": None,
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 0, "output_tokens": 0},
|
||||
"usage": {
|
||||
"input_tokens": input_tokens,
|
||||
"output_tokens": 0,
|
||||
"cache_creation_input_tokens": 0,
|
||||
"cache_read_input_tokens": 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
|
@ -492,7 +569,7 @@ class AnthropicPassthroughEmitter:
|
|||
# New tool call — close prior block, open tool_use block
|
||||
if self._current_block_type is not None:
|
||||
events.append(self._close_current_block())
|
||||
tc_id = tc.get("id", "")
|
||||
tc_id = anthropic_tool_use_id(tc.get("id", ""))
|
||||
tc_name = fn.get("name", "")
|
||||
self.block_index += 1
|
||||
self._current_block_type = "tool_use"
|
||||
|
|
@ -535,12 +612,7 @@ class AnthropicPassthroughEmitter:
|
|||
|
||||
# ── Finish reason ──
|
||||
if finish_reason:
|
||||
if finish_reason == "tool_calls":
|
||||
self._stop_reason = "tool_use"
|
||||
elif finish_reason == "length":
|
||||
self._stop_reason = "max_tokens"
|
||||
else:
|
||||
self._stop_reason = "end_turn"
|
||||
self._stop_reason = openai_finish_to_anthropic_stop(finish_reason)
|
||||
|
||||
return events
|
||||
|
||||
|
|
@ -555,11 +627,9 @@ class AnthropicPassthroughEmitter:
|
|||
"type": "message_delta",
|
||||
"delta": {
|
||||
"stop_reason": self._stop_reason,
|
||||
"stop_sequence": None,
|
||||
},
|
||||
"usage": {
|
||||
"output_tokens": self._usage.get("completion_tokens", 0),
|
||||
"stop_sequence": self._stop_sequence,
|
||||
},
|
||||
"usage": _message_delta_usage(self._usage),
|
||||
},
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -4116,6 +4116,7 @@ class LlamaCppBackend:
|
|||
enable_thinking: Optional[bool] = None,
|
||||
reasoning_effort: Optional[str] = None,
|
||||
preserve_thinking: Optional[bool] = None,
|
||||
seed: Optional[int] = None,
|
||||
) -> Generator[str | dict, None, None]:
|
||||
"""
|
||||
Send a chat completion to llama-server and stream tokens back.
|
||||
|
|
@ -4156,6 +4157,8 @@ class LlamaCppBackend:
|
|||
payload["t_max_predict_ms"] = _DEFAULT_T_MAX_PREDICT_MS
|
||||
if stop:
|
||||
payload["stop"] = stop
|
||||
if seed is not None:
|
||||
payload["seed"] = seed
|
||||
payload["stream_options"] = {"include_usage": True}
|
||||
|
||||
url = f"{self.base_url}/v1/chat/completions"
|
||||
|
|
@ -4164,6 +4167,7 @@ class LlamaCppBackend:
|
|||
_stream_done = False
|
||||
_metadata_usage = None
|
||||
_metadata_timings = None
|
||||
_metadata_finish_reason = None
|
||||
|
||||
try:
|
||||
# _stream_with_retry uses a 120 s read timeout so prefill can
|
||||
|
|
@ -4228,6 +4232,9 @@ class LlamaCppBackend:
|
|||
choices = data.get("choices", [])
|
||||
if choices:
|
||||
delta = choices[0].get("delta", {})
|
||||
_fr = choices[0].get("finish_reason")
|
||||
if _fr:
|
||||
_metadata_finish_reason = _fr
|
||||
|
||||
# Reasoning/thinking tokens: llama-server
|
||||
# sends these as "reasoning_content"; wrap
|
||||
|
|
@ -4253,14 +4260,18 @@ class LlamaCppBackend:
|
|||
logger.debug(f"Skipping malformed SSE line: {line[:100]}")
|
||||
if _stream_done:
|
||||
break # exit outer for
|
||||
if _metadata_usage or _metadata_timings:
|
||||
if _metadata_usage or _metadata_timings or _metadata_finish_reason:
|
||||
_metadata_usage = _backfill_usage_from_timings(
|
||||
_metadata_usage, _metadata_timings
|
||||
)
|
||||
yield {
|
||||
"type": "metadata",
|
||||
"usage": _metadata_usage,
|
||||
# Never None: a finish-only metadata event (no usage,
|
||||
# no timings) would otherwise crash consumers that do
|
||||
# usage.get(...) on the non-streaming paths.
|
||||
"usage": _metadata_usage or {},
|
||||
"timings": _metadata_timings,
|
||||
"finish_reason": _metadata_finish_reason,
|
||||
}
|
||||
|
||||
except httpx.ConnectError:
|
||||
|
|
@ -4292,6 +4303,8 @@ class LlamaCppBackend:
|
|||
auto_heal_tool_calls: bool = True,
|
||||
tool_call_timeout: int = 300,
|
||||
session_id: Optional[str] = None,
|
||||
seed: Optional[int] = None,
|
||||
disable_parallel_tool_use: bool = False,
|
||||
) -> Generator[dict, None, None]:
|
||||
"""
|
||||
Agentic loop: let the model call tools, execute them, and continue.
|
||||
|
|
@ -4394,6 +4407,8 @@ class LlamaCppBackend:
|
|||
payload["t_max_predict_ms"] = _DEFAULT_T_MAX_PREDICT_MS
|
||||
if stop:
|
||||
payload["stop"] = stop
|
||||
if seed is not None:
|
||||
payload["seed"] = seed
|
||||
|
||||
try:
|
||||
_auth_headers = (
|
||||
|
|
@ -4419,6 +4434,7 @@ class LlamaCppBackend:
|
|||
has_structured_tc = False
|
||||
_iter_usage = None
|
||||
_iter_timings = None
|
||||
_iter_finish_reason = None
|
||||
_stream_done = False
|
||||
_last_emitted = ""
|
||||
provisional_render_html_tool_call_ids = set()
|
||||
|
|
@ -4498,6 +4514,9 @@ class LlamaCppBackend:
|
|||
continue
|
||||
|
||||
delta = choices[0].get("delta", {})
|
||||
_fr = choices[0].get("finish_reason")
|
||||
if _fr:
|
||||
_iter_finish_reason = _fr
|
||||
|
||||
# ── Structured tool_calls ──
|
||||
tc_deltas = delta.get("tool_calls")
|
||||
|
|
@ -4840,6 +4859,7 @@ class LlamaCppBackend:
|
|||
"total_tokens": _fp + _tc,
|
||||
},
|
||||
"timings": _mt,
|
||||
"finish_reason": _iter_finish_reason,
|
||||
}
|
||||
return
|
||||
|
||||
|
|
@ -4915,6 +4935,7 @@ class LlamaCppBackend:
|
|||
"total_tokens": _fp + _tc,
|
||||
},
|
||||
"timings": _mt,
|
||||
"finish_reason": _iter_finish_reason,
|
||||
}
|
||||
return
|
||||
|
||||
|
|
@ -4926,6 +4947,12 @@ class LlamaCppBackend:
|
|||
_accumulated_predicted_ms += _it.get("predicted_ms", 0)
|
||||
_accumulated_predicted_n += _it.get("predicted_n", 0)
|
||||
|
||||
# disable_parallel_tool_use: execute only the first tool call
|
||||
# this turn. Truncate before building assistant_msg so the
|
||||
# conversation stays consistent and extra calls are never executed.
|
||||
if disable_parallel_tool_use and tool_calls and len(tool_calls) > 1:
|
||||
tool_calls = tool_calls[:1]
|
||||
|
||||
assistant_msg: dict = {"role": "assistant", "content": content_text}
|
||||
assistant_appended = False
|
||||
|
||||
|
|
@ -5040,6 +5067,8 @@ class LlamaCppBackend:
|
|||
stream_payload["t_max_predict_ms"] = _DEFAULT_T_MAX_PREDICT_MS
|
||||
if stop:
|
||||
stream_payload["stop"] = stop
|
||||
if seed is not None:
|
||||
stream_payload["seed"] = seed
|
||||
stream_payload["stream_options"] = {"include_usage": True}
|
||||
|
||||
cumulative = ""
|
||||
|
|
@ -5049,6 +5078,7 @@ class LlamaCppBackend:
|
|||
reasoning_text = ""
|
||||
_metadata_usage = None
|
||||
_metadata_timings = None
|
||||
_metadata_finish_reason = None
|
||||
_stream_done = False
|
||||
|
||||
try:
|
||||
|
|
@ -5107,6 +5137,9 @@ class LlamaCppBackend:
|
|||
choices = chunk_data.get("choices", [])
|
||||
if choices:
|
||||
delta = choices[0].get("delta", {})
|
||||
_fr = choices[0].get("finish_reason")
|
||||
if _fr:
|
||||
_metadata_finish_reason = _fr
|
||||
|
||||
reasoning = delta.get("reasoning_content", "")
|
||||
if reasoning:
|
||||
|
|
@ -5137,7 +5170,7 @@ class LlamaCppBackend:
|
|||
_final_completion = _final_usage.get("completion_tokens", 0)
|
||||
_final_prompt = _final_usage.get("prompt_tokens", 0)
|
||||
_total_completion = _final_completion + _accumulated_completion_tokens
|
||||
if _metadata_usage or _metadata_timings:
|
||||
if _metadata_usage or _metadata_timings or _metadata_finish_reason:
|
||||
_merged_timings = dict(_metadata_timings) if _metadata_timings else {}
|
||||
if _accumulated_predicted_ms or _accumulated_predicted_n:
|
||||
_merged_timings["predicted_ms"] = (
|
||||
|
|
@ -5160,6 +5193,7 @@ class LlamaCppBackend:
|
|||
"total_tokens": _final_prompt + _total_completion,
|
||||
},
|
||||
"timings": _merged_timings,
|
||||
"finish_reason": _metadata_finish_reason,
|
||||
}
|
||||
|
||||
except httpx.ConnectError:
|
||||
|
|
@ -5169,6 +5203,144 @@ class LlamaCppBackend:
|
|||
return
|
||||
raise
|
||||
|
||||
# ── Prompt token counting ──────────────────────────────────
|
||||
|
||||
def count_chat_tokens(
|
||||
self,
|
||||
messages,
|
||||
system = None,
|
||||
tools = None,
|
||||
strict: bool = False,
|
||||
) -> int:
|
||||
"""Count prompt tokens for a chat request via llama-server.
|
||||
|
||||
Non-strict callers keep the historical best-effort behavior and receive
|
||||
0 when a count cannot be determined. Strict callers (public count_tokens
|
||||
endpoints) get an exception instead of a successful-looking zero when
|
||||
tokenizer/template calls fail or a multimodal prompt would fall back to a
|
||||
text-only approximation.
|
||||
"""
|
||||
if not self.is_loaded:
|
||||
if strict:
|
||||
raise RuntimeError("llama-server is not loaded")
|
||||
return 0
|
||||
|
||||
def _has_non_text_content(content) -> bool:
|
||||
if isinstance(content, list):
|
||||
for block in content:
|
||||
if isinstance(block, str):
|
||||
continue
|
||||
if not isinstance(block, dict):
|
||||
return True
|
||||
if block.get("type") == "text" and isinstance(block.get("text"), str):
|
||||
continue
|
||||
if isinstance(block.get("text"), str):
|
||||
continue
|
||||
return True
|
||||
return False
|
||||
|
||||
def _has_non_text_prompt_parts() -> bool:
|
||||
if _has_non_text_content(system):
|
||||
return True
|
||||
for msg in messages or []:
|
||||
if isinstance(msg, dict) and _has_non_text_content(msg.get("content", "")):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _block_text(content) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts = []
|
||||
for block in content:
|
||||
if isinstance(block, dict):
|
||||
if block.get("type") == "text" and isinstance(block.get("text"), str):
|
||||
parts.append(block["text"])
|
||||
elif isinstance(block.get("text"), str):
|
||||
parts.append(block["text"])
|
||||
elif isinstance(block, str):
|
||||
parts.append(block)
|
||||
return "".join(parts)
|
||||
return ""
|
||||
|
||||
# Normalize system into a leading message / plain text.
|
||||
system_text = ""
|
||||
if isinstance(system, str):
|
||||
system_text = system
|
||||
elif isinstance(system, list):
|
||||
system_text = _block_text(system)
|
||||
|
||||
try:
|
||||
_auth_headers = {"Authorization": f"Bearer {self._api_key}"} if self._api_key else None
|
||||
with httpx.Client(timeout = 10, headers = _auth_headers) as client:
|
||||
|
||||
def _tokenize(text: str) -> int:
|
||||
r = client.post(
|
||||
f"{self.base_url}/tokenize",
|
||||
json = {"content": text, "add_special": True},
|
||||
)
|
||||
if r.status_code != 200:
|
||||
if strict:
|
||||
raise RuntimeError("llama-server tokenizer failed")
|
||||
return 0
|
||||
tokens = r.json().get("tokens", [])
|
||||
if not isinstance(tokens, list):
|
||||
if strict:
|
||||
raise RuntimeError("llama-server tokenizer returned invalid tokens")
|
||||
return 0
|
||||
return len(tokens)
|
||||
|
||||
# 1. Try /apply-template to render the real chat prompt.
|
||||
template_messages = list(messages) if messages else []
|
||||
if system_text:
|
||||
template_messages = [
|
||||
{"role": "system", "content": system_text}
|
||||
] + template_messages
|
||||
apply_template_failed = False
|
||||
try:
|
||||
# llama-server's /apply-template renders tool declarations
|
||||
# into the prompt when ``tools`` is supplied, so pass them
|
||||
# through — otherwise tool-schema tokens go uncounted.
|
||||
template_body = {"messages": template_messages}
|
||||
if tools:
|
||||
template_body["tools"] = tools
|
||||
resp = client.post(
|
||||
f"{self.base_url}/apply-template",
|
||||
json = template_body,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
prompt = resp.json().get("prompt", "")
|
||||
if isinstance(prompt, str):
|
||||
return _tokenize(prompt)
|
||||
apply_template_failed = True
|
||||
except Exception:
|
||||
apply_template_failed = True
|
||||
|
||||
if strict and apply_template_failed and _has_non_text_prompt_parts():
|
||||
raise RuntimeError(
|
||||
"cannot fall back to text-only token counting for multimodal messages"
|
||||
)
|
||||
|
||||
# 2. Fallback: concatenate plain text and tokenize. Append a
|
||||
# serialized form of the tools so they still contribute to the
|
||||
# count when /apply-template is unavailable.
|
||||
parts = []
|
||||
if system_text:
|
||||
parts.append(system_text)
|
||||
for msg in messages or []:
|
||||
if isinstance(msg, dict):
|
||||
parts.append(_block_text(msg.get("content", "")))
|
||||
if tools:
|
||||
try:
|
||||
parts.append(json.dumps(tools, ensure_ascii = False))
|
||||
except Exception:
|
||||
pass
|
||||
return _tokenize("\n".join(p for p in parts if p))
|
||||
except Exception:
|
||||
if strict:
|
||||
raise
|
||||
return 0
|
||||
|
||||
# ── TTS support ────────────────────────────────────────────
|
||||
|
||||
def detect_audio_type(self) -> Optional[str]:
|
||||
|
|
|
|||
|
|
@ -247,6 +247,7 @@ from utils.update_status import (
|
|||
get_studio_update_status,
|
||||
)
|
||||
from utils.studio_version import get_studio_version
|
||||
from utils.api_errors import install_api_error_handlers
|
||||
|
||||
|
||||
def get_unsloth_version() -> str:
|
||||
|
|
@ -719,6 +720,7 @@ app.add_middleware(
|
|||
allow_headers = ["*"],
|
||||
)
|
||||
|
||||
|
||||
# ============ Register API Routes ============
|
||||
|
||||
# Register routers
|
||||
|
|
@ -743,6 +745,10 @@ app.include_router(training_history_router, prefix = "/api/train", tags = ["trai
|
|||
app.include_router(hub_inventory_router, prefix = "/api/hub", tags = ["hub"])
|
||||
app.include_router(hub_datasets_router, prefix = "/api/hub/datasets", tags = ["hub"])
|
||||
|
||||
# Re-wrap client-error responses on the /v1/* surface into OpenAI/Anthropic
|
||||
# error envelopes; non-/v1 paths keep FastAPI's default {"detail": ...} shape.
|
||||
install_api_error_handlers(app)
|
||||
|
||||
|
||||
# ============ Health and System Endpoints ============
|
||||
|
||||
|
|
@ -1055,8 +1061,12 @@ def setup_frontend(app: FastAPI, build_path: Path):
|
|||
|
||||
@app.get("/{full_path:path}")
|
||||
async def serve_frontend(request: Request, full_path: str):
|
||||
# Unknown API paths: raise a real 404 so the api_errors handlers can
|
||||
# render the correct envelope for /v1/* (and {"detail":...} for /api/*).
|
||||
# This handler only sees paths NOT matched by a real route. The full
|
||||
# request path is "/" + full_path.
|
||||
if full_path in {"api", "v1"} or full_path.startswith(("api/", "v1/")):
|
||||
return {"error": "API endpoint not found"}
|
||||
raise HTTPException(status_code = 404, detail = "API endpoint not found")
|
||||
|
||||
file_path = (build_path / full_path).resolve()
|
||||
|
||||
|
|
|
|||
|
|
@ -495,7 +495,9 @@ class ChatMessage(BaseModel):
|
|||
``role="tool"`` is resolved at the ``ChatCompletionRequest`` layer.
|
||||
"""
|
||||
|
||||
role: Literal["system", "user", "assistant", "tool"] = Field(..., description = "Message role")
|
||||
role: Literal["system", "user", "assistant", "tool", "developer"] = Field(
|
||||
..., description = "Message role"
|
||||
)
|
||||
content: Optional[Union[str, list[ContentPart]]] = Field(
|
||||
None, description = "Message content (string or multimodal parts)"
|
||||
)
|
||||
|
|
@ -590,6 +592,34 @@ class ChatCompletionRequest(BaseModel):
|
|||
"{'type': 'function', 'function': {'name': ...}}"
|
||||
),
|
||||
)
|
||||
max_completion_tokens: Optional[int] = Field(
|
||||
None,
|
||||
ge = 1,
|
||||
description = "OpenAI upper bound on generated tokens (supersedes the deprecated max_tokens).",
|
||||
)
|
||||
n: Optional[int] = Field(
|
||||
None,
|
||||
ge = 1,
|
||||
le = 128,
|
||||
description = "Number of chat completion choices to generate.",
|
||||
)
|
||||
logprobs: Optional[bool] = Field(
|
||||
None, description = "Whether to return log probabilities of the output tokens."
|
||||
)
|
||||
top_logprobs: Optional[int] = Field(
|
||||
None,
|
||||
ge = 0,
|
||||
le = 20,
|
||||
description = "Number of most likely tokens (0-20) to return per position; requires logprobs=true.",
|
||||
)
|
||||
parallel_tool_calls: Optional[bool] = Field(
|
||||
None, description = "Whether to enable parallel function calling during tool use."
|
||||
)
|
||||
seed: Optional[int] = Field(None, description = "Best-effort deterministic sampling seed.")
|
||||
stream_options: Optional[dict] = Field(
|
||||
None,
|
||||
description = 'Streaming options, e.g. {"include_usage": true} to emit a final usage chunk.',
|
||||
)
|
||||
|
||||
# ── Unsloth extensions (ignored by standard OpenAI clients) ──
|
||||
top_k: int = Field(20, ge = -1, le = 100, description = "[x-unsloth] Top-k sampling")
|
||||
|
|
@ -933,12 +963,16 @@ class ChoiceDelta(BaseModel):
|
|||
content: Optional[str] = None
|
||||
|
||||
|
||||
OpenAIFinishReason = Literal["stop", "length", "tool_calls", "content_filter", "function_call"]
|
||||
|
||||
|
||||
class ChunkChoice(BaseModel):
|
||||
"""A single choice in a streaming chunk."""
|
||||
|
||||
index: int = 0
|
||||
delta: ChoiceDelta
|
||||
finish_reason: Optional[Literal["stop", "length"]] = None
|
||||
finish_reason: Optional[OpenAIFinishReason] = None
|
||||
logprobs: Optional[dict] = None
|
||||
|
||||
|
||||
class ChatCompletionChunk(BaseModel):
|
||||
|
|
@ -961,6 +995,7 @@ class CompletionMessage(BaseModel):
|
|||
|
||||
role: Literal["assistant"] = "assistant"
|
||||
content: str
|
||||
refusal: Optional[str] = None
|
||||
|
||||
|
||||
class CompletionChoice(BaseModel):
|
||||
|
|
@ -968,7 +1003,8 @@ class CompletionChoice(BaseModel):
|
|||
|
||||
index: int = 0
|
||||
message: CompletionMessage
|
||||
finish_reason: Literal["stop", "length"] = "stop"
|
||||
finish_reason: OpenAIFinishReason = "stop"
|
||||
logprobs: Optional[dict] = None
|
||||
|
||||
|
||||
class CompletionUsage(BaseModel):
|
||||
|
|
@ -977,6 +1013,17 @@ class CompletionUsage(BaseModel):
|
|||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
total_tokens: int = 0
|
||||
prompt_tokens_details: Optional[dict] = Field(
|
||||
default_factory = lambda: {"cached_tokens": 0, "audio_tokens": 0}
|
||||
)
|
||||
completion_tokens_details: Optional[dict] = Field(
|
||||
default_factory = lambda: {
|
||||
"reasoning_tokens": 0,
|
||||
"audio_tokens": 0,
|
||||
"accepted_prediction_tokens": 0,
|
||||
"rejected_prediction_tokens": 0,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class ChatCompletion(BaseModel):
|
||||
|
|
@ -988,6 +1035,7 @@ class ChatCompletion(BaseModel):
|
|||
model: str = "default"
|
||||
choices: list[CompletionChoice]
|
||||
usage: CompletionUsage = Field(default_factory = CompletionUsage)
|
||||
system_fingerprint: Optional[str] = None
|
||||
|
||||
|
||||
# =====================================================================
|
||||
|
|
@ -1434,6 +1482,8 @@ class AnthropicMessagesRequest(BaseModel):
|
|||
|
||||
class AnthropicUsage(BaseModel):
|
||||
input_tokens: int = 0
|
||||
cache_creation_input_tokens: int = 0
|
||||
cache_read_input_tokens: int = 0
|
||||
output_tokens: int = 0
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -243,6 +243,12 @@ def _inject_local_structured_response_format(
|
|||
"type": "json_schema",
|
||||
"schema": output_format,
|
||||
}
|
||||
# The OpenAI chat endpoint now returns raw JSON by default for
|
||||
# response_format requests (spec compliance for public clients). This
|
||||
# internal opt-in flag rides through the OpenAI SDK's extra_body
|
||||
# passthrough alongside response_format and re-enables the ```json
|
||||
# markdown fence that data_designer's structured-output parser expects.
|
||||
extra_body["_unsloth_guided_fence"] = True
|
||||
params["extra_body"] = extra_body
|
||||
new_configs.append(clone)
|
||||
column["model_alias"] = clone_alias
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -6,7 +6,9 @@
|
|||
import sys
|
||||
import os
|
||||
import json
|
||||
import threading
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
_backend = os.path.join(os.path.dirname(__file__), "..")
|
||||
|
|
@ -36,6 +38,7 @@ from routes.inference import (
|
|||
_normalize_anthropic_openai_images,
|
||||
_select_anthropic_server_tools,
|
||||
_anthropic_requested_studio_tools,
|
||||
_anthropic_passthrough_stream,
|
||||
_anthropic_tool_non_streaming,
|
||||
anthropic_messages,
|
||||
)
|
||||
|
|
@ -671,7 +674,7 @@ class TestAnthropicStreamEmitter:
|
|||
and payload["content_block"]["type"] == "tool_use"
|
||||
]
|
||||
assert len(tool_starts) == 1
|
||||
assert tool_starts[0]["content_block"]["id"] == "call_0"
|
||||
assert tool_starts[0]["content_block"]["id"].startswith("toolu_")
|
||||
assert second_payloads == [
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
|
|
@ -686,7 +689,7 @@ class TestAnthropicStreamEmitter:
|
|||
def test_tool_end_closes_tool_opens_new_text_block(self):
|
||||
e = AnthropicStreamEmitter()
|
||||
e.start("msg_1", "m")
|
||||
e.feed(
|
||||
start_events = e.feed(
|
||||
{
|
||||
"type": "tool_start",
|
||||
"tool_name": "t",
|
||||
|
|
@ -694,6 +697,13 @@ class TestAnthropicStreamEmitter:
|
|||
"arguments": {},
|
||||
}
|
||||
)
|
||||
start_payload = next(
|
||||
json.loads(event.split("data: ")[1])
|
||||
for event in start_events
|
||||
if "content_block_start" in event
|
||||
)
|
||||
tool_use_id = start_payload["content_block"]["id"]
|
||||
assert tool_use_id.startswith("toolu_")
|
||||
events = e.feed(
|
||||
{
|
||||
"type": "tool_end",
|
||||
|
|
@ -708,7 +718,7 @@ class TestAnthropicStreamEmitter:
|
|||
assert "tool_result" in events[1]
|
||||
parsed = json.loads(events[1].split("data: ")[1])
|
||||
assert parsed["content"] == "done"
|
||||
assert parsed["tool_use_id"] == "tc_1"
|
||||
assert parsed["tool_use_id"] == tool_use_id
|
||||
assert "content_block_start" in events[2]
|
||||
assert '"type": "text"' in events[2]
|
||||
|
||||
|
|
@ -837,14 +847,11 @@ class TestAnthropicToolNonStreaming:
|
|||
body = json.loads(response.body)
|
||||
tool_blocks = [block for block in body["content"] if block["type"] == "tool_use"]
|
||||
|
||||
assert tool_blocks == [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "call_0",
|
||||
"name": "render_html",
|
||||
"input": {"code": "<!doctype html><html></html>"},
|
||||
}
|
||||
]
|
||||
assert len(tool_blocks) == 1
|
||||
assert tool_blocks[0]["type"] == "tool_use"
|
||||
assert tool_blocks[0]["id"].startswith("toolu_")
|
||||
assert tool_blocks[0]["name"] == "render_html"
|
||||
assert tool_blocks[0]["input"] == {"code": "<!doctype html><html></html>"}
|
||||
|
||||
|
||||
# =====================================================================
|
||||
|
|
@ -912,7 +919,7 @@ class TestAnthropicPassthroughEmitter:
|
|||
parsed = self._parse(events[0])
|
||||
assert parsed["type"] == "content_block_start"
|
||||
assert parsed["content_block"]["type"] == "tool_use"
|
||||
assert parsed["content_block"]["id"] == "call_1"
|
||||
assert parsed["content_block"]["id"].startswith("toolu_")
|
||||
assert parsed["content_block"]["name"] == "Bash"
|
||||
|
||||
def test_tool_call_arguments_streamed_as_input_json_delta(self):
|
||||
|
|
@ -1117,7 +1124,102 @@ class TestAnthropicPassthroughEmitter:
|
|||
assert "content_block_start" in events[1]
|
||||
parsed = self._parse(events[1])
|
||||
assert parsed["content_block"]["name"] == "Read"
|
||||
assert parsed["content_block"]["id"] == "c2"
|
||||
assert parsed["content_block"]["id"].startswith("toolu_")
|
||||
|
||||
|
||||
class TestAnthropicPassthroughStreamAdapter:
|
||||
class _Request:
|
||||
async def is_disconnected(self):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
async def _collect(response):
|
||||
chunks = []
|
||||
async for chunk in response.body_iterator:
|
||||
chunks.append(chunk.decode() if isinstance(chunk, bytes) else chunk)
|
||||
return chunks
|
||||
|
||||
@staticmethod
|
||||
def _payloads(lines, event_name):
|
||||
prefix = f"event: {event_name}\n"
|
||||
return [
|
||||
json.loads(line.split("data: ", 1)[1].strip())
|
||||
for line in lines
|
||||
if line.startswith(prefix)
|
||||
]
|
||||
|
||||
def test_stream_requests_usage_for_final_message_delta(self, monkeypatch):
|
||||
import routes.inference as inf_mod
|
||||
|
||||
captured = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured["body"] = json.loads(request.content.decode())
|
||||
chunks = [
|
||||
{"choices": [{"delta": {"content": "hi"}}]},
|
||||
{
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": 2,
|
||||
"completion_tokens": 4,
|
||||
"total_tokens": 6,
|
||||
},
|
||||
},
|
||||
]
|
||||
content = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks)
|
||||
content += "data: [DONE]\n\n"
|
||||
return httpx.Response(
|
||||
200,
|
||||
content = content.encode(),
|
||||
headers = {"content-type": "text/event-stream"},
|
||||
)
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
real_async_client = httpx.AsyncClient
|
||||
|
||||
def _client(*args, **kwargs):
|
||||
return real_async_client(
|
||||
transport = transport,
|
||||
timeout = kwargs.get("timeout", 600),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(inf_mod.httpx, "AsyncClient", _client)
|
||||
backend = SimpleNamespace(
|
||||
base_url = "http://llama.test",
|
||||
context_length = 4096,
|
||||
count_chat_tokens = lambda *args, **kwargs: 2,
|
||||
)
|
||||
|
||||
async def run():
|
||||
response = await _anthropic_passthrough_stream(
|
||||
self._Request(),
|
||||
threading.Event(),
|
||||
backend,
|
||||
[{"role": "user", "content": "hi"}],
|
||||
[
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "lookup",
|
||||
"parameters": {"type": "object"},
|
||||
},
|
||||
}
|
||||
],
|
||||
0.7,
|
||||
0.95,
|
||||
20,
|
||||
16,
|
||||
"msg_1",
|
||||
"test-model",
|
||||
)
|
||||
return await self._collect(response)
|
||||
|
||||
lines = asyncio.run(run())
|
||||
|
||||
assert captured["body"]["stream_options"] == {"include_usage": True}
|
||||
message_delta = self._payloads(lines, "message_delta")[0]
|
||||
assert message_delta["usage"]["input_tokens"] == 2
|
||||
assert message_delta["usage"]["output_tokens"] == 4
|
||||
|
||||
|
||||
# =====================================================================
|
||||
|
|
@ -1271,27 +1373,23 @@ class TestAnthropicRequestedStudioTools:
|
|||
# =====================================================================
|
||||
|
||||
|
||||
class _PlainPathCalled(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class _ToolPathCalled(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _mock_backend(monkeypatch, **overrides):
|
||||
"""Install a minimal stub backend on routes.inference.
|
||||
|
||||
Generation methods raise sentinels so the caller can assert which path
|
||||
the route entered.
|
||||
Generation methods record which path the route entered, then yield one
|
||||
content event so the route can complete normally.
|
||||
"""
|
||||
import routes.inference as inf_mod
|
||||
|
||||
calls = []
|
||||
|
||||
def _gen_plain(**kwargs):
|
||||
raise _PlainPathCalled()
|
||||
calls.append(("plain", kwargs))
|
||||
yield {"type": "content", "text": "ok"}
|
||||
|
||||
def _gen_tools(**kwargs):
|
||||
raise _ToolPathCalled()
|
||||
calls.append(("tools", kwargs))
|
||||
yield {"type": "content", "text": "ok"}
|
||||
|
||||
backend = SimpleNamespace(
|
||||
is_loaded = True,
|
||||
|
|
@ -1300,6 +1398,7 @@ def _mock_backend(monkeypatch, **overrides):
|
|||
model_identifier = "test-model",
|
||||
generate_chat_completion = _gen_plain,
|
||||
generate_chat_completion_with_tools = _gen_tools,
|
||||
calls = calls,
|
||||
)
|
||||
backend.__dict__.update(overrides)
|
||||
monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: backend)
|
||||
|
|
@ -1414,42 +1513,42 @@ class TestAnthropicMessagesToolRouting:
|
|||
assert "input_schema" in exc.value.detail
|
||||
|
||||
def test_unrecognized_server_tool_accepted_as_noop(self, monkeypatch):
|
||||
_mock_backend(monkeypatch)
|
||||
backend = _mock_backend(monkeypatch)
|
||||
payload = _basic_payload(
|
||||
tools = [{"type": "code_execution_20250825", "name": "code_execution"}],
|
||||
)
|
||||
|
||||
with pytest.raises(_PlainPathCalled):
|
||||
_drive(anthropic_messages(payload, request = None, current_subject = "t"))
|
||||
_drive(anthropic_messages(payload, request = None, current_subject = "t"))
|
||||
assert backend.calls[0][0] == "plain"
|
||||
|
||||
def test_disable_tools_policy_overrides_server_tool_alias(self, monkeypatch):
|
||||
# CLI `unsloth run --disable-tools` sets policy=False. A request with
|
||||
# a Studio server-tool alias must NOT enter the agentic loop then.
|
||||
_mock_backend(monkeypatch)
|
||||
backend = _mock_backend(monkeypatch)
|
||||
set_tool_policy(False)
|
||||
payload = _basic_payload(
|
||||
tools = [{"type": "web_search_20250305", "name": "web_search"}],
|
||||
)
|
||||
|
||||
with pytest.raises(_PlainPathCalled):
|
||||
_drive(anthropic_messages(payload, request = None, current_subject = "t"))
|
||||
_drive(anthropic_messages(payload, request = None, current_subject = "t"))
|
||||
assert backend.calls[0][0] == "plain"
|
||||
|
||||
def test_server_tool_alias_enters_tool_path_when_policy_unset(self, monkeypatch):
|
||||
# Mirror of the previous test for the default (None) policy.
|
||||
_mock_backend(monkeypatch)
|
||||
backend = _mock_backend(monkeypatch)
|
||||
payload = _basic_payload(
|
||||
tools = [{"type": "web_search_20250305", "name": "web_search"}],
|
||||
)
|
||||
|
||||
with pytest.raises(_ToolPathCalled):
|
||||
_drive(anthropic_messages(payload, request = None, current_subject = "t"))
|
||||
_drive(anthropic_messages(payload, request = None, current_subject = "t"))
|
||||
assert backend.calls[0][0] == "tools"
|
||||
|
||||
def test_per_request_enable_tools_false_blocks_server_tool_alias(self, monkeypatch):
|
||||
_mock_backend(monkeypatch)
|
||||
backend = _mock_backend(monkeypatch)
|
||||
payload = _basic_payload(
|
||||
enable_tools = False,
|
||||
tools = [{"type": "web_search_20250305", "name": "web_search"}],
|
||||
)
|
||||
|
||||
with pytest.raises(_PlainPathCalled):
|
||||
_drive(anthropic_messages(payload, request = None, current_subject = "t"))
|
||||
_drive(anthropic_messages(payload, request = None, current_subject = "t"))
|
||||
assert backend.calls[0][0] == "plain"
|
||||
|
|
|
|||
|
|
@ -53,10 +53,16 @@ def test_non_streaming_gguf_completion_includes_generated_usage(monkeypatch):
|
|||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["usage"] == {
|
||||
"prompt_tokens": 23,
|
||||
"completion_tokens": 1283,
|
||||
"total_tokens": 1306,
|
||||
usage = response.json()["usage"]
|
||||
assert usage["prompt_tokens"] == 23
|
||||
assert usage["completion_tokens"] == 1283
|
||||
assert usage["total_tokens"] == 1306
|
||||
assert usage["prompt_tokens_details"] == {"cached_tokens": 0, "audio_tokens": 0}
|
||||
assert usage["completion_tokens_details"] == {
|
||||
"reasoning_tokens": 0,
|
||||
"audio_tokens": 0,
|
||||
"accepted_prediction_tokens": 0,
|
||||
"rejected_prediction_tokens": 0,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -67,8 +73,14 @@ def test_non_streaming_gguf_completion_defaults_nullable_usage_to_zero(monkeypat
|
|||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["usage"] == {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 1283,
|
||||
"total_tokens": 0,
|
||||
usage = response.json()["usage"]
|
||||
assert usage["prompt_tokens"] == 0
|
||||
assert usage["completion_tokens"] == 1283
|
||||
assert usage["total_tokens"] == 1283
|
||||
assert usage["prompt_tokens_details"] == {"cached_tokens": 0, "audio_tokens": 0}
|
||||
assert usage["completion_tokens_details"] == {
|
||||
"reasoning_tokens": 0,
|
||||
"audio_tokens": 0,
|
||||
"accepted_prediction_tokens": 0,
|
||||
"rejected_prediction_tokens": 0,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ mapping. No server or GPU required.
|
|||
import os
|
||||
import sys
|
||||
import asyncio
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
_backend = os.path.join(os.path.dirname(__file__), "..")
|
||||
|
|
@ -25,13 +26,19 @@ from pydantic import ValidationError
|
|||
from models.inference import (
|
||||
ChatCompletionRequest,
|
||||
ChatMessage,
|
||||
CompletionChoice,
|
||||
CompletionMessage,
|
||||
)
|
||||
from core.inference.anthropic_compat import (
|
||||
anthropic_tool_choice_to_openai,
|
||||
)
|
||||
from routes.inference import (
|
||||
_build_passthrough_payload,
|
||||
_clamp_finish_reason,
|
||||
_effective_max_tokens,
|
||||
_extract_content_parts,
|
||||
_friendly_error,
|
||||
_openai_stream_usage_chunk,
|
||||
_set_or_prepend_system_message,
|
||||
openai_chat_completions,
|
||||
)
|
||||
|
|
@ -271,17 +278,19 @@ class TestChatCompletionRequestToolFields:
|
|||
assert req.stop is None
|
||||
|
||||
def test_extra_fields_accepted(self):
|
||||
# `frequency_penalty`, `seed`, `response_format` aren't explicitly
|
||||
# declared but must survive Pydantic parsing now that extra="allow".
|
||||
# `frequency_penalty` and `response_format` are not yet explicitly
|
||||
# declared but must survive Pydantic parsing now that extra="allow" is
|
||||
# set. `seed` is declared and should land on the typed field instead.
|
||||
req = self._make(
|
||||
frequency_penalty = 0.5,
|
||||
seed = 42,
|
||||
response_format = {"type": "json_object"},
|
||||
)
|
||||
assert req.seed == 42
|
||||
# Extras land in model_extra
|
||||
assert req.model_extra is not None
|
||||
assert req.model_extra.get("frequency_penalty") == 0.5
|
||||
assert req.model_extra.get("seed") == 42
|
||||
assert "seed" not in req.model_extra
|
||||
assert req.model_extra.get("response_format") == {"type": "json_object"}
|
||||
|
||||
def test_unsloth_extensions_still_work(self):
|
||||
|
|
@ -337,6 +346,153 @@ class TestChatCompletionRequestToolFields:
|
|||
assert "text/event-stream" not in resp.headers["content-type"]
|
||||
assert captured["stream"] is False
|
||||
|
||||
def _v1_client(
|
||||
self,
|
||||
monkeypatch,
|
||||
llama_backend,
|
||||
inference_backend = None,
|
||||
):
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import routes.inference as inference_route
|
||||
from auth.authentication import get_current_subject
|
||||
from utils.api_errors import install_api_error_handlers
|
||||
|
||||
monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: llama_backend)
|
||||
if inference_backend is not None:
|
||||
monkeypatch.setattr(inference_route, "get_inference_backend", lambda: inference_backend)
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(inference_route.router, prefix = "/v1")
|
||||
install_api_error_handlers(app)
|
||||
app.dependency_overrides[get_current_subject] = lambda: "test-user"
|
||||
return TestClient(app)
|
||||
|
||||
def _assert_unsupported_param(self, response, param):
|
||||
assert response.status_code == 400
|
||||
body = response.json()
|
||||
assert body["error"]["param"] == param
|
||||
assert body["error"]["code"] == "unsupported_parameter"
|
||||
|
||||
def _assert_unsupported_n(self, response):
|
||||
self._assert_unsupported_param(response, "n")
|
||||
|
||||
def test_n_allows_openai_chat_completion_range(self):
|
||||
req = self._make(n = 128)
|
||||
assert req.n == 128
|
||||
with pytest.raises(ValidationError):
|
||||
self._make(n = 129)
|
||||
|
||||
def test_n_rejected_for_external_provider_path(self, monkeypatch):
|
||||
class _UnusedBackend:
|
||||
is_loaded = False
|
||||
|
||||
client = self._v1_client(monkeypatch, _UnusedBackend())
|
||||
resp = client.post(
|
||||
"/v1/chat/completions",
|
||||
json = {
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"provider_type": "openai",
|
||||
"n": 2,
|
||||
},
|
||||
)
|
||||
self._assert_unsupported_n(resp)
|
||||
|
||||
def test_logprobs_rejected_until_supported(self, monkeypatch):
|
||||
class _UnusedBackend:
|
||||
is_loaded = False
|
||||
|
||||
client = self._v1_client(monkeypatch, _UnusedBackend())
|
||||
resp = client.post(
|
||||
"/v1/chat/completions",
|
||||
json = {
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"provider_type": "openai",
|
||||
"logprobs": True,
|
||||
},
|
||||
)
|
||||
self._assert_unsupported_param(resp, "logprobs")
|
||||
|
||||
def test_top_logprobs_rejected_until_supported(self, monkeypatch):
|
||||
class _UnusedBackend:
|
||||
is_loaded = False
|
||||
|
||||
client = self._v1_client(monkeypatch, _UnusedBackend())
|
||||
resp = client.post(
|
||||
"/v1/chat/completions",
|
||||
json = {
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"provider_type": "openai",
|
||||
"top_logprobs": 3,
|
||||
},
|
||||
)
|
||||
self._assert_unsupported_param(resp, "top_logprobs")
|
||||
|
||||
def test_n_rejected_for_gguf_streaming_path(self, monkeypatch):
|
||||
class _GGUFBackend:
|
||||
is_loaded = True
|
||||
model_identifier = "test-gguf"
|
||||
supports_tools = False
|
||||
is_vision = False
|
||||
_is_audio = False
|
||||
|
||||
client = self._v1_client(monkeypatch, _GGUFBackend())
|
||||
resp = client.post(
|
||||
"/v1/chat/completions",
|
||||
json = {
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"stream": True,
|
||||
"n": 2,
|
||||
},
|
||||
)
|
||||
self._assert_unsupported_n(resp)
|
||||
|
||||
def test_n_rejected_for_gguf_tools_passthrough_path(self, monkeypatch):
|
||||
class _GGUFBackend:
|
||||
is_loaded = True
|
||||
model_identifier = "test-gguf"
|
||||
supports_tools = True
|
||||
is_vision = False
|
||||
_is_audio = False
|
||||
|
||||
client = self._v1_client(monkeypatch, _GGUFBackend())
|
||||
resp = client.post(
|
||||
"/v1/chat/completions",
|
||||
json = {
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "lookup",
|
||||
"parameters": {"type": "object"},
|
||||
},
|
||||
}
|
||||
],
|
||||
"n": 2,
|
||||
},
|
||||
)
|
||||
self._assert_unsupported_n(resp)
|
||||
|
||||
def test_n_rejected_for_non_gguf_path(self, monkeypatch):
|
||||
class _NoGGUFBackend:
|
||||
is_loaded = False
|
||||
|
||||
class _InferenceBackend:
|
||||
active_model_name = "test-model"
|
||||
models = {"test-model": {}}
|
||||
|
||||
client = self._v1_client(monkeypatch, _NoGGUFBackend(), _InferenceBackend())
|
||||
resp = client.post(
|
||||
"/v1/chat/completions",
|
||||
json = {
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"n": 2,
|
||||
},
|
||||
)
|
||||
self._assert_unsupported_n(resp)
|
||||
|
||||
def test_multiturn_tool_loop_messages(self):
|
||||
req = ChatCompletionRequest(
|
||||
messages = [
|
||||
|
|
@ -449,18 +605,115 @@ class TestBuildPassthroughPayloadToolChoice:
|
|||
body = _build_passthrough_payload(**self._args(), tool_choice = tc)
|
||||
assert body["tool_choice"] == tc
|
||||
|
||||
def test_stream_adds_include_usage(self):
|
||||
def test_stream_omits_usage_options_when_client_did_not_request_them(self):
|
||||
args = self._args()
|
||||
args["stream"] = True
|
||||
body = _build_passthrough_payload(**args)
|
||||
assert "stream_options" not in body
|
||||
|
||||
def test_stream_forwards_include_usage_when_client_requests_it(self):
|
||||
args = self._args()
|
||||
args["stream"] = True
|
||||
body = _build_passthrough_payload(
|
||||
**args,
|
||||
stream_options = {"include_usage": True},
|
||||
)
|
||||
assert body.get("stream_options") == {"include_usage": True}
|
||||
|
||||
def test_stream_forwards_include_usage_false_when_client_requests_it(self):
|
||||
args = self._args()
|
||||
args["stream"] = True
|
||||
body = _build_passthrough_payload(
|
||||
**args,
|
||||
stream_options = {"include_usage": False},
|
||||
)
|
||||
assert body.get("stream_options") == {"include_usage": False}
|
||||
|
||||
def test_repetition_penalty_renamed(self):
|
||||
body = _build_passthrough_payload(**self._args(), repetition_penalty = 1.1)
|
||||
assert body.get("repeat_penalty") == 1.1
|
||||
assert "repetition_penalty" not in body
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# OpenAI API compatibility helpers — verified spec edge cases
|
||||
# =====================================================================
|
||||
|
||||
|
||||
class TestOpenAICompatibilityHelpers:
|
||||
def test_max_completion_tokens_wins_over_deprecated_max_tokens(self):
|
||||
payload = SimpleNamespace(max_tokens = 128, max_completion_tokens = 64)
|
||||
assert _effective_max_tokens(payload) == 64
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"finish_reason",
|
||||
["stop", "length", "tool_calls", "content_filter", "function_call"],
|
||||
)
|
||||
def test_clamp_finish_reason_preserves_openai_finish_reasons(self, finish_reason):
|
||||
assert _clamp_finish_reason(finish_reason) == finish_reason
|
||||
|
||||
def test_clamp_finish_reason_defaults_unknown_to_stop(self):
|
||||
assert _clamp_finish_reason(None) == "stop"
|
||||
assert _clamp_finish_reason("unexpected") == "stop"
|
||||
|
||||
def test_non_streaming_completion_choice_accepts_tool_calls_finish_reason(self):
|
||||
choice = CompletionChoice(
|
||||
index = 0,
|
||||
message = CompletionMessage(content = ""),
|
||||
finish_reason = "tool_calls",
|
||||
)
|
||||
assert choice.finish_reason == "tool_calls"
|
||||
|
||||
def test_stream_usage_chunk_requires_include_usage(self):
|
||||
usage = {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}
|
||||
payload = SimpleNamespace(stream_options = None)
|
||||
assert (
|
||||
_openai_stream_usage_chunk(payload, "chatcmpl-test", 123, "model", usage, None) is None
|
||||
)
|
||||
|
||||
payload.stream_options = {"include_usage": True}
|
||||
line = _openai_stream_usage_chunk(payload, "chatcmpl-test", 123, "model", usage, None)
|
||||
assert line is not None
|
||||
assert '"choices":[]' in line
|
||||
assert '"usage"' in line
|
||||
|
||||
def test_stream_usage_chunk_coerces_nullable_counts(self):
|
||||
payload = SimpleNamespace(stream_options = {"include_usage": True})
|
||||
line = _openai_stream_usage_chunk(
|
||||
payload,
|
||||
"chatcmpl-test",
|
||||
123,
|
||||
"model",
|
||||
{"prompt_tokens": None, "completion_tokens": 7, "total_tokens": None},
|
||||
None,
|
||||
)
|
||||
|
||||
assert line is not None
|
||||
parsed = json.loads(line.removeprefix("data: "))
|
||||
usage = parsed["usage"]
|
||||
assert usage["prompt_tokens"] == 0
|
||||
assert usage["completion_tokens"] == 7
|
||||
assert usage["total_tokens"] == 7
|
||||
|
||||
def test_developer_message_preserves_existing_system_prompt(self):
|
||||
payload = ChatCompletionRequest(
|
||||
messages = [
|
||||
{"role": "system", "content": "original system"},
|
||||
{"role": "developer", "content": "developer rules"},
|
||||
{"role": "user", "content": "hi"},
|
||||
]
|
||||
)
|
||||
for message in payload.messages:
|
||||
if message.role == "developer":
|
||||
message.role = "system"
|
||||
|
||||
system_prompt, chat_messages, image_b64 = _extract_content_parts(payload.messages)
|
||||
|
||||
assert system_prompt == "original system\n\ndeveloper rules"
|
||||
assert chat_messages == [{"role": "user", "content": "hi"}]
|
||||
assert image_b64 is None
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# _friendly_error — httpx transport failures
|
||||
# =====================================================================
|
||||
|
|
@ -800,3 +1053,90 @@ class TestGgufVisionToolRouting:
|
|||
assert tool_messages[0]["role"] == "system"
|
||||
assert tool_messages[1]["role"] == "user"
|
||||
assert tool_messages[1]["content"][1]["type"] == "image_url"
|
||||
|
||||
def test_parallel_tool_calls_false_reaches_gguf_tool_loop(self, monkeypatch):
|
||||
import routes.inference as inf_mod
|
||||
|
||||
reset_tool_policy()
|
||||
captured = {}
|
||||
|
||||
def _plain(**kwargs):
|
||||
raise AssertionError("plain GGUF path should not be used")
|
||||
|
||||
def _tools(**kwargs):
|
||||
captured["kwargs"] = kwargs
|
||||
yield {"type": "content", "text": "done"}
|
||||
|
||||
backend = SimpleNamespace(
|
||||
is_loaded = True,
|
||||
is_vision = False,
|
||||
supports_tools = True,
|
||||
model_identifier = "test-gguf",
|
||||
generate_chat_completion = _plain,
|
||||
generate_chat_completion_with_tools = _tools,
|
||||
)
|
||||
monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: backend)
|
||||
|
||||
payload = ChatCompletionRequest(
|
||||
model = "default",
|
||||
enable_tools = True,
|
||||
enabled_tools = ["web_search"],
|
||||
parallel_tool_calls = False,
|
||||
messages = [{"role": "user", "content": "search once"}],
|
||||
)
|
||||
|
||||
response = self._drive(
|
||||
openai_chat_completions(payload, request = self._Request(), current_subject = "test")
|
||||
)
|
||||
self._consume_response(response)
|
||||
|
||||
assert captured["kwargs"]["disable_parallel_tool_use"] is True
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("seed", "expected"),
|
||||
[
|
||||
(41, [41, 42, 43]),
|
||||
(-1, [-1, -1, -1]),
|
||||
],
|
||||
)
|
||||
def test_gguf_n_choices_vary_explicit_non_negative_seed(self, monkeypatch, seed, expected):
|
||||
import routes.inference as inf_mod
|
||||
|
||||
seen_seeds = []
|
||||
|
||||
def _generate(**kwargs):
|
||||
seen_seeds.append(kwargs.get("seed"))
|
||||
yield f"choice-{len(seen_seeds)}"
|
||||
yield {
|
||||
"type": "metadata",
|
||||
"usage": {
|
||||
"prompt_tokens": 5,
|
||||
"completion_tokens": 7,
|
||||
"total_tokens": 12,
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
|
||||
backend = SimpleNamespace(
|
||||
is_loaded = True,
|
||||
is_vision = False,
|
||||
supports_tools = False,
|
||||
model_identifier = "test-gguf",
|
||||
generate_chat_completion = _generate,
|
||||
)
|
||||
monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: backend)
|
||||
|
||||
payload = ChatCompletionRequest(
|
||||
model = "default",
|
||||
messages = [{"role": "user", "content": "hi"}],
|
||||
n = 3,
|
||||
seed = seed,
|
||||
)
|
||||
|
||||
response = self._drive(
|
||||
openai_chat_completions(payload, request = self._Request(), current_subject = "test")
|
||||
)
|
||||
body = json.loads(response.body)
|
||||
|
||||
assert seen_seeds == expected
|
||||
assert [choice["index"] for choice in body["choices"]] == [0, 1, 2]
|
||||
|
|
|
|||
|
|
@ -26,12 +26,15 @@ No running server or GPU required.
|
|||
|
||||
import os
|
||||
import sys
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
|
||||
_backend = os.path.join(os.path.dirname(__file__), "..")
|
||||
sys.path.insert(0, _backend)
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
|
|
@ -52,8 +55,10 @@ from models.inference import (
|
|||
ResponsesUsage,
|
||||
)
|
||||
from routes.inference import (
|
||||
_build_chat_request,
|
||||
_chat_tool_calls_to_responses_output,
|
||||
_normalise_responses_input,
|
||||
_responses_stream,
|
||||
_translate_responses_tool_choice_to_chat,
|
||||
_translate_responses_tools_to_chat,
|
||||
)
|
||||
|
|
@ -259,6 +264,26 @@ class TestToolChoiceTranslation:
|
|||
assert _translate_responses_tool_choice_to_chat(obj) == obj
|
||||
|
||||
|
||||
class TestBuildChatRequest:
|
||||
def test_parallel_tool_calls_false_is_preserved_for_passthrough_caps(self):
|
||||
payload = ResponsesRequest(
|
||||
input = "hi",
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "lookup",
|
||||
"parameters": {"type": "object"},
|
||||
}
|
||||
],
|
||||
parallel_tool_calls = False,
|
||||
)
|
||||
messages = [ChatMessage(role = "user", content = "hi")]
|
||||
|
||||
chat_req = _build_chat_request(payload, messages, stream = True)
|
||||
|
||||
assert chat_req.parallel_tool_calls is False
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# _normalise_responses_input — multi-turn tool mapping
|
||||
# =====================================================================
|
||||
|
|
@ -424,6 +449,130 @@ class TestChatToolCallsToResponsesOutput:
|
|||
assert items[0]["arguments"] == ""
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# Streaming Responses adapter
|
||||
# =====================================================================
|
||||
|
||||
|
||||
class TestResponsesStreamAdapter:
|
||||
class _Request:
|
||||
async def is_disconnected(self):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
async def _collect(response):
|
||||
chunks = []
|
||||
async for chunk in response.body_iterator:
|
||||
chunks.append(chunk.decode() if isinstance(chunk, bytes) else chunk)
|
||||
return chunks
|
||||
|
||||
@staticmethod
|
||||
def _payloads(lines, event_name):
|
||||
prefix = f"event: {event_name}\n"
|
||||
return [
|
||||
json.loads(line.split("data: ", 1)[1].strip())
|
||||
for line in lines
|
||||
if line.startswith(prefix)
|
||||
]
|
||||
|
||||
def test_requests_usage_and_caps_parallel_tool_calls(self, monkeypatch):
|
||||
import routes.inference as inf_mod
|
||||
|
||||
captured = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
captured["body"] = json.loads(request.content.decode())
|
||||
chunks = [
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"delta": {
|
||||
"tool_calls": [
|
||||
{
|
||||
"index": 0,
|
||||
"id": "call_0",
|
||||
"type": "function",
|
||||
"function": {"name": "first", "arguments": "{}"},
|
||||
},
|
||||
{
|
||||
"index": 1,
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "second", "arguments": "{}"},
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{"choices": [], "usage": {"prompt_tokens": 2, "completion_tokens": 3}},
|
||||
]
|
||||
content = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks)
|
||||
content += "data: [DONE]\n\n"
|
||||
return httpx.Response(
|
||||
200,
|
||||
content = content.encode(),
|
||||
headers = {"content-type": "text/event-stream"},
|
||||
)
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
real_async_client = httpx.AsyncClient
|
||||
|
||||
def _client(*args, **kwargs):
|
||||
return real_async_client(
|
||||
transport = transport,
|
||||
timeout = kwargs.get("timeout", 600),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(inf_mod.httpx, "AsyncClient", _client)
|
||||
monkeypatch.setattr(
|
||||
inf_mod,
|
||||
"get_llama_cpp_backend",
|
||||
lambda: SimpleNamespace(
|
||||
is_loaded = True,
|
||||
is_vision = False,
|
||||
context_length = 4096,
|
||||
base_url = "http://llama.test",
|
||||
),
|
||||
)
|
||||
|
||||
payload = ResponsesRequest(
|
||||
input = "hi",
|
||||
stream = True,
|
||||
parallel_tool_calls = False,
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "first",
|
||||
"parameters": {"type": "object"},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"name": "second",
|
||||
"parameters": {"type": "object"},
|
||||
},
|
||||
],
|
||||
)
|
||||
messages = [ChatMessage(role = "user", content = "hi")]
|
||||
|
||||
async def run():
|
||||
response = await _responses_stream(payload, messages, self._Request())
|
||||
return await self._collect(response)
|
||||
|
||||
lines = asyncio.run(run())
|
||||
|
||||
assert captured["body"]["stream_options"] == {"include_usage": True}
|
||||
joined = "".join(lines)
|
||||
assert "call_0" in joined
|
||||
assert "call_1" not in joined
|
||||
completed = self._payloads(lines, "response.completed")[0]
|
||||
assert completed["response"]["usage"] == {
|
||||
"input_tokens": 2,
|
||||
"output_tokens": 3,
|
||||
"total_tokens": 5,
|
||||
}
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# Response model — ResponsesOutputFunctionCall / mixed output
|
||||
# =====================================================================
|
||||
|
|
|
|||
252
studio/backend/utils/api_errors.py
Normal file
252
studio/backend/utils/api_errors.py
Normal file
|
|
@ -0,0 +1,252 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
"""Error-envelope helpers for the OpenAI/Anthropic-compatible ``/v1/*`` API surface.
|
||||
|
||||
FastAPI's defaults emit ``{"detail": ...}`` bodies (status 422 for validation,
|
||||
``exc.status_code`` for ``HTTPException``). Real OpenAI/Anthropic clients expect
|
||||
provider-specific error envelopes instead, so this module re-wraps Unsloth's own
|
||||
client-error responses on the ``/v1/*`` surface:
|
||||
|
||||
- OpenAI surface (``/v1/chat/completions``, ``/v1/completions``, ``/v1/models``,
|
||||
``/v1/responses``, ``/v1/embeddings``, ...)::
|
||||
|
||||
{"error": {"message": str, "type": str, "param": None|str, "code": None|str}}
|
||||
|
||||
- Anthropic surface (any path starting with ``/v1/messages``)::
|
||||
|
||||
{"type": "error", "error": {"type": str, "message": str}}
|
||||
|
||||
CRITICAL: the exception handlers installed by :func:`install_api_error_handlers`
|
||||
are global, but they ONLY transform responses for paths that start with ``/v1/``.
|
||||
For every other path (``/api/...``, frontend routes) they reproduce FastAPI's
|
||||
default behavior byte-for-byte, because the Studio frontend depends on the
|
||||
``{"detail": ...}`` shape for ``/api/*``.
|
||||
|
||||
Public contract (other modules depend on these):
|
||||
|
||||
- ``OPENAI_TYPE_BY_STATUS`` / ``ANTHROPIC_TYPE_BY_STATUS``: status -> type maps.
|
||||
- ``openai_error_body(message, *, status=400, err_type=None, code=None, param=None)``
|
||||
- ``anthropic_error_body(message, *, status=400, err_type=None)``
|
||||
- ``is_anthropic_path(path)``
|
||||
- ``error_body_for_path(path, message, *, status, err_type=None, code=None, param=None)``
|
||||
- ``install_api_error_handlers(app)``
|
||||
"""
|
||||
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.utils import is_body_allowed_for_status_code
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
|
||||
# Status-code -> error ``type`` string for the OpenAI error envelope.
|
||||
OPENAI_TYPE_BY_STATUS = {
|
||||
400: "invalid_request_error",
|
||||
401: "authentication_error",
|
||||
403: "permission_error",
|
||||
404: "not_found_error",
|
||||
409: "conflict_error",
|
||||
413: "invalid_request_error",
|
||||
422: "invalid_request_error",
|
||||
429: "rate_limit_error",
|
||||
500: "api_error",
|
||||
502: "api_error",
|
||||
503: "api_error",
|
||||
}
|
||||
|
||||
# Status-code -> error ``type`` string for the Anthropic error envelope.
|
||||
ANTHROPIC_TYPE_BY_STATUS = {
|
||||
400: "invalid_request_error",
|
||||
401: "authentication_error",
|
||||
403: "permission_error",
|
||||
404: "not_found_error",
|
||||
409: "conflict_error",
|
||||
413: "request_too_large",
|
||||
422: "invalid_request_error",
|
||||
429: "rate_limit_error",
|
||||
500: "api_error",
|
||||
502: "api_error",
|
||||
503: "api_error",
|
||||
529: "overloaded_error",
|
||||
}
|
||||
|
||||
|
||||
def openai_error_body(
|
||||
message,
|
||||
*,
|
||||
status = 400,
|
||||
err_type = None,
|
||||
code = None,
|
||||
param = None,
|
||||
) -> dict:
|
||||
"""Build an OpenAI-style error envelope.
|
||||
|
||||
Returns ``{"error": {"message", "type", "param", "code"}}``. The ``param``
|
||||
and ``code`` keys are always present (value may be ``None``). ``err_type``
|
||||
defaults to :data:`OPENAI_TYPE_BY_STATUS` for ``status`` (``"api_error"``
|
||||
fallback).
|
||||
"""
|
||||
return {
|
||||
"error": {
|
||||
"message": str(message),
|
||||
"type": err_type or OPENAI_TYPE_BY_STATUS.get(status, "api_error"),
|
||||
"param": param,
|
||||
"code": code,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def anthropic_error_body(
|
||||
message,
|
||||
*,
|
||||
status = 400,
|
||||
err_type = None,
|
||||
) -> dict:
|
||||
"""Build an Anthropic-style error envelope.
|
||||
|
||||
Returns ``{"type": "error", "request_id": None, "error": {"type", "message"}}``.
|
||||
``request_id`` is a required (nullable) field on the spec's ErrorResponse;
|
||||
Studio has no request-id system, so it is null. ``err_type`` defaults to
|
||||
:data:`ANTHROPIC_TYPE_BY_STATUS` for ``status`` (``"api_error"`` fallback).
|
||||
"""
|
||||
return {
|
||||
"type": "error",
|
||||
"request_id": None,
|
||||
"error": {
|
||||
"type": err_type or ANTHROPIC_TYPE_BY_STATUS.get(status, "api_error"),
|
||||
"message": str(message),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def is_anthropic_path(path: str) -> bool:
|
||||
"""True iff ``path`` belongs to the Anthropic surface (``/v1/messages*``)."""
|
||||
return path.startswith("/v1/messages")
|
||||
|
||||
|
||||
def error_body_for_path(
|
||||
path,
|
||||
message,
|
||||
*,
|
||||
status,
|
||||
err_type = None,
|
||||
code = None,
|
||||
param = None,
|
||||
) -> dict:
|
||||
"""Dispatch to the correct envelope builder based on ``path``.
|
||||
|
||||
Anthropic surface paths use :func:`anthropic_error_body` (``code``/``param``
|
||||
are not part of that envelope and are ignored); all other ``/v1/*`` paths use
|
||||
:func:`openai_error_body`.
|
||||
"""
|
||||
if is_anthropic_path(path):
|
||||
return anthropic_error_body(message, status = status, err_type = err_type)
|
||||
return openai_error_body(message, status = status, err_type = err_type, code = code, param = param)
|
||||
|
||||
|
||||
def _summarize_validation_errors(errors) -> tuple:
|
||||
"""Derive a readable one-line message and (optional) body param from ``exc.errors()``.
|
||||
|
||||
Returns ``(summary, param)``. ``summary`` is a human-readable string like
|
||||
``"messages: Field required"``. ``param`` is the offending body field name when
|
||||
one can be extracted (used as the OpenAI envelope ``param``), else ``None``.
|
||||
|
||||
Malformed-JSON bodies surface here as ``type == "json_invalid"`` and get a
|
||||
dedicated message.
|
||||
"""
|
||||
if not errors:
|
||||
return "Invalid request", None
|
||||
|
||||
first = errors[0]
|
||||
if first.get("type") == "json_invalid":
|
||||
return "Invalid JSON in request body", None
|
||||
|
||||
loc = first.get("loc", ()) or ()
|
||||
msg = first.get("msg", "Invalid request")
|
||||
|
||||
# Extract the body field name (the loc element after a leading "body").
|
||||
param = None
|
||||
loc_parts = [p for p in loc if p not in ("body",)]
|
||||
if loc and loc[0] == "body" and loc_parts:
|
||||
# First non-"body" element that is a field name (string).
|
||||
for part in loc_parts:
|
||||
if isinstance(part, str):
|
||||
param = part
|
||||
break
|
||||
|
||||
label = ".".join(str(p) for p in loc_parts) if loc_parts else ".".join(str(p) for p in loc)
|
||||
summary = f"{label}: {msg}" if label else str(msg)
|
||||
return summary, param
|
||||
|
||||
|
||||
def install_api_error_handlers(app) -> None:
|
||||
"""Register validation + HTTPException handlers that emit ``/v1/*`` envelopes.
|
||||
|
||||
Both handlers are global but only transform responses for paths starting with
|
||||
``/v1/``. Non-``/v1/`` paths reproduce FastAPI's default ``{"detail": ...}``
|
||||
behavior exactly so the Studio frontend keeps working.
|
||||
"""
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def _handle_validation_error(request, exc):
|
||||
path = request.url.path
|
||||
if path.startswith("/v1/"):
|
||||
summary, param = _summarize_validation_errors(exc.errors())
|
||||
return JSONResponse(
|
||||
status_code = 400,
|
||||
content = error_body_for_path(path, summary, status = 400, param = param),
|
||||
)
|
||||
# Default FastAPI behavior for every other path.
|
||||
return JSONResponse(
|
||||
status_code = 422,
|
||||
content = {"detail": jsonable_encoder(exc.errors())},
|
||||
)
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def _handle_http_exception(request, exc):
|
||||
path = request.url.path
|
||||
headers = getattr(exc, "headers", None)
|
||||
# Statuses like 204/304/1xx must not carry a body — mirror FastAPI's
|
||||
# default http_exception_handler, which returns a bodiless Response.
|
||||
if not is_body_allowed_for_status_code(exc.status_code):
|
||||
return Response(status_code = exc.status_code, headers = headers)
|
||||
if path.startswith("/v1/"):
|
||||
detail = exc.detail
|
||||
# Already a fully-formed envelope: pass through untouched.
|
||||
if isinstance(detail, dict) and ("error" in detail or detail.get("type") == "error"):
|
||||
return JSONResponse(
|
||||
status_code = exc.status_code,
|
||||
content = detail,
|
||||
headers = headers,
|
||||
)
|
||||
# A dict carrying our individual fields.
|
||||
if isinstance(detail, dict):
|
||||
message = detail.get("message", detail)
|
||||
err_type = detail.get("type")
|
||||
code = detail.get("code")
|
||||
param = detail.get("param")
|
||||
else:
|
||||
# Plain message string (the common HTTPException case).
|
||||
message = detail
|
||||
err_type = None
|
||||
code = None
|
||||
param = None
|
||||
return JSONResponse(
|
||||
status_code = exc.status_code,
|
||||
content = error_body_for_path(
|
||||
path,
|
||||
message,
|
||||
status = exc.status_code,
|
||||
err_type = err_type,
|
||||
code = code,
|
||||
param = param,
|
||||
),
|
||||
headers = headers,
|
||||
)
|
||||
# Default FastAPI behavior for every other path.
|
||||
return JSONResponse(
|
||||
status_code = exc.status_code,
|
||||
content = {"detail": exc.detail},
|
||||
headers = headers,
|
||||
)
|
||||
|
|
@ -2339,6 +2339,9 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
model: params.checkpoint,
|
||||
messages: outboundMessages,
|
||||
stream: true,
|
||||
// Opt into the trailing usage chunk so the context-usage bar
|
||||
// and tok/s readout populate (backend gates it on include_usage).
|
||||
stream_options: { include_usage: true },
|
||||
temperature: params.temperature,
|
||||
top_p: params.topP,
|
||||
max_tokens: params.maxTokens,
|
||||
|
|
|
|||
|
|
@ -309,6 +309,13 @@ export interface OpenAIChatCompletionsRequest {
|
|||
* See https://platform.claude.com/docs/en/build-with-claude/fast-mode
|
||||
*/
|
||||
fast_mode?: boolean | null;
|
||||
/**
|
||||
* Opt into the OpenAI-standard trailing usage chunk on streams
|
||||
* (`choices: []` with `usage` + llama-server `timings` populated). The
|
||||
* backend only emits it when `include_usage` is set; the local chat UI
|
||||
* sends it so the context-usage bar and tok/s readout populate.
|
||||
*/
|
||||
stream_options?: { include_usage?: boolean } | null;
|
||||
}
|
||||
|
||||
export interface OpenAIChatDelta {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue