Studio: expose Anthropic / OpenAI sampling knobs per provider
Adds the missing sampling parameters that the upstream APIs accept and that Studio's chat UI previously hid. Each knob is gated per provider so the picker never offers a field the upstream would 400 on, and the per-provider stream functions translate / drop fields to match each API's naming. New `InferenceParams` fields (round-trip through PersistedInferenceParams and the chat-settings server store automatically): - frequencyPenalty (-2..2): OpenAI Chat Completions only. - seed (int | null): OpenAI Chat + OpenAI-compat local backends. - stop (string[]): all OpenAI Chat + Anthropic Messages. Backend truncates to 4 entries on OpenAI Chat per docs and renames to `stop_sequences` on Anthropic. - serviceTier (auto|default|flex|priority|scale|standard_only): per-provider enum sets resolved by getServiceTierOptions. - parallelToolCalls (bool, default true): forwarded as `parallel_tool_calls` on both OpenAI APIs and inverted into `disable_parallel_tool_use` on Anthropic. OpenAI Responses (gpt-5.x / o3) explicitly drops frequencyPenalty / seed / stop alongside the existing temperature / top_p drop, since the upstream 400s on all of them. service_tier on Responses accepts a subset (no `scale`) which the dispatch already enforces. UI rows land in the existing Sampling section of the chat settings sheet using ParamSlider (frequency penalty), a numeric Input (seed), a new chips editor `StopSequencesInput` (stop), Select (service tier), and Switch (parallel tool calls). Each row's visibility follows the new ProviderCapabilities flag. Tests pin the gating contract: stop_sequences renamed on Anthropic, 4-entry truncation on OpenAI Chat, every Responses-rejected field dropped, schema-level validation for the service_tier Literal and frequency_penalty range. Plan: plans/hashed-riding-porcupine.md
This commit is contained in:
Daniel Han
parent
83b20976f7
commit
91d04741ff
11 changed files with 1002 additions and 4 deletions
|
|
@ -11,7 +11,7 @@ Anthropic uses native Messages API with translation in this client.
|
|||
import json as _json
|
||||
import re
|
||||
import time
|
||||
from typing import Any, AsyncGenerator, Literal, NamedTuple, Optional
|
||||
from typing import Any, AsyncGenerator, Literal, NamedTuple, Optional, Union
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
|
@ -348,6 +348,11 @@ class ExternalProviderClient:
|
|||
anthropic_code_exec_container_id: Optional[str] = None,
|
||||
prompt_cache_ttl: Optional[str] = None,
|
||||
compaction_threshold: Optional[int] = None,
|
||||
frequency_penalty: Optional[float] = None,
|
||||
seed: Optional[int] = None,
|
||||
stop: Optional[Union[str, list[str]]] = None,
|
||||
service_tier: Optional[str] = None,
|
||||
parallel_tool_calls: Optional[bool] = None,
|
||||
stream: bool = True,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""
|
||||
|
|
@ -360,6 +365,12 @@ class ExternalProviderClient:
|
|||
supplies a value the provider accepts — the frontend's
|
||||
provider-capability map already filters these per provider, so we
|
||||
treat them as opt-in here.
|
||||
|
||||
``frequency_penalty``, ``seed``, ``stop``, ``service_tier``,
|
||||
``parallel_tool_calls`` follow the same rule: the per-provider
|
||||
stream helpers silently drop fields the upstream API does not
|
||||
accept (e.g. Responses rejects all of seed / frequency / stop;
|
||||
Anthropic does not implement seed / frequency / logprobs).
|
||||
"""
|
||||
if not self._is_openai_compatible():
|
||||
async for line in self._stream_anthropic(
|
||||
|
|
@ -376,6 +387,9 @@ class ExternalProviderClient:
|
|||
anthropic_code_exec_container_id,
|
||||
prompt_cache_ttl,
|
||||
compaction_threshold,
|
||||
stop = stop,
|
||||
service_tier = service_tier,
|
||||
parallel_tool_calls = parallel_tool_calls,
|
||||
):
|
||||
yield line
|
||||
return
|
||||
|
|
@ -398,6 +412,8 @@ class ExternalProviderClient:
|
|||
enable_prompt_caching,
|
||||
openai_code_exec_container_id,
|
||||
compaction_threshold,
|
||||
service_tier = service_tier,
|
||||
parallel_tool_calls = parallel_tool_calls,
|
||||
):
|
||||
yield line
|
||||
return
|
||||
|
|
@ -438,6 +454,38 @@ class ExternalProviderClient:
|
|||
else:
|
||||
body["max_tokens"] = max_tokens
|
||||
|
||||
# Optional sampling extensions (added in #5XXX). Only forwarded
|
||||
# when the caller passed a value. Each upstream provider that
|
||||
# 400s on the field appears in `body_omit` (see providers.py)
|
||||
# so the registry-driven drop loop below removes them before
|
||||
# the request hits the wire. The Responses path
|
||||
# (_stream_openai_responses) drops these explicitly because it
|
||||
# never reaches this body construction.
|
||||
if frequency_penalty is not None:
|
||||
body["frequency_penalty"] = frequency_penalty
|
||||
if seed is not None:
|
||||
body["seed"] = seed
|
||||
if stop is not None:
|
||||
# OpenAI Chat caps the list at 4 entries. Truncate with a
|
||||
# warning rather than letting the upstream 400; users editing
|
||||
# chips in the picker shouldn't get a cryptic API error.
|
||||
if isinstance(stop, list) and len(stop) > 4:
|
||||
logger.warning(
|
||||
"stop sequences truncated to 4 entries "
|
||||
"(received %d, OpenAI's hard cap is 4)",
|
||||
len(stop),
|
||||
)
|
||||
body["stop"] = stop[:4]
|
||||
elif isinstance(stop, list) and len(stop) == 0:
|
||||
# Empty list = unset; don't ship `stop: []`.
|
||||
pass
|
||||
else:
|
||||
body["stop"] = stop
|
||||
if service_tier is not None:
|
||||
body["service_tier"] = service_tier
|
||||
if parallel_tool_calls is not None:
|
||||
body["parallel_tool_calls"] = parallel_tool_calls
|
||||
|
||||
# Strip body fields a provider's registry entry declares unusable —
|
||||
# reasoning-class models that lock these to fixed defaults (e.g.
|
||||
# Kimi k2.5/k2.6 only accept temperature=1, top_p=1) 400 otherwise.
|
||||
|
|
@ -1186,6 +1234,10 @@ class ExternalProviderClient:
|
|||
anthropic_code_exec_container_id: Optional[str] = None,
|
||||
prompt_cache_ttl: Optional[str] = None,
|
||||
compaction_threshold: Optional[int] = None,
|
||||
*,
|
||||
stop: Optional[Union[str, list[str]]] = None,
|
||||
service_tier: Optional[str] = None,
|
||||
parallel_tool_calls: Optional[bool] = None,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""
|
||||
Call the Anthropic Messages API and translate its SSE to OpenAI format.
|
||||
|
|
@ -1347,6 +1399,29 @@ class ExternalProviderClient:
|
|||
body["temperature"] = temperature
|
||||
if top_k is not None and top_k > 0 and not sampling_removed:
|
||||
body["top_k"] = top_k
|
||||
|
||||
# Optional sampling extensions. Anthropic has no
|
||||
# frequency_penalty / seed / logprobs equivalents, so those are
|
||||
# silently dropped by virtue of not being forwarded from
|
||||
# stream_chat_completion. The three knobs Anthropic does accept
|
||||
# land here:
|
||||
# stop → stop_sequences (renamed)
|
||||
# service_tier → service_tier (auto|standard_only only)
|
||||
# parallel_tool_calls → disable_parallel_tool_use (inverted)
|
||||
if stop is not None:
|
||||
sequences: list[str]
|
||||
if isinstance(stop, str):
|
||||
sequences = [stop] if stop else []
|
||||
else:
|
||||
sequences = [s for s in stop if isinstance(s, str) and s]
|
||||
if sequences:
|
||||
body["stop_sequences"] = sequences
|
||||
if service_tier in ("auto", "standard_only"):
|
||||
body["service_tier"] = service_tier
|
||||
if parallel_tool_calls is False:
|
||||
# Default upstream behavior is parallel-allowed; only
|
||||
# forward when the user explicitly disabled it.
|
||||
body["disable_parallel_tool_use"] = True
|
||||
# Anthropic only caches a prefix when at least one cache_control
|
||||
# marker is attached to it — the frontend defaults
|
||||
# enable_prompt_caching to True for Anthropic, so treat `None` the
|
||||
|
|
@ -2558,6 +2633,9 @@ class ExternalProviderClient:
|
|||
enable_prompt_caching: Optional[bool] = None,
|
||||
openai_code_exec_container_id: Optional[str] = None,
|
||||
compaction_threshold: Optional[int] = None,
|
||||
*,
|
||||
service_tier: Optional[str] = None,
|
||||
parallel_tool_calls: Optional[bool] = None,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""
|
||||
Call OpenAI's /v1/responses endpoint and translate its SSE stream back
|
||||
|
|
@ -2665,6 +2743,15 @@ class ExternalProviderClient:
|
|||
"input": input_items,
|
||||
"stream": True,
|
||||
}
|
||||
# Responses accepts service_tier on the same enum set as Chat
|
||||
# Completions minus `scale`. parallel_tool_calls follows the
|
||||
# same shape (default true). The frontend capability gate
|
||||
# (provider-capabilities.ts) already filters the option lists
|
||||
# per provider, so we just forward what we got.
|
||||
if service_tier in ("auto", "default", "flex", "priority"):
|
||||
body["service_tier"] = service_tier
|
||||
if parallel_tool_calls is not None:
|
||||
body["parallel_tool_calls"] = bool(parallel_tool_calls)
|
||||
# `summary: "auto"` is what makes /v1/responses emit reasoning
|
||||
# summary events — without it OpenAI returns no thinking text on
|
||||
# most reasoning models, the SSE handler has no <think>…</think>
|
||||
|
|
|
|||
|
|
@ -786,6 +786,48 @@ class ChatCompletionRequest(BaseModel):
|
|||
"to auto-create."
|
||||
),
|
||||
)
|
||||
frequency_penalty: Optional[float] = Field(
|
||||
None,
|
||||
ge = -2.0,
|
||||
le = 2.0,
|
||||
description = (
|
||||
"OpenAI Chat Completions frequency penalty (-2.0 to 2.0). "
|
||||
"Forwarded only on providers that accept it; Anthropic "
|
||||
"Messages and the OpenAI Responses family silently drop it."
|
||||
),
|
||||
)
|
||||
seed: Optional[int] = Field(
|
||||
None,
|
||||
description = (
|
||||
"Best-effort determinism seed. Forwarded to OpenAI Chat "
|
||||
"Completions and OpenAI-compatible local backends. The "
|
||||
"Responses family rejects it server-side and Anthropic does "
|
||||
"not implement it, so it is silently dropped on those routes."
|
||||
),
|
||||
)
|
||||
service_tier: Optional[
|
||||
Literal["auto", "default", "flex", "priority", "scale", "standard_only"]
|
||||
] = Field(
|
||||
None,
|
||||
description = (
|
||||
"Provider service tier. Anthropic accepts only `auto` and "
|
||||
"`standard_only`; OpenAI Chat accepts "
|
||||
"`auto|default|flex|priority|scale`; OpenAI Responses accepts "
|
||||
"`auto|default|flex|priority`. Unsupported values per provider "
|
||||
"are dropped in the per-provider stream helper instead of "
|
||||
"422'ing here so a stale frontend never breaks a request."
|
||||
),
|
||||
)
|
||||
parallel_tool_calls: Optional[bool] = Field(
|
||||
None,
|
||||
description = (
|
||||
"Whether the provider may dispatch tool calls in parallel. "
|
||||
"OpenAI: forwarded as `parallel_tool_calls`. Anthropic: "
|
||||
"inverted into `disable_parallel_tool_use` on the Messages "
|
||||
"body. Default `None` preserves each provider's upstream "
|
||||
"default (which is `true` everywhere today)."
|
||||
),
|
||||
)
|
||||
|
||||
@model_validator(mode = "after")
|
||||
def _resolve_missing_tool_call_ids(self) -> "ChatCompletionRequest":
|
||||
|
|
|
|||
|
|
@ -1868,6 +1868,11 @@ async def _proxy_to_external_provider(
|
|||
anthropic_code_exec_container_id = payload.anthropic_code_exec_container_id,
|
||||
prompt_cache_ttl = payload.prompt_cache_ttl,
|
||||
compaction_threshold = payload.compaction_threshold,
|
||||
frequency_penalty = payload.frequency_penalty,
|
||||
seed = payload.seed,
|
||||
stop = payload.stop,
|
||||
service_tier = payload.service_tier,
|
||||
parallel_tool_calls = payload.parallel_tool_calls,
|
||||
stream = payload.stream,
|
||||
)
|
||||
try:
|
||||
|
|
|
|||
385
studio/backend/tests/test_sampling_params_routing.py
Normal file
385
studio/backend/tests/test_sampling_params_routing.py
Normal file
|
|
@ -0,0 +1,385 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
"""End-to-end routing tests for the new sampling parameters.
|
||||
|
||||
Pins the per-provider gating contract added by the
|
||||
expose-sampling-params PR: each of `frequency_penalty`, `seed`, `stop`
|
||||
/ `stop_sequences`, `service_tier`, `parallel_tool_calls` only appears
|
||||
on the outbound body when the upstream provider actually accepts it.
|
||||
|
||||
The provider matrix is captured per docs:
|
||||
- Anthropic Messages: accepts stop_sequences, service_tier
|
||||
(auto|standard_only), disable_parallel_tool_use (inverted). REJECTS
|
||||
frequency_penalty, seed, logprobs (silently dropped client-side).
|
||||
- OpenAI Chat Completions (default OAI-compat branch): accepts every
|
||||
field; OpenAI cloud uses `max_completion_tokens` rather than
|
||||
`max_tokens`.
|
||||
- OpenAI Responses (gpt-5.x / o3): rejects temperature, top_p,
|
||||
frequency_penalty, seed, stop, logprobs. Accepts service_tier
|
||||
(auto|default|flex|priority) and parallel_tool_calls.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from core.inference import external_provider as ep_mod
|
||||
from core.inference.external_provider import ExternalProviderClient
|
||||
|
||||
|
||||
def _drive(coro):
|
||||
return asyncio.new_event_loop().run_until_complete(coro)
|
||||
|
||||
|
||||
def _install_mock(monkeypatch, *, sse_payload: bytes | None = None) -> dict:
|
||||
captured: dict = {}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
try:
|
||||
captured["body"] = json.loads(request.content.decode("utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
captured["body"] = None
|
||||
captured["url"] = str(request.url)
|
||||
captured["headers"] = dict(request.headers)
|
||||
return httpx.Response(
|
||||
200,
|
||||
content = sse_payload
|
||||
or (b"event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"),
|
||||
headers = {"content-type": "text/event-stream"},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
ep_mod,
|
||||
"_http_client",
|
||||
httpx.AsyncClient(transport = httpx.MockTransport(handler)),
|
||||
)
|
||||
return captured
|
||||
|
||||
|
||||
# ── Anthropic ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _drive_anthropic(captured, **kwargs) -> dict:
|
||||
async def run():
|
||||
client = ExternalProviderClient(
|
||||
provider_type = "anthropic",
|
||||
base_url = "https://api.anthropic.com/v1",
|
||||
api_key = "sk-ant-test",
|
||||
)
|
||||
async for _ in client.stream_chat_completion(
|
||||
messages = [{"role": "user", "content": "hi"}],
|
||||
model = "claude-opus-4-7",
|
||||
temperature = 0.7,
|
||||
top_p = 0.95,
|
||||
max_tokens = 64,
|
||||
**kwargs,
|
||||
):
|
||||
pass
|
||||
await client.close()
|
||||
|
||||
_drive(run())
|
||||
return captured["body"]
|
||||
|
||||
|
||||
def test_anthropic_stop_sequences_forwarded_as_renamed_field(monkeypatch):
|
||||
captured = _install_mock(monkeypatch)
|
||||
body = _drive_anthropic(captured, stop = ["END", "DONE"])
|
||||
assert body.get("stop_sequences") == ["END", "DONE"], body
|
||||
# Anthropic does not have a `stop` field; the unrenamed key must not appear.
|
||||
assert "stop" not in body, body
|
||||
|
||||
|
||||
def test_anthropic_single_string_stop_is_wrapped(monkeypatch):
|
||||
captured = _install_mock(monkeypatch)
|
||||
body = _drive_anthropic(captured, stop = "STOPHERE")
|
||||
assert body.get("stop_sequences") == ["STOPHERE"], body
|
||||
|
||||
|
||||
def test_anthropic_empty_stop_omitted(monkeypatch):
|
||||
captured = _install_mock(monkeypatch)
|
||||
body = _drive_anthropic(captured, stop = [])
|
||||
assert "stop_sequences" not in body, body
|
||||
assert "stop" not in body, body
|
||||
|
||||
|
||||
def test_anthropic_service_tier_forwarded_when_valid(monkeypatch):
|
||||
captured = _install_mock(monkeypatch)
|
||||
body = _drive_anthropic(captured, service_tier = "standard_only")
|
||||
assert body.get("service_tier") == "standard_only", body
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bogus", ["flex", "priority", "scale", "default", "", "auto-foo"])
|
||||
def test_anthropic_service_tier_unsupported_values_dropped(monkeypatch, bogus):
|
||||
captured = _install_mock(monkeypatch)
|
||||
body = _drive_anthropic(captured, service_tier = bogus)
|
||||
assert "service_tier" not in body, body
|
||||
|
||||
|
||||
def test_anthropic_disable_parallel_tool_use_only_when_false(monkeypatch):
|
||||
captured = _install_mock(monkeypatch)
|
||||
body = _drive_anthropic(captured, parallel_tool_calls = False)
|
||||
assert body.get("disable_parallel_tool_use") is True, body
|
||||
# Anthropic has no `parallel_tool_calls` field.
|
||||
assert "parallel_tool_calls" not in body, body
|
||||
|
||||
|
||||
def test_anthropic_parallel_tool_calls_default_not_sent(monkeypatch):
|
||||
captured = _install_mock(monkeypatch)
|
||||
body = _drive_anthropic(captured, parallel_tool_calls = True)
|
||||
# True is the upstream default; do not surface
|
||||
# `disable_parallel_tool_use: false` which would over-specify the request.
|
||||
assert "disable_parallel_tool_use" not in body, body
|
||||
assert "parallel_tool_calls" not in body, body
|
||||
|
||||
|
||||
def test_anthropic_rejects_openai_only_knobs(monkeypatch):
|
||||
"""frequency_penalty / seed are dropped at the dispatch layer.
|
||||
|
||||
Anthropic has no equivalent; the keyword args are not even forwarded
|
||||
from stream_chat_completion to _stream_anthropic. This test pins
|
||||
that no such field reaches the Messages body.
|
||||
"""
|
||||
captured = _install_mock(monkeypatch)
|
||||
body = _drive_anthropic(
|
||||
captured,
|
||||
frequency_penalty = 1.5,
|
||||
seed = 42,
|
||||
)
|
||||
assert "frequency_penalty" not in body, body
|
||||
assert "seed" not in body, body
|
||||
|
||||
|
||||
# ── OpenAI Chat Completions (default OAI-compat) ─────────────────────────
|
||||
|
||||
|
||||
def _drive_openai_compat(captured, **kwargs) -> dict:
|
||||
"""Send through the default OAI-compat branch (NOT /v1/responses).
|
||||
|
||||
Use a non-OpenAI provider_type so the dispatcher takes the default
|
||||
branch at the bottom of stream_chat_completion rather than the
|
||||
Responses translator path that routes provider_type=="openai".
|
||||
"""
|
||||
|
||||
async def run():
|
||||
client = ExternalProviderClient(
|
||||
provider_type = "mistral",
|
||||
base_url = "https://api.mistral.ai/v1",
|
||||
api_key = "test-key",
|
||||
)
|
||||
# mistral's OpenAI-compat /v1/chat/completions returns OpenAI
|
||||
# SSE; a single DONE frame is enough to drain the stream.
|
||||
async for _ in client.stream_chat_completion(
|
||||
messages = [{"role": "user", "content": "hi"}],
|
||||
model = "mistral-small-latest",
|
||||
temperature = 0.5,
|
||||
top_p = 0.9,
|
||||
max_tokens = 64,
|
||||
**kwargs,
|
||||
):
|
||||
pass
|
||||
await client.close()
|
||||
|
||||
_drive(run())
|
||||
return captured["body"]
|
||||
|
||||
|
||||
def _oai_done_payload() -> bytes:
|
||||
return b"data: [DONE]\n\n"
|
||||
|
||||
|
||||
def test_openai_compat_forwards_frequency_penalty(monkeypatch):
|
||||
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
||||
body = _drive_openai_compat(captured, frequency_penalty = 1.25)
|
||||
assert body.get("frequency_penalty") == 1.25, body
|
||||
|
||||
|
||||
def test_openai_compat_forwards_seed(monkeypatch):
|
||||
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
||||
body = _drive_openai_compat(captured, seed = 12345)
|
||||
assert body.get("seed") == 12345, body
|
||||
|
||||
|
||||
def test_openai_compat_forwards_stop_array(monkeypatch):
|
||||
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
||||
body = _drive_openai_compat(captured, stop = ["END", "DONE"])
|
||||
assert body.get("stop") == ["END", "DONE"], body
|
||||
# The default OAI-compat branch does not rename to stop_sequences.
|
||||
assert "stop_sequences" not in body, body
|
||||
|
||||
|
||||
def test_openai_compat_truncates_stop_to_four(monkeypatch):
|
||||
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
||||
body = _drive_openai_compat(
|
||||
captured, stop = ["a", "b", "c", "d", "e", "f"]
|
||||
)
|
||||
assert body.get("stop") == ["a", "b", "c", "d"], body
|
||||
|
||||
|
||||
def test_openai_compat_empty_stop_omitted(monkeypatch):
|
||||
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
||||
body = _drive_openai_compat(captured, stop = [])
|
||||
assert "stop" not in body, body
|
||||
|
||||
|
||||
def test_openai_compat_forwards_service_tier(monkeypatch):
|
||||
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
||||
body = _drive_openai_compat(captured, service_tier = "flex")
|
||||
assert body.get("service_tier") == "flex", body
|
||||
|
||||
|
||||
def test_openai_compat_forwards_parallel_tool_calls(monkeypatch):
|
||||
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
||||
body = _drive_openai_compat(captured, parallel_tool_calls = False)
|
||||
assert body.get("parallel_tool_calls") is False, body
|
||||
|
||||
|
||||
def test_openai_compat_omits_unset_optionals(monkeypatch):
|
||||
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
||||
body = _drive_openai_compat(captured)
|
||||
# Optional knobs default to None / unset -> never appear.
|
||||
assert "frequency_penalty" not in body, body
|
||||
assert "seed" not in body, body
|
||||
assert "stop" not in body, body
|
||||
assert "service_tier" not in body, body
|
||||
assert "parallel_tool_calls" not in body, body
|
||||
|
||||
|
||||
# ── OpenAI Responses (gpt-5.x via /v1/responses) ─────────────────────────
|
||||
|
||||
|
||||
def _responses_done_payload() -> bytes:
|
||||
return (
|
||||
b"event: response.completed\n"
|
||||
b"data: {\"type\":\"response.completed\",\"response\":{\"usage\":{}}}\n\n"
|
||||
)
|
||||
|
||||
|
||||
def _drive_openai_responses(captured, **kwargs) -> dict:
|
||||
async def run():
|
||||
client = ExternalProviderClient(
|
||||
provider_type = "openai",
|
||||
base_url = "https://api.openai.com/v1",
|
||||
api_key = "sk-test",
|
||||
)
|
||||
async for _ in client.stream_chat_completion(
|
||||
messages = [{"role": "user", "content": "hi"}],
|
||||
model = "gpt-5.5",
|
||||
temperature = 1.0,
|
||||
top_p = 1.0,
|
||||
max_tokens = 64,
|
||||
**kwargs,
|
||||
):
|
||||
pass
|
||||
await client.close()
|
||||
|
||||
_drive(run())
|
||||
return captured["body"]
|
||||
|
||||
|
||||
def test_openai_responses_drops_temperature_top_p(monkeypatch):
|
||||
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
||||
body = _drive_openai_responses(captured)
|
||||
assert "temperature" not in body, body
|
||||
assert "top_p" not in body, body
|
||||
|
||||
|
||||
def test_openai_responses_drops_frequency_penalty_seed_stop(monkeypatch):
|
||||
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
||||
body = _drive_openai_responses(
|
||||
captured,
|
||||
frequency_penalty = 1.5,
|
||||
seed = 99,
|
||||
stop = ["END"],
|
||||
)
|
||||
# Responses 400s on any of these; the dispatch must drop them
|
||||
# before they hit the wire.
|
||||
assert "frequency_penalty" not in body, body
|
||||
assert "seed" not in body, body
|
||||
assert "stop" not in body, body
|
||||
assert "stop_sequences" not in body, body
|
||||
|
||||
|
||||
def test_openai_responses_forwards_service_tier(monkeypatch):
|
||||
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
||||
body = _drive_openai_responses(captured, service_tier = "priority")
|
||||
assert body.get("service_tier") == "priority", body
|
||||
|
||||
|
||||
def test_openai_responses_rejects_chat_only_service_tier(monkeypatch):
|
||||
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
||||
body = _drive_openai_responses(captured, service_tier = "scale")
|
||||
# Responses only accepts auto|default|flex|priority -- `scale` is
|
||||
# silently dropped so a stale frontend cannot 400 the request.
|
||||
assert "service_tier" not in body, body
|
||||
|
||||
|
||||
def test_openai_responses_forwards_parallel_tool_calls(monkeypatch):
|
||||
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
||||
body = _drive_openai_responses(captured, parallel_tool_calls = False)
|
||||
assert body.get("parallel_tool_calls") is False, body
|
||||
|
||||
|
||||
def test_openai_responses_omits_unset_optionals(monkeypatch):
|
||||
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
||||
body = _drive_openai_responses(captured)
|
||||
assert "service_tier" not in body, body
|
||||
assert "parallel_tool_calls" not in body, body
|
||||
|
||||
|
||||
# ── Schema-level smoke tests ─────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_chat_completion_request_accepts_new_sampling_fields():
|
||||
from models.inference import ChatCompletionRequest
|
||||
|
||||
payload = ChatCompletionRequest.model_validate(
|
||||
{
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"frequency_penalty": -1.0,
|
||||
"seed": 0,
|
||||
"stop": ["END"],
|
||||
"service_tier": "auto",
|
||||
"parallel_tool_calls": True,
|
||||
}
|
||||
)
|
||||
assert payload.frequency_penalty == -1.0
|
||||
assert payload.seed == 0
|
||||
assert payload.stop == ["END"]
|
||||
assert payload.service_tier == "auto"
|
||||
assert payload.parallel_tool_calls is True
|
||||
|
||||
|
||||
def test_chat_completion_request_rejects_bad_service_tier():
|
||||
import pydantic
|
||||
from models.inference import ChatCompletionRequest
|
||||
|
||||
with pytest.raises(pydantic.ValidationError):
|
||||
ChatCompletionRequest.model_validate(
|
||||
{
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"service_tier": "bogus",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_chat_completion_request_clamps_frequency_penalty_range():
|
||||
import pydantic
|
||||
from models.inference import ChatCompletionRequest
|
||||
|
||||
with pytest.raises(pydantic.ValidationError):
|
||||
ChatCompletionRequest.model_validate(
|
||||
{
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"frequency_penalty": 3.0,
|
||||
}
|
||||
)
|
||||
with pytest.raises(pydantic.ValidationError):
|
||||
ChatCompletionRequest.model_validate(
|
||||
{
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"frequency_penalty": -3.0,
|
||||
}
|
||||
)
|
||||
117
studio/frontend/src/components/ui/stop-sequences-input.tsx
Normal file
117
studio/frontend/src/components/ui/stop-sequences-input.tsx
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { XIcon } from "lucide-react";
|
||||
import { type KeyboardEvent, useState } from "react";
|
||||
|
||||
/**
|
||||
* Chips editor for the `stop` / `stop_sequences` array.
|
||||
*
|
||||
* Commit a chip with Enter or comma; press Backspace on an empty input
|
||||
* to delete the most recent chip. Caps at `maxEntries` -- OpenAI Chat
|
||||
* Completions documents a hard cap of 4 stop sequences; Anthropic
|
||||
* Messages accepts arbitrarily many. Pass `Infinity` to disable the cap.
|
||||
*/
|
||||
export interface StopSequencesInputProps {
|
||||
value: string[];
|
||||
onChange: (next: string[]) => void;
|
||||
maxEntries?: number;
|
||||
disabled?: boolean;
|
||||
placeholder?: string;
|
||||
className?: string;
|
||||
"aria-label"?: string;
|
||||
}
|
||||
|
||||
export function StopSequencesInput({
|
||||
value,
|
||||
onChange,
|
||||
maxEntries = 4,
|
||||
disabled,
|
||||
placeholder = "Add stop sequence",
|
||||
className,
|
||||
"aria-label": ariaLabel,
|
||||
}: StopSequencesInputProps) {
|
||||
const [draft, setDraft] = useState("");
|
||||
const atCap = value.length >= maxEntries;
|
||||
|
||||
function commitDraft() {
|
||||
const trimmed = draft.trim();
|
||||
if (!trimmed) return;
|
||||
if (atCap) return;
|
||||
if (value.includes(trimmed)) {
|
||||
setDraft("");
|
||||
return;
|
||||
}
|
||||
onChange([...value, trimmed]);
|
||||
setDraft("");
|
||||
}
|
||||
|
||||
function removeChip(index: number) {
|
||||
if (disabled) return;
|
||||
onChange(value.filter((_, i) => i !== index));
|
||||
}
|
||||
|
||||
function handleKeyDown(event: KeyboardEvent<HTMLInputElement>) {
|
||||
if (disabled) return;
|
||||
if (event.key === "Enter" || event.key === ",") {
|
||||
event.preventDefault();
|
||||
commitDraft();
|
||||
return;
|
||||
}
|
||||
if (event.key === "Backspace" && !draft && value.length > 0) {
|
||||
event.preventDefault();
|
||||
onChange(value.slice(0, -1));
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
data-slot="stop-sequences-input"
|
||||
className={cn(
|
||||
"flex flex-wrap items-center gap-1.5 rounded-md border border-input bg-transparent px-2 py-1.5 text-sm",
|
||||
"focus-within:border-ring focus-within:ring-[1px] focus-within:ring-ring/40",
|
||||
disabled && "cursor-not-allowed opacity-60",
|
||||
className,
|
||||
)}
|
||||
aria-label={ariaLabel}
|
||||
>
|
||||
{value.map((entry, index) => (
|
||||
<Badge
|
||||
// Stop sequences are not guaranteed unique across edits (the
|
||||
// user could enter "END" twice if the previous one was just
|
||||
// deleted), so combine value + index for a stable key.
|
||||
key={`${entry}-${index}`}
|
||||
variant="secondary"
|
||||
className="gap-1 pl-2 pr-1"
|
||||
>
|
||||
<span className="font-mono">{entry}</span>
|
||||
{!disabled ? (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => removeChip(index)}
|
||||
className="ml-0.5 rounded-full p-0.5 text-muted-foreground transition-colors hover:bg-muted hover:text-foreground"
|
||||
aria-label={`Remove stop sequence ${entry}`}
|
||||
>
|
||||
<XIcon className="size-3" />
|
||||
</button>
|
||||
) : null}
|
||||
</Badge>
|
||||
))}
|
||||
<Input
|
||||
value={draft}
|
||||
onChange={(event) => setDraft(event.target.value)}
|
||||
onKeyDown={handleKeyDown}
|
||||
onBlur={commitDraft}
|
||||
placeholder={atCap ? `Max ${maxEntries} stops` : placeholder}
|
||||
disabled={disabled || atCap}
|
||||
className={cn(
|
||||
"h-6 min-w-[8ch] flex-1 border-0 bg-transparent p-0 text-sm shadow-none",
|
||||
"focus-visible:ring-0 focus-visible:border-0",
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -1403,6 +1403,29 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
...(externalCapabilities?.presencePenalty
|
||||
? { presence_penalty: params.presencePenalty }
|
||||
: {}),
|
||||
// Optional sampling extensions. Each gate is per-provider
|
||||
// (see provider-capabilities.ts); the backend additionally
|
||||
// drops fields the upstream API does not accept, so a
|
||||
// stale frontend cannot 400 the request.
|
||||
...(externalCapabilities?.frequencyPenalty
|
||||
? { frequency_penalty: params.frequencyPenalty }
|
||||
: {}),
|
||||
...(externalCapabilities?.seed && params.seed !== null
|
||||
? { seed: params.seed }
|
||||
: {}),
|
||||
...(externalCapabilities?.stop && params.stop.length > 0
|
||||
? { stop: params.stop }
|
||||
: {}),
|
||||
...(externalCapabilities?.serviceTier && params.serviceTier
|
||||
? { service_tier: params.serviceTier }
|
||||
: {}),
|
||||
// Forward parallel_tool_calls when the user explicitly
|
||||
// turned it off (the upstream default is `true` on every
|
||||
// provider we ship, so default true is a no-op).
|
||||
...(externalCapabilities?.parallelToolCalls &&
|
||||
params.parallelToolCalls === false
|
||||
? { parallel_tool_calls: false }
|
||||
: {}),
|
||||
// Built-in tools: Search pill maps to provider-side
|
||||
// web_search (currently OpenAI / Anthropic / OpenRouter /
|
||||
// Kimi); Code pill maps to Anthropic's server-side
|
||||
|
|
@ -1503,6 +1526,18 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
min_p: params.minP,
|
||||
repetition_penalty: params.repetitionPenalty,
|
||||
presence_penalty: params.presencePenalty,
|
||||
// Optional sampling extensions; local llama-server already
|
||||
// accepts `stop` / `seed` / `frequency_penalty` via
|
||||
// _build_passthrough_payload (routes/inference.py:4884) and
|
||||
// silently ignores fields it does not recognise.
|
||||
...(params.frequencyPenalty !== 0
|
||||
? { frequency_penalty: params.frequencyPenalty }
|
||||
: {}),
|
||||
...(params.seed !== null ? { seed: params.seed } : {}),
|
||||
...(params.stop.length > 0 ? { stop: params.stop } : {}),
|
||||
...(params.parallelToolCalls === false
|
||||
? { parallel_tool_calls: false }
|
||||
: {}),
|
||||
image_base64: imageBase64,
|
||||
audio_base64: audioBase64,
|
||||
cancel_id: cancelId,
|
||||
|
|
|
|||
|
|
@ -86,10 +86,13 @@ import {
|
|||
EXTERNAL_MAX_OUTPUT_TOKENS,
|
||||
type ProviderCapabilities,
|
||||
getExternalMinOutputTokens,
|
||||
getServiceTierOptions,
|
||||
providerSupportsBuiltinCodeExecution,
|
||||
} from "./provider-capabilities";
|
||||
import { StopSequencesInput } from "@/components/ui/stop-sequences-input";
|
||||
import { useChatRuntimeStore } from "./stores/chat-runtime-store";
|
||||
import type { InferenceParams } from "./types/runtime";
|
||||
import type { InferenceParams, ServiceTier } from "./types/runtime";
|
||||
import { Input } from "@/components/ui/input";
|
||||
|
||||
export { defaultInferenceParams, type Preset } from "./presets/preset-policy";
|
||||
export type { InferenceParams } from "./types/runtime";
|
||||
|
|
@ -416,6 +419,19 @@ export function ChatSettingsPanel({
|
|||
!isExternalModel || Boolean(providerCapabilities?.repetitionPenalty);
|
||||
const showPresencePenalty =
|
||||
!isExternalModel || Boolean(providerCapabilities?.presencePenalty);
|
||||
const showFrequencyPenalty =
|
||||
!isExternalModel || Boolean(providerCapabilities?.frequencyPenalty);
|
||||
const showSeed = !isExternalModel || Boolean(providerCapabilities?.seed);
|
||||
const showStop = !isExternalModel || Boolean(providerCapabilities?.stop);
|
||||
const showServiceTier =
|
||||
isExternalModel && Boolean(providerCapabilities?.serviceTier);
|
||||
const showParallelToolCalls =
|
||||
!isExternalModel || Boolean(providerCapabilities?.parallelToolCalls);
|
||||
// OpenAI Chat docs cap `stop` at 4 entries; Anthropic accepts more.
|
||||
// Pick the right ceiling per active connection so the chips editor's
|
||||
// placeholder doesn't lie.
|
||||
const stopMaxEntries = externalProviderType === "anthropic" ? 16 : 4;
|
||||
const serviceTierOptions = getServiceTierOptions(externalProviderType);
|
||||
const isMobile = useIsMobile();
|
||||
const isGguf = useChatRuntimeStore((s) => s.activeGgufVariant) != null;
|
||||
const hasModelContent =
|
||||
|
|
@ -1259,6 +1275,145 @@ export function ChatSettingsPanel({
|
|||
info="Penalizes any token that has already appeared at least once, encouraging the model to introduce new topics. 0 = off."
|
||||
/>
|
||||
) : null}
|
||||
{showFrequencyPenalty ? (
|
||||
<ParamSlider
|
||||
label="Frequency Penalty"
|
||||
value={params.frequencyPenalty}
|
||||
min={-2}
|
||||
max={2}
|
||||
step={0.1}
|
||||
onChange={set("frequencyPenalty")}
|
||||
displayValue={
|
||||
params.frequencyPenalty === 0 ? "Off" : undefined
|
||||
}
|
||||
info="Down-weights tokens proportionally to how often they have already appeared. Negative values encourage repetition. 0 = off. OpenAI Chat Completions only; Anthropic and the OpenAI Responses family ignore it."
|
||||
/>
|
||||
) : null}
|
||||
{showSeed ? (
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<div className="flex min-w-0 items-center gap-1.5">
|
||||
<span className="min-w-0 text-[13px] font-medium leading-[1.25] tracking-nav text-nav-fg">
|
||||
Seed
|
||||
</span>
|
||||
<InfoHint>
|
||||
Best-effort determinism seed. Same seed + prompt =
|
||||
same output (approximately). OpenAI Chat Completions
|
||||
and OpenAI-compat local backends honor it; OpenAI
|
||||
Responses and Anthropic silently drop it.
|
||||
</InfoHint>
|
||||
</div>
|
||||
<Input
|
||||
type="number"
|
||||
inputMode="numeric"
|
||||
value={params.seed ?? ""}
|
||||
placeholder="Random"
|
||||
onChange={(event) => {
|
||||
const raw = event.target.value;
|
||||
if (raw === "") {
|
||||
set("seed")(null);
|
||||
return;
|
||||
}
|
||||
const parsed = Number.parseInt(raw, 10);
|
||||
if (Number.isFinite(parsed)) {
|
||||
set("seed")(parsed);
|
||||
}
|
||||
}}
|
||||
className="h-8 w-[124px] shrink-0 text-right font-mono text-xs"
|
||||
aria-label="Seed"
|
||||
/>
|
||||
</div>
|
||||
) : null}
|
||||
{showStop ? (
|
||||
<div className="flex flex-col gap-1.5">
|
||||
<div className="flex min-w-0 items-center gap-1.5">
|
||||
<span className="min-w-0 text-[13px] font-medium leading-[1.25] tracking-nav text-nav-fg">
|
||||
Stop sequences
|
||||
</span>
|
||||
<InfoHint>
|
||||
Strings that halt generation as soon as the model
|
||||
emits them. Enter a value and press Enter or comma
|
||||
to commit a chip. Backend translates to
|
||||
`stop_sequences` on Anthropic and `stop` on OpenAI
|
||||
Chat (capped at 4 entries).
|
||||
</InfoHint>
|
||||
</div>
|
||||
<StopSequencesInput
|
||||
value={params.stop}
|
||||
onChange={set("stop")}
|
||||
maxEntries={stopMaxEntries}
|
||||
aria-label="Stop sequences"
|
||||
/>
|
||||
</div>
|
||||
) : null}
|
||||
{showServiceTier ? (
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<div className="flex min-w-0 items-center gap-1.5">
|
||||
<span className="min-w-0 text-[13px] font-medium leading-[1.25] tracking-nav text-nav-fg">
|
||||
Service tier
|
||||
</span>
|
||||
<InfoHint>
|
||||
Provider routing tier. `auto` (default) lets the
|
||||
provider choose. `flex` / `priority` / `scale` route
|
||||
to higher-latency-tolerant or premium queues on
|
||||
OpenAI; `standard_only` opts out of Anthropic's
|
||||
priority tier.
|
||||
</InfoHint>
|
||||
</div>
|
||||
<Select
|
||||
value={params.serviceTier ?? "auto"}
|
||||
onValueChange={(value) => {
|
||||
if (value === "auto") {
|
||||
set("serviceTier")(null);
|
||||
return;
|
||||
}
|
||||
const allowed: readonly ServiceTier[] = serviceTierOptions;
|
||||
if (allowed.includes(value as ServiceTier)) {
|
||||
set("serviceTier")(value as ServiceTier);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<SelectTrigger
|
||||
className="panel-select-trigger h-8 w-[140px] shrink-0"
|
||||
aria-label="Service tier"
|
||||
>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{serviceTierOptions.map((option) => (
|
||||
<SelectItem key={option} value={option}>
|
||||
{option}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
) : null}
|
||||
{showParallelToolCalls ? (
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<div className="flex min-w-0 items-center gap-1.5">
|
||||
<span className="min-w-0 text-[13px] font-medium leading-[1.25] tracking-nav text-nav-fg">
|
||||
Parallel tool calls
|
||||
</span>
|
||||
<InfoHint>
|
||||
When on, the model may dispatch multiple tool calls
|
||||
in a single turn (default). Turn off to force one
|
||||
tool call at a time. Anthropic implements this as
|
||||
`disable_parallel_tool_use`; OpenAI as
|
||||
`parallel_tool_calls`.
|
||||
</InfoHint>
|
||||
</div>
|
||||
<Switch
|
||||
className="panel-switch shrink-0"
|
||||
checked={params.parallelToolCalls}
|
||||
onCheckedChange={set("parallelToolCalls")}
|
||||
aria-label={
|
||||
params.parallelToolCalls
|
||||
? "Disable parallel tool calls"
|
||||
: "Enable parallel tool calls"
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
) : null}
|
||||
{!isExternalModel && !isGguf && (
|
||||
<ParamSlider
|
||||
label="Max Seq Length"
|
||||
|
|
|
|||
|
|
@ -29,6 +29,62 @@ export interface ProviderCapabilities {
|
|||
repetitionPenalty: boolean;
|
||||
/** OpenAI-style presence penalty. */
|
||||
presencePenalty: boolean;
|
||||
/**
|
||||
* OpenAI-style frequency penalty. Accepted by Chat Completions only.
|
||||
* Anthropic and the OpenAI Responses family both reject it (the latter
|
||||
* with `Unsupported parameter`).
|
||||
*/
|
||||
frequencyPenalty: boolean;
|
||||
/**
|
||||
* Best-effort determinism seed. Accepted by OpenAI Chat Completions and
|
||||
* most OpenAI-compatible local backends (vLLM, llama.cpp). Rejected by
|
||||
* the Responses family and silently dropped by Anthropic.
|
||||
*/
|
||||
seed: boolean;
|
||||
/**
|
||||
* Custom stop sequences. Maps to `stop` (OpenAI Chat) or `stop_sequences`
|
||||
* (Anthropic). Not accepted by the Responses family.
|
||||
*/
|
||||
stop: boolean;
|
||||
/**
|
||||
* Provider service tier (`auto` / `standard_only` for Anthropic,
|
||||
* `auto`/`default`/`flex`/`priority`(+`scale`) for OpenAI). See
|
||||
* {@link getServiceTierOptions} for the legal values per provider.
|
||||
*/
|
||||
serviceTier: boolean;
|
||||
/**
|
||||
* Whether the provider supports turning off parallel tool dispatch.
|
||||
* Maps to `parallel_tool_calls: false` on both OpenAI APIs and
|
||||
* `disable_parallel_tool_use: true` on Anthropic (inverted).
|
||||
*/
|
||||
parallelToolCalls: boolean;
|
||||
}
|
||||
|
||||
export type ServiceTierOption =
|
||||
| "auto"
|
||||
| "default"
|
||||
| "flex"
|
||||
| "priority"
|
||||
| "scale"
|
||||
| "standard_only";
|
||||
|
||||
/**
|
||||
* Legal `service_tier` values per provider, sourced from each upstream's
|
||||
* docs. Anthropic exposes only `auto` and `standard_only`; OpenAI Chat
|
||||
* Completions adds `scale` on top of the Responses-family set. Other
|
||||
* providers fall through to a permissive `auto` / `default` pair so the
|
||||
* picker stays usable for OpenAI-compat backends.
|
||||
*/
|
||||
export function getServiceTierOptions(
|
||||
providerType: string | null | undefined,
|
||||
): readonly ServiceTierOption[] {
|
||||
if (providerType === "anthropic") {
|
||||
return ["auto", "standard_only"] as const;
|
||||
}
|
||||
if (providerType === "openai") {
|
||||
return ["auto", "default", "flex", "priority", "scale"] as const;
|
||||
}
|
||||
return ["auto", "default"] as const;
|
||||
}
|
||||
|
||||
export type ExternalReasoningCapabilities = {
|
||||
|
|
@ -286,6 +342,11 @@ const OPENAI_COMPAT_BASE: ProviderCapabilities = {
|
|||
minP: false,
|
||||
repetitionPenalty: false,
|
||||
presencePenalty: true,
|
||||
frequencyPenalty: true,
|
||||
seed: true,
|
||||
stop: true,
|
||||
serviceTier: false,
|
||||
parallelToolCalls: true,
|
||||
};
|
||||
|
||||
const ALL_SUPPORTED: ProviderCapabilities = {
|
||||
|
|
@ -295,6 +356,11 @@ const ALL_SUPPORTED: ProviderCapabilities = {
|
|||
minP: true,
|
||||
repetitionPenalty: true,
|
||||
presencePenalty: true,
|
||||
frequencyPenalty: true,
|
||||
seed: true,
|
||||
stop: true,
|
||||
serviceTier: false,
|
||||
parallelToolCalls: true,
|
||||
};
|
||||
|
||||
const PROVIDER_CAPABILITIES: Record<string, ProviderCapabilities> = {
|
||||
|
|
@ -302,6 +368,8 @@ const PROVIDER_CAPABILITIES: Record<string, ProviderCapabilities> = {
|
|||
// models served via /v1/responses, which rejects temperature, top_p, and
|
||||
// presence/frequency penalty. See backend
|
||||
// external_provider._stream_openai_responses for the proxy.
|
||||
// service_tier and parallel_tool_calls are accepted on /v1/responses;
|
||||
// seed / stop / frequency_penalty are 400'd alongside temperature/top_p.
|
||||
openai: {
|
||||
temperature: false,
|
||||
topP: false,
|
||||
|
|
@ -309,14 +377,22 @@ const PROVIDER_CAPABILITIES: Record<string, ProviderCapabilities> = {
|
|||
minP: false,
|
||||
repetitionPenalty: false,
|
||||
presencePenalty: false,
|
||||
frequencyPenalty: false,
|
||||
seed: false,
|
||||
stop: false,
|
||||
serviceTier: true,
|
||||
parallelToolCalls: true,
|
||||
},
|
||||
// Anthropic's Messages API accepts top_k on 3.x and 4.5/4.6, but Claude
|
||||
// 4.7 (Opus/Sonnet/Haiku) deprecated it and returns 400 if it is set.
|
||||
// We surface top_k in the panel for all Anthropic providers and let the
|
||||
// backend strip it per-model — see _stream_anthropic in
|
||||
// studio/backend/core/inference/external_provider.py.
|
||||
// Presence/frequency penalty is not part of the Messages API on any
|
||||
// Claude generation.
|
||||
// Presence/frequency penalty / seed / logprobs are not part of the
|
||||
// Messages API on any Claude generation. stop_sequences (Anthropic name
|
||||
// for `stop`), service_tier (auto|standard_only), and
|
||||
// disable_parallel_tool_use (inverse of parallel_tool_calls) ARE
|
||||
// supported.
|
||||
anthropic: {
|
||||
temperature: true,
|
||||
topP: true,
|
||||
|
|
@ -324,6 +400,11 @@ const PROVIDER_CAPABILITIES: Record<string, ProviderCapabilities> = {
|
|||
minP: false,
|
||||
repetitionPenalty: false,
|
||||
presencePenalty: false,
|
||||
frequencyPenalty: false,
|
||||
seed: false,
|
||||
stop: true,
|
||||
serviceTier: true,
|
||||
parallelToolCalls: true,
|
||||
},
|
||||
mistral: OPENAI_COMPAT_BASE,
|
||||
gemini: OPENAI_COMPAT_BASE,
|
||||
|
|
@ -340,6 +421,11 @@ const PROVIDER_CAPABILITIES: Record<string, ProviderCapabilities> = {
|
|||
minP: false,
|
||||
repetitionPenalty: false,
|
||||
presencePenalty: true,
|
||||
frequencyPenalty: true,
|
||||
seed: true,
|
||||
stop: true,
|
||||
serviceTier: false,
|
||||
parallelToolCalls: true,
|
||||
},
|
||||
// DeepSeek deprecated presence/frequency penalty in their current docs.
|
||||
deepseek: {
|
||||
|
|
@ -349,6 +435,11 @@ const PROVIDER_CAPABILITIES: Record<string, ProviderCapabilities> = {
|
|||
minP: false,
|
||||
repetitionPenalty: false,
|
||||
presencePenalty: false,
|
||||
frequencyPenalty: false,
|
||||
seed: true,
|
||||
stop: true,
|
||||
serviceTier: false,
|
||||
parallelToolCalls: true,
|
||||
},
|
||||
qwen: OPENAI_COMPAT_BASE,
|
||||
huggingface: OPENAI_COMPAT_BASE,
|
||||
|
|
|
|||
|
|
@ -373,6 +373,11 @@ const PERSISTED_INFERENCE_PARAM_KEYS = [
|
|||
"minP",
|
||||
"repetitionPenalty",
|
||||
"presencePenalty",
|
||||
"frequencyPenalty",
|
||||
"seed",
|
||||
"stop",
|
||||
"serviceTier",
|
||||
"parallelToolCalls",
|
||||
"maxSeqLength",
|
||||
"maxTokens",
|
||||
"systemPrompt",
|
||||
|
|
|
|||
|
|
@ -262,6 +262,41 @@ export interface OpenAIChatCompletionsRequest {
|
|||
* the Anthropic provider with `code_execution` in `enabled_tools`.
|
||||
*/
|
||||
anthropic_code_exec_container_id?: string | null;
|
||||
/**
|
||||
* OpenAI Chat Completions only; rejected by the Responses family and
|
||||
* silently dropped by Anthropic. Range -2.0 .. 2.0.
|
||||
*/
|
||||
frequency_penalty?: number;
|
||||
/**
|
||||
* Best-effort determinism seed. OpenAI Chat / OpenAI-compat backends
|
||||
* forward it; Responses + Anthropic drop it server-side.
|
||||
*/
|
||||
seed?: number;
|
||||
/**
|
||||
* Custom stop sequences. Backend translates to `stop_sequences` for
|
||||
* Anthropic; OpenAI Chat caps at 4 entries (server-side truncates
|
||||
* with a warning). Empty arrays are omitted.
|
||||
*/
|
||||
stop?: string[];
|
||||
/**
|
||||
* Provider service tier. Anthropic accepts `auto|standard_only`;
|
||||
* OpenAI Chat accepts `auto|default|flex|priority|scale`; OpenAI
|
||||
* Responses accepts `auto|default|flex|priority`.
|
||||
*/
|
||||
service_tier?:
|
||||
| "auto"
|
||||
| "default"
|
||||
| "flex"
|
||||
| "priority"
|
||||
| "scale"
|
||||
| "standard_only";
|
||||
/**
|
||||
* Whether the provider may dispatch tool calls in parallel.
|
||||
* OpenAI: forwarded as `parallel_tool_calls`. Anthropic: inverted
|
||||
* into `disable_parallel_tool_use` server-side. Default `undefined`
|
||||
* keeps each provider's upstream default.
|
||||
*/
|
||||
parallel_tool_calls?: boolean;
|
||||
}
|
||||
|
||||
export interface OpenAIChatDelta {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,14 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
export type ServiceTier =
|
||||
| "auto"
|
||||
| "default"
|
||||
| "flex"
|
||||
| "priority"
|
||||
| "scale"
|
||||
| "standard_only";
|
||||
|
||||
export interface InferenceParams {
|
||||
temperature: number;
|
||||
topP: number;
|
||||
|
|
@ -8,6 +16,34 @@ export interface InferenceParams {
|
|||
minP: number;
|
||||
repetitionPenalty: number;
|
||||
presencePenalty: number;
|
||||
/** OpenAI Chat Completions only; rejected by Responses + Anthropic. */
|
||||
frequencyPenalty: number;
|
||||
/**
|
||||
* Best-effort determinism seed. OpenAI Chat Completions only; the
|
||||
* Responses family and Anthropic reject it (silently dropped server-side).
|
||||
* `null` = unset (no `seed` field on the wire).
|
||||
*/
|
||||
seed: number | null;
|
||||
/**
|
||||
* Custom stop sequences. Maps to `stop` on OpenAI Chat Completions and
|
||||
* `stop_sequences` on Anthropic Messages. OpenAI caps the array at 4
|
||||
* entries; backend truncates with a warning. Empty array = unset.
|
||||
*/
|
||||
stop: string[];
|
||||
/**
|
||||
* Provider service tier. Each provider accepts a different enum set;
|
||||
* `getServiceTierOptions(providerType)` resolves the legal values. `null`
|
||||
* means "let the provider pick its default" and is the safe choice on
|
||||
* provider switch.
|
||||
*/
|
||||
serviceTier: ServiceTier | null;
|
||||
/**
|
||||
* Whether the provider may dispatch tool calls in parallel. Maps to
|
||||
* `parallel_tool_calls` on both OpenAI APIs and is inverted into
|
||||
* `disable_parallel_tool_use` for Anthropic. Default true matches the
|
||||
* upstream defaults across all three.
|
||||
*/
|
||||
parallelToolCalls: boolean;
|
||||
maxSeqLength: number;
|
||||
maxTokens: number;
|
||||
systemPrompt: string;
|
||||
|
|
@ -23,6 +59,11 @@ export const DEFAULT_INFERENCE_PARAMS: InferenceParams = {
|
|||
minP: 0.01,
|
||||
repetitionPenalty: 1.0,
|
||||
presencePenalty: 0.0,
|
||||
frequencyPenalty: 0.0,
|
||||
seed: null,
|
||||
stop: [],
|
||||
serviceTier: null,
|
||||
parallelToolCalls: true,
|
||||
maxSeqLength: 4096,
|
||||
maxTokens: 8192,
|
||||
systemPrompt: "",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue