Studio: thread sampling extensions through local + Kimi-search paths
Round-2 round of review-feedback fixes for the sampling-knobs PR: - studio/backend/routes/chat_history.py: ChatInferenceSettings still had the pre-PR field list with extra="forbid", so every settings save the new frontend issued would 422 on the new keys (frequencyPenalty, seed, stop, serviceTier, parallelToolCalls). Add the fields with the same range / enum constraints the chat-completions schema uses, so the settings-persistence path round-trips cleanly. - studio/backend/routes/inference.py: _build_passthrough_payload and _build_openai_passthrough_body now thread frequency_penalty, seed, and parallel_tool_calls through to llama-server. The frontend exposes these knobs for local backends; without the forwarding the UI was a decoration. Each field is gated on `is not None` so 0 / False / "0" still reach the body. - studio/backend/core/inference/external_provider.py: the Kimi $web_search bypass takes an early return into _stream_kimi_web_search before the default OAI-compat body builder runs, so the new sampling fields never landed on Kimi-with-search. Forward them through the helper, with the same dedupe / truncate behavior the main path applies to `stop`. Also extend the OpenAI Responses service_tier allowlist to include `scale` per the live openai-python SDK (response_create_params.py declares Literal["auto","default","flex","scale","priority"]). - studio/frontend/src/features/chat/provider-capabilities.ts + types/runtime.ts: add `scale` to ServiceTier / ServiceTierOption and surface it on the OpenAI Responses options so the UI matches the upstream enum. - studio/backend/tests/test_sampling_params_routing.py: add tests for every gap above: Kimi web-search bypass forwarding, local OpenAI passthrough forwarding, ChatSettingsPayload round-trip, and the full Responses service_tier enum (parametrized over the five accepted values plus a drop check for the Anthropic-only standard_only).
This commit is contained in:
Daniel Han
parent
3ef64c2d65
commit
d6765fddce
6 changed files with 214 additions and 17 deletions
|
|
@ -435,6 +435,11 @@ class ExternalProviderClient:
|
|||
messages,
|
||||
model,
|
||||
max_tokens,
|
||||
frequency_penalty = frequency_penalty,
|
||||
seed = seed,
|
||||
stop = stop,
|
||||
parallel_tool_calls = parallel_tool_calls,
|
||||
presence_penalty = presence_penalty,
|
||||
):
|
||||
yield line
|
||||
return
|
||||
|
|
@ -850,6 +855,12 @@ class ExternalProviderClient:
|
|||
messages: list[dict[str, Any]],
|
||||
model: str,
|
||||
max_tokens: Optional[int],
|
||||
*,
|
||||
frequency_penalty: Optional[float] = None,
|
||||
seed: Optional[int] = None,
|
||||
stop: Optional[Union[str, list[str]]] = None,
|
||||
parallel_tool_calls: Optional[bool] = None,
|
||||
presence_penalty: Optional[float] = None,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""
|
||||
Kimi $web_search round-trip.
|
||||
|
|
@ -889,6 +900,37 @@ class ExternalProviderClient:
|
|||
if max_tokens is not None:
|
||||
body["max_tokens"] = max_tokens
|
||||
|
||||
# Forward the new optional sampling extensions (#5711) on the
|
||||
# web-search bypass too. The default OAI-compat body construction
|
||||
# (which adds these) is skipped because this helper returns
|
||||
# early; forwarding here ensures kimi-with-search honours the
|
||||
# same sampling controls as kimi-without-search.
|
||||
if presence_penalty is not None:
|
||||
body["presence_penalty"] = presence_penalty
|
||||
if frequency_penalty is not None:
|
||||
body["frequency_penalty"] = frequency_penalty
|
||||
if seed is not None:
|
||||
body["seed"] = seed
|
||||
if stop:
|
||||
if isinstance(stop, str):
|
||||
if stop.strip():
|
||||
body["stop"] = stop
|
||||
elif isinstance(stop, list):
|
||||
sequences = list(
|
||||
dict.fromkeys(s for s in stop if isinstance(s, str) and s)
|
||||
)
|
||||
if len(sequences) > 4:
|
||||
logger.warning(
|
||||
"stop sequences truncated to 4 entries "
|
||||
"(received %d, OpenAI's hard cap is 4)",
|
||||
len(sequences),
|
||||
)
|
||||
body["stop"] = sequences[:4]
|
||||
elif sequences:
|
||||
body["stop"] = sequences
|
||||
if parallel_tool_calls is not None:
|
||||
body["parallel_tool_calls"] = parallel_tool_calls
|
||||
|
||||
# Strip body fields the Kimi registry declares unusable
|
||||
# (temperature/top_p — see body_omit in providers.py).
|
||||
from core.inference.providers import get_provider_info
|
||||
|
|
@ -2783,12 +2825,18 @@ class ExternalProviderClient:
|
|||
"input": input_items,
|
||||
"stream": True,
|
||||
}
|
||||
# Responses accepts service_tier on the same enum set as Chat
|
||||
# Completions minus `scale`. parallel_tool_calls follows the
|
||||
# same shape (default true). The frontend capability gate
|
||||
# (provider-capabilities.ts) already filters the option lists
|
||||
# per provider, so we just forward what we got.
|
||||
if service_tier in ("auto", "default", "flex", "priority"):
|
||||
# Responses accepts the same service_tier enum set as Chat
|
||||
# Completions (auto|default|flex|scale|priority) per the live
|
||||
# `openai-python` SDK
|
||||
# (`src/openai/types/responses/response_create_params.py`
|
||||
# declares `Optional[Literal["auto", "default", "flex",
|
||||
# "scale", "priority"]]`). parallel_tool_calls follows the same
|
||||
# shape (default true). The frontend capability gate
|
||||
# (provider-capabilities.ts) already filters per-provider, so
|
||||
# we just forward whatever value the dispatcher hands us, with
|
||||
# `standard_only` (Anthropic-only) being the one value Responses
|
||||
# has never accepted.
|
||||
if service_tier in ("auto", "default", "flex", "scale", "priority"):
|
||||
body["service_tier"] = service_tier
|
||||
if parallel_tool_calls is not None:
|
||||
body["parallel_tool_calls"] = bool(parallel_tool_calls)
|
||||
|
|
|
|||
|
|
@ -107,6 +107,18 @@ class ChatInferenceSettings(BaseModel):
|
|||
minP: Optional[float] = None
|
||||
repetitionPenalty: Optional[float] = None
|
||||
presencePenalty: Optional[float] = None
|
||||
# New per-provider sampling knobs. `extra="forbid"` would 422 any
|
||||
# settings save from a frontend on the new code if these were not
|
||||
# listed here, breaking the entire chat-settings persistence path.
|
||||
# Keep these aligned with `InferenceParams` in
|
||||
# studio/frontend/src/features/chat/types/runtime.ts.
|
||||
frequencyPenalty: Optional[float] = Field(default = None, ge = -2.0, le = 2.0)
|
||||
seed: Optional[int] = None
|
||||
stop: Optional[list[str]] = None
|
||||
serviceTier: Optional[
|
||||
Literal["auto", "default", "flex", "priority", "scale", "standard_only"]
|
||||
] = None
|
||||
parallelToolCalls: Optional[bool] = None
|
||||
maxSeqLength: Optional[float] = None
|
||||
maxTokens: Optional[float] = None
|
||||
systemPrompt: Optional[str] = None
|
||||
|
|
|
|||
|
|
@ -4898,6 +4898,9 @@ def _build_passthrough_payload(
|
|||
min_p = None,
|
||||
repetition_penalty = None,
|
||||
presence_penalty = None,
|
||||
frequency_penalty = None,
|
||||
seed = None,
|
||||
parallel_tool_calls = None,
|
||||
tool_choice = "auto",
|
||||
response_format = None,
|
||||
chat_template_kwargs = None,
|
||||
|
|
@ -4929,6 +4932,18 @@ def _build_passthrough_payload(
|
|||
body["repeat_penalty"] = repetition_penalty
|
||||
if presence_penalty is not None:
|
||||
body["presence_penalty"] = presence_penalty
|
||||
# New per-provider sampling extensions (PR #5711). llama-server's
|
||||
# /v1/chat/completions endpoint accepts the standard OpenAI fields,
|
||||
# so forward them straight through. parallel_tool_calls is a no-op
|
||||
# on llama-server today (the upstream always dispatches sequentially)
|
||||
# but forward it anyway so a future llama-server release that
|
||||
# implements it picks up the user's preference automatically.
|
||||
if frequency_penalty is not None:
|
||||
body["frequency_penalty"] = frequency_penalty
|
||||
if seed is not None:
|
||||
body["seed"] = seed
|
||||
if parallel_tool_calls is not None:
|
||||
body["parallel_tool_calls"] = parallel_tool_calls
|
||||
if response_format is not None:
|
||||
# llama-server applies a GBNF grammar derived from the JSON schema
|
||||
# when response_format is present. Field is documented flat at the
|
||||
|
|
@ -5347,6 +5362,9 @@ def _build_openai_passthrough_body(payload, backend_ctx = None) -> dict:
|
|||
min_p = payload.min_p,
|
||||
repetition_penalty = payload.repetition_penalty,
|
||||
presence_penalty = payload.presence_penalty,
|
||||
frequency_penalty = payload.frequency_penalty,
|
||||
seed = payload.seed,
|
||||
parallel_tool_calls = payload.parallel_tool_calls,
|
||||
tool_choice = tool_choice,
|
||||
response_format = _extract_response_format(payload),
|
||||
chat_template_kwargs = tpl_kwargs,
|
||||
|
|
|
|||
|
|
@ -385,11 +385,24 @@ def test_openai_responses_forwards_service_tier(monkeypatch):
|
|||
assert body.get("service_tier") == "priority", body
|
||||
|
||||
|
||||
def test_openai_responses_rejects_chat_only_service_tier(monkeypatch):
|
||||
@pytest.mark.parametrize(
|
||||
"value", ["auto", "default", "flex", "scale", "priority"]
|
||||
)
|
||||
def test_openai_responses_accepts_full_service_tier_enum(monkeypatch, value):
|
||||
"""`openai-python`'s ResponseCreateParams declares
|
||||
`Optional[Literal["auto", "default", "flex", "scale", "priority"]]`
|
||||
so every value in that set forwards untouched."""
|
||||
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
||||
body = _drive_openai_responses(captured, service_tier = "scale")
|
||||
# Responses only accepts auto|default|flex|priority -- `scale` is
|
||||
# silently dropped so a stale frontend cannot 400 the request.
|
||||
body = _drive_openai_responses(captured, service_tier = value)
|
||||
assert body.get("service_tier") == value, body
|
||||
|
||||
|
||||
def test_openai_responses_drops_anthropic_only_service_tier(monkeypatch):
|
||||
"""`standard_only` is Anthropic-only and Responses has never accepted
|
||||
it. Drop it client-side so a stale frontend cannot 400 the request.
|
||||
"""
|
||||
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
||||
body = _drive_openai_responses(captured, service_tier = "standard_only")
|
||||
assert "service_tier" not in body, body
|
||||
|
||||
|
||||
|
|
@ -460,3 +473,106 @@ def test_chat_completion_request_clamps_frequency_penalty_range():
|
|||
"frequency_penalty": -3.0,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ── Kimi web-search bypass forwards new sampling fields ────────────────
|
||||
|
||||
|
||||
def test_kimi_web_search_bypass_forwards_new_sampling_fields(monkeypatch):
|
||||
"""The Kimi `enabled_tools=["web_search"]` path takes an early
|
||||
return into `_stream_kimi_web_search` BEFORE the default OAI-compat
|
||||
body builder runs. PR #5711 added new sampling fields to the
|
||||
default builder; this test pins that the web-search bypass also
|
||||
forwards them so Kimi-with-search and Kimi-without-search behave
|
||||
consistently."""
|
||||
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
||||
|
||||
async def run():
|
||||
client = ExternalProviderClient(
|
||||
provider_type = "kimi",
|
||||
base_url = "https://api.moonshot.ai/v1",
|
||||
api_key = "kimi-test",
|
||||
)
|
||||
async for _ in client.stream_chat_completion(
|
||||
messages = [{"role": "user", "content": "hi"}],
|
||||
model = "kimi-k2.6",
|
||||
temperature = 1.0,
|
||||
top_p = 1.0,
|
||||
max_tokens = 256,
|
||||
enabled_tools = ["web_search"],
|
||||
presence_penalty = 0.5,
|
||||
frequency_penalty = 1.25,
|
||||
seed = 7,
|
||||
stop = ["END"],
|
||||
parallel_tool_calls = False,
|
||||
):
|
||||
pass
|
||||
await client.close()
|
||||
|
||||
_drive(run())
|
||||
body = captured["body"]
|
||||
assert body.get("frequency_penalty") == 1.25, body
|
||||
assert body.get("seed") == 7, body
|
||||
assert body.get("stop") == ["END"], body
|
||||
assert body.get("parallel_tool_calls") is False, body
|
||||
assert body.get("presence_penalty") == 0.5, body
|
||||
# body_omit still strips temperature / top_p for Kimi.
|
||||
assert "temperature" not in body, body
|
||||
assert "top_p" not in body, body
|
||||
|
||||
|
||||
# ── Local OpenAI passthrough forwards new sampling fields ──────────────
|
||||
|
||||
|
||||
def test_local_openai_passthrough_forwards_new_sampling_fields():
|
||||
"""Round 1 reviewers (10/20) flagged that
|
||||
`_build_openai_passthrough_body` dropped frequency_penalty / seed /
|
||||
parallel_tool_calls when forwarding to llama-server. Pin the
|
||||
extended contract."""
|
||||
from models.inference import ChatCompletionRequest
|
||||
from routes.inference import _build_openai_passthrough_body
|
||||
|
||||
payload = ChatCompletionRequest.model_validate(
|
||||
{
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"stream": True,
|
||||
"frequency_penalty": 1.25,
|
||||
"seed": 123,
|
||||
"stop": ["END"],
|
||||
"parallel_tool_calls": False,
|
||||
}
|
||||
)
|
||||
body = _build_openai_passthrough_body(payload, backend_ctx = 4096)
|
||||
assert body["frequency_penalty"] == 1.25, body
|
||||
assert body["seed"] == 123, body
|
||||
assert body["stop"] == ["END"], body
|
||||
assert body["parallel_tool_calls"] is False, body
|
||||
|
||||
|
||||
# ── Backend ChatInferenceSettings schema accepts new fields ────────────
|
||||
|
||||
|
||||
def test_chat_settings_payload_accepts_new_sampling_keys():
|
||||
"""Round 1 reviewers flagged that `ChatSettingsPayload.extra="forbid"`
|
||||
with the old field list 422'd every settings save that contained
|
||||
any of the new keys. Pin that the new keys round-trip."""
|
||||
from routes.chat_history import ChatSettingsPayload
|
||||
|
||||
parsed = ChatSettingsPayload.model_validate(
|
||||
{
|
||||
"inferenceParams": {
|
||||
"frequencyPenalty": 0.7,
|
||||
"seed": 42,
|
||||
"stop": ["END"],
|
||||
"serviceTier": "standard_only",
|
||||
"parallelToolCalls": False,
|
||||
}
|
||||
}
|
||||
)
|
||||
ip = parsed.inferenceParams
|
||||
assert ip is not None
|
||||
assert ip.frequencyPenalty == 0.7
|
||||
assert ip.seed == 42
|
||||
assert ip.stop == ["END"]
|
||||
assert ip.serviceTier == "standard_only"
|
||||
assert ip.parallelToolCalls is False
|
||||
|
|
|
|||
|
|
@ -65,17 +65,19 @@ export type ServiceTierOption =
|
|||
| "default"
|
||||
| "flex"
|
||||
| "priority"
|
||||
| "scale"
|
||||
| "standard_only";
|
||||
|
||||
/**
|
||||
* Legal `service_tier` values per provider, sourced from each upstream's
|
||||
* docs. Anthropic exposes only `auto` and `standard_only`. OpenAI in
|
||||
* Studio is routed through `/v1/responses` (not Chat Completions), and
|
||||
* the Responses endpoint only accepts `auto` / `default` / `flex` /
|
||||
* `priority` — `scale` is Chat-only and would be silently dropped here,
|
||||
* so the option list omits it to avoid misleading the user. Other
|
||||
* providers fall through to a permissive `auto` / `default` pair so the
|
||||
* picker stays usable for OpenAI-compat backends.
|
||||
* Studio is routed through `/v1/responses` (not Chat Completions); the
|
||||
* live `openai-python` SDK declares the Responses-side service_tier as
|
||||
* `Literal["auto", "default", "flex", "scale", "priority"]`
|
||||
* (`src/openai/types/responses/response_create_params.py`), so the
|
||||
* full set is exposed. Other providers fall through to a permissive
|
||||
* `auto` / `default` pair so the picker stays usable for
|
||||
* OpenAI-compat backends.
|
||||
*/
|
||||
export function getServiceTierOptions(
|
||||
providerType: string | null | undefined,
|
||||
|
|
@ -84,7 +86,7 @@ export function getServiceTierOptions(
|
|||
return ["auto", "standard_only"] as const;
|
||||
}
|
||||
if (providerType === "openai") {
|
||||
return ["auto", "default", "flex", "priority"] as const;
|
||||
return ["auto", "default", "flex", "scale", "priority"] as const;
|
||||
}
|
||||
return ["auto", "default"] as const;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ export type ServiceTier =
|
|||
| "default"
|
||||
| "flex"
|
||||
| "priority"
|
||||
| "scale"
|
||||
| "standard_only";
|
||||
|
||||
export interface InferenceParams {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue