diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index c78c60e6ad..002e2548c6 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -435,6 +435,11 @@ class ExternalProviderClient: messages, model, max_tokens, + frequency_penalty = frequency_penalty, + seed = seed, + stop = stop, + parallel_tool_calls = parallel_tool_calls, + presence_penalty = presence_penalty, ): yield line return @@ -850,6 +855,12 @@ class ExternalProviderClient: messages: list[dict[str, Any]], model: str, max_tokens: Optional[int], + *, + frequency_penalty: Optional[float] = None, + seed: Optional[int] = None, + stop: Optional[Union[str, list[str]]] = None, + parallel_tool_calls: Optional[bool] = None, + presence_penalty: Optional[float] = None, ) -> AsyncGenerator[str, None]: """ Kimi $web_search round-trip. @@ -889,6 +900,37 @@ class ExternalProviderClient: if max_tokens is not None: body["max_tokens"] = max_tokens + # Forward the new optional sampling extensions (#5711) on the + # web-search bypass too. The default OAI-compat body construction + # (which adds these) is skipped because this helper returns + # early; forwarding here ensures kimi-with-search honours the + # same sampling controls as kimi-without-search. + if presence_penalty is not None: + body["presence_penalty"] = presence_penalty + if frequency_penalty is not None: + body["frequency_penalty"] = frequency_penalty + if seed is not None: + body["seed"] = seed + if stop: + if isinstance(stop, str): + if stop.strip(): + body["stop"] = stop + elif isinstance(stop, list): + sequences = list( + dict.fromkeys(s for s in stop if isinstance(s, str) and s) + ) + if len(sequences) > 4: + logger.warning( + "stop sequences truncated to 4 entries " + "(received %d, OpenAI's hard cap is 4)", + len(sequences), + ) + body["stop"] = sequences[:4] + elif sequences: + body["stop"] = sequences + if parallel_tool_calls is not None: + body["parallel_tool_calls"] = parallel_tool_calls + # Strip body fields the Kimi registry declares unusable # (temperature/top_p — see body_omit in providers.py). from core.inference.providers import get_provider_info @@ -2783,12 +2825,18 @@ class ExternalProviderClient: "input": input_items, "stream": True, } - # Responses accepts service_tier on the same enum set as Chat - # Completions minus `scale`. parallel_tool_calls follows the - # same shape (default true). The frontend capability gate - # (provider-capabilities.ts) already filters the option lists - # per provider, so we just forward what we got. - if service_tier in ("auto", "default", "flex", "priority"): + # Responses accepts the same service_tier enum set as Chat + # Completions (auto|default|flex|scale|priority) per the live + # `openai-python` SDK + # (`src/openai/types/responses/response_create_params.py` + # declares `Optional[Literal["auto", "default", "flex", + # "scale", "priority"]]`). parallel_tool_calls follows the same + # shape (default true). The frontend capability gate + # (provider-capabilities.ts) already filters per-provider, so + # we just forward whatever value the dispatcher hands us, with + # `standard_only` (Anthropic-only) being the one value Responses + # has never accepted. + if service_tier in ("auto", "default", "flex", "scale", "priority"): body["service_tier"] = service_tier if parallel_tool_calls is not None: body["parallel_tool_calls"] = bool(parallel_tool_calls) diff --git a/studio/backend/routes/chat_history.py b/studio/backend/routes/chat_history.py index ed808040d2..6f7efb6eb5 100644 --- a/studio/backend/routes/chat_history.py +++ b/studio/backend/routes/chat_history.py @@ -107,6 +107,18 @@ class ChatInferenceSettings(BaseModel): minP: Optional[float] = None repetitionPenalty: Optional[float] = None presencePenalty: Optional[float] = None + # New per-provider sampling knobs. `extra="forbid"` would 422 any + # settings save from a frontend on the new code if these were not + # listed here, breaking the entire chat-settings persistence path. + # Keep these aligned with `InferenceParams` in + # studio/frontend/src/features/chat/types/runtime.ts. + frequencyPenalty: Optional[float] = Field(default = None, ge = -2.0, le = 2.0) + seed: Optional[int] = None + stop: Optional[list[str]] = None + serviceTier: Optional[ + Literal["auto", "default", "flex", "priority", "scale", "standard_only"] + ] = None + parallelToolCalls: Optional[bool] = None maxSeqLength: Optional[float] = None maxTokens: Optional[float] = None systemPrompt: Optional[str] = None diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index b6e4c4e370..1ee20c6459 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -4898,6 +4898,9 @@ def _build_passthrough_payload( min_p = None, repetition_penalty = None, presence_penalty = None, + frequency_penalty = None, + seed = None, + parallel_tool_calls = None, tool_choice = "auto", response_format = None, chat_template_kwargs = None, @@ -4929,6 +4932,18 @@ def _build_passthrough_payload( body["repeat_penalty"] = repetition_penalty if presence_penalty is not None: body["presence_penalty"] = presence_penalty + # New per-provider sampling extensions (PR #5711). llama-server's + # /v1/chat/completions endpoint accepts the standard OpenAI fields, + # so forward them straight through. parallel_tool_calls is a no-op + # on llama-server today (the upstream always dispatches sequentially) + # but forward it anyway so a future llama-server release that + # implements it picks up the user's preference automatically. + if frequency_penalty is not None: + body["frequency_penalty"] = frequency_penalty + if seed is not None: + body["seed"] = seed + if parallel_tool_calls is not None: + body["parallel_tool_calls"] = parallel_tool_calls if response_format is not None: # llama-server applies a GBNF grammar derived from the JSON schema # when response_format is present. Field is documented flat at the @@ -5347,6 +5362,9 @@ def _build_openai_passthrough_body(payload, backend_ctx = None) -> dict: min_p = payload.min_p, repetition_penalty = payload.repetition_penalty, presence_penalty = payload.presence_penalty, + frequency_penalty = payload.frequency_penalty, + seed = payload.seed, + parallel_tool_calls = payload.parallel_tool_calls, tool_choice = tool_choice, response_format = _extract_response_format(payload), chat_template_kwargs = tpl_kwargs, diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index 699d4d4b0e..27f430f8d3 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -385,11 +385,24 @@ def test_openai_responses_forwards_service_tier(monkeypatch): assert body.get("service_tier") == "priority", body -def test_openai_responses_rejects_chat_only_service_tier(monkeypatch): +@pytest.mark.parametrize( + "value", ["auto", "default", "flex", "scale", "priority"] +) +def test_openai_responses_accepts_full_service_tier_enum(monkeypatch, value): + """`openai-python`'s ResponseCreateParams declares + `Optional[Literal["auto", "default", "flex", "scale", "priority"]]` + so every value in that set forwards untouched.""" captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) - body = _drive_openai_responses(captured, service_tier = "scale") - # Responses only accepts auto|default|flex|priority -- `scale` is - # silently dropped so a stale frontend cannot 400 the request. + body = _drive_openai_responses(captured, service_tier = value) + assert body.get("service_tier") == value, body + + +def test_openai_responses_drops_anthropic_only_service_tier(monkeypatch): + """`standard_only` is Anthropic-only and Responses has never accepted + it. Drop it client-side so a stale frontend cannot 400 the request. + """ + captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) + body = _drive_openai_responses(captured, service_tier = "standard_only") assert "service_tier" not in body, body @@ -460,3 +473,106 @@ def test_chat_completion_request_clamps_frequency_penalty_range(): "frequency_penalty": -3.0, } ) + + +# ── Kimi web-search bypass forwards new sampling fields ──────────────── + + +def test_kimi_web_search_bypass_forwards_new_sampling_fields(monkeypatch): + """The Kimi `enabled_tools=["web_search"]` path takes an early + return into `_stream_kimi_web_search` BEFORE the default OAI-compat + body builder runs. PR #5711 added new sampling fields to the + default builder; this test pins that the web-search bypass also + forwards them so Kimi-with-search and Kimi-without-search behave + consistently.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "kimi", + base_url = "https://api.moonshot.ai/v1", + api_key = "kimi-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "kimi-k2.6", + temperature = 1.0, + top_p = 1.0, + max_tokens = 256, + enabled_tools = ["web_search"], + presence_penalty = 0.5, + frequency_penalty = 1.25, + seed = 7, + stop = ["END"], + parallel_tool_calls = False, + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert body.get("frequency_penalty") == 1.25, body + assert body.get("seed") == 7, body + assert body.get("stop") == ["END"], body + assert body.get("parallel_tool_calls") is False, body + assert body.get("presence_penalty") == 0.5, body + # body_omit still strips temperature / top_p for Kimi. + assert "temperature" not in body, body + assert "top_p" not in body, body + + +# ── Local OpenAI passthrough forwards new sampling fields ────────────── + + +def test_local_openai_passthrough_forwards_new_sampling_fields(): + """Round 1 reviewers (10/20) flagged that + `_build_openai_passthrough_body` dropped frequency_penalty / seed / + parallel_tool_calls when forwarding to llama-server. Pin the + extended contract.""" + from models.inference import ChatCompletionRequest + from routes.inference import _build_openai_passthrough_body + + payload = ChatCompletionRequest.model_validate( + { + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + "frequency_penalty": 1.25, + "seed": 123, + "stop": ["END"], + "parallel_tool_calls": False, + } + ) + body = _build_openai_passthrough_body(payload, backend_ctx = 4096) + assert body["frequency_penalty"] == 1.25, body + assert body["seed"] == 123, body + assert body["stop"] == ["END"], body + assert body["parallel_tool_calls"] is False, body + + +# ── Backend ChatInferenceSettings schema accepts new fields ──────────── + + +def test_chat_settings_payload_accepts_new_sampling_keys(): + """Round 1 reviewers flagged that `ChatSettingsPayload.extra="forbid"` + with the old field list 422'd every settings save that contained + any of the new keys. Pin that the new keys round-trip.""" + from routes.chat_history import ChatSettingsPayload + + parsed = ChatSettingsPayload.model_validate( + { + "inferenceParams": { + "frequencyPenalty": 0.7, + "seed": 42, + "stop": ["END"], + "serviceTier": "standard_only", + "parallelToolCalls": False, + } + } + ) + ip = parsed.inferenceParams + assert ip is not None + assert ip.frequencyPenalty == 0.7 + assert ip.seed == 42 + assert ip.stop == ["END"] + assert ip.serviceTier == "standard_only" + assert ip.parallelToolCalls is False diff --git a/studio/frontend/src/features/chat/provider-capabilities.ts b/studio/frontend/src/features/chat/provider-capabilities.ts index 0c63fed5bc..fb1c2ea0f9 100644 --- a/studio/frontend/src/features/chat/provider-capabilities.ts +++ b/studio/frontend/src/features/chat/provider-capabilities.ts @@ -65,17 +65,19 @@ export type ServiceTierOption = | "default" | "flex" | "priority" + | "scale" | "standard_only"; /** * Legal `service_tier` values per provider, sourced from each upstream's * docs. Anthropic exposes only `auto` and `standard_only`. OpenAI in - * Studio is routed through `/v1/responses` (not Chat Completions), and - * the Responses endpoint only accepts `auto` / `default` / `flex` / - * `priority` — `scale` is Chat-only and would be silently dropped here, - * so the option list omits it to avoid misleading the user. Other - * providers fall through to a permissive `auto` / `default` pair so the - * picker stays usable for OpenAI-compat backends. + * Studio is routed through `/v1/responses` (not Chat Completions); the + * live `openai-python` SDK declares the Responses-side service_tier as + * `Literal["auto", "default", "flex", "scale", "priority"]` + * (`src/openai/types/responses/response_create_params.py`), so the + * full set is exposed. Other providers fall through to a permissive + * `auto` / `default` pair so the picker stays usable for + * OpenAI-compat backends. */ export function getServiceTierOptions( providerType: string | null | undefined, @@ -84,7 +86,7 @@ export function getServiceTierOptions( return ["auto", "standard_only"] as const; } if (providerType === "openai") { - return ["auto", "default", "flex", "priority"] as const; + return ["auto", "default", "flex", "scale", "priority"] as const; } return ["auto", "default"] as const; } diff --git a/studio/frontend/src/features/chat/types/runtime.ts b/studio/frontend/src/features/chat/types/runtime.ts index 8efd5bab2c..49c7c291e7 100644 --- a/studio/frontend/src/features/chat/types/runtime.ts +++ b/studio/frontend/src/features/chat/types/runtime.ts @@ -6,6 +6,7 @@ export type ServiceTier = | "default" | "flex" | "priority" + | "scale" | "standard_only"; export interface InferenceParams {