From 95e143545f209eb07331b41a04fbd98c82d63bef Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 14:49:57 +0000 Subject: [PATCH] Per-provider stop cap on Kimi web-search bypass and frontend sheet Round 5 review flagged two asymmetries: 1. Kimi web-search bypass hard-capped stops at 4 while the default OAI-compat path honours provider_info["stop_max"]. Apply the same provider-aware logic in _stream_kimi_web_search so kimi-with-search and kimi-without-search match. Also add Kimi's documented 5-stop max (https://platform.kimi.ai/docs/api/chat) to the provider registry so the cap actually fires. 2. chat-settings-sheet.tsx caps every non-Anthropic external provider at 4 stops. Replace with a per-provider getProviderStopMax helper in provider-capabilities.ts so DeepSeek, Mistral, and local backends are not artificially restricted while OpenAI Chat still hits its 4-entry hard limit and Kimi hits its documented 5-entry cap. Tests pin the Kimi 5-cap on both Kimi paths. --- .../core/inference/external_provider.py | 27 ++++---- studio/backend/core/inference/providers.py | 3 + .../tests/test_sampling_params_routing.py | 66 +++++++++++++++++-- .../src/features/chat/chat-settings-sheet.tsx | 11 ++-- .../features/chat/provider-capabilities.ts | 26 ++++++++ 5 files changed, 107 insertions(+), 26 deletions(-) diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index 904380ac5d..df06098156 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -895,41 +895,40 @@ class ExternalProviderClient: body["max_tokens"] = max_tokens # The default OAI-compat body construction is skipped because - # this helper returns early. Forward the optional sampling - # extensions here so kimi-with-search behaves the same as + # this helper returns early. Apply the same provider-aware + # sampling / stop logic here so kimi-with-search matches # kimi-without-search. + from core.inference.providers import get_provider_info + + provider_info = get_provider_info(self.provider_type) or {} if presence_penalty is not None: body["presence_penalty"] = presence_penalty if frequency_penalty is not None: body["frequency_penalty"] = frequency_penalty if seed is not None: - body["seed"] = seed + seed_field = provider_info.get("seed_field", "seed") + body[seed_field] = seed if stop: - # Match the default OAI-compat path: forward a single string - # verbatim, dedupe + cap lists to 4 (OpenAI hard limit). + stop_max = int(provider_info.get("stop_max", 16)) if isinstance(stop, str): body["stop"] = stop elif isinstance(stop, list): sequences = list( dict.fromkeys(s for s in stop if isinstance(s, str) and s) ) - if len(sequences) > 4: + if len(sequences) > stop_max: logger.warning( - "stop sequences truncated to 4 entries " - "(received %d, OpenAI's hard cap is 4)", + "stop sequences truncated to %d entries (received %d)", + stop_max, len(sequences), ) - body["stop"] = sequences[:4] + body["stop"] = sequences[:stop_max] elif sequences: body["stop"] = sequences if parallel_tool_calls is not None: body["parallel_tool_calls"] = parallel_tool_calls - # Strip body fields the Kimi registry declares unusable - # (temperature/top_p — see body_omit in providers.py). - from core.inference.providers import get_provider_info - - provider_info = get_provider_info(self.provider_type) or {} + # Drop body fields the provider's registry entry locks down. for field in provider_info.get("body_omit", ()): body.pop(field, None) diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index 98c8abe53b..4a47e90850 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -162,6 +162,9 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { # (and the same shape for top_p). Strip both fields from the # outbound body so the server falls back to its required defaults. "body_omit": ("temperature", "top_p"), + # Kimi accepts at most 5 stop strings (each <= 32 bytes) per + # https://platform.kimi.ai/docs/api/chat + "stop_max": 5, }, "qwen": { "display_name": "Qwen", diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index 8d8b199d78..012c2266f2 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -537,12 +537,10 @@ def test_chat_completion_request_clamps_frequency_penalty_range(): def test_kimi_web_search_bypass_forwards_new_sampling_fields(monkeypatch): - """The Kimi `enabled_tools=["web_search"]` path takes an early - return into `_stream_kimi_web_search` BEFORE the default OAI-compat - body builder runs. PR #5711 added new sampling fields to the - default builder; this test pins that the web-search bypass also - forwards them so Kimi-with-search and Kimi-without-search behave - consistently.""" + """The Kimi $web_search path takes an early return into + `_stream_kimi_web_search` before the default OAI-compat body + builder runs; forwarding here keeps Kimi-with-search and + Kimi-without-search in lockstep.""" captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) async def run(): @@ -579,6 +577,62 @@ def test_kimi_web_search_bypass_forwards_new_sampling_fields(monkeypatch): assert "top_p" not in body, body +def test_kimi_web_search_uses_kimi_stop_cap_5(monkeypatch): + """Kimi documents a 5-stop max; the web-search bypass must honour + `provider_info["stop_max"]` rather than the OpenAI 4-cap or the + permissive default.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "kimi", + base_url = "https://api.moonshot.ai/v1", + api_key = "kimi-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "kimi-k2.6", + temperature = 1.0, + top_p = 1.0, + max_tokens = 256, + enabled_tools = ["web_search"], + stop = [f"S{i}" for i in range(10)], + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert len(body.get("stop", [])) == 5, body + assert body["stop"] == ["S0", "S1", "S2", "S3", "S4"], body + + +def test_kimi_default_path_uses_kimi_stop_cap_5(monkeypatch): + """The normal Kimi path must also honour the documented 5-cap.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "kimi", + base_url = "https://api.moonshot.ai/v1", + api_key = "kimi-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "kimi-k2.6", + temperature = 1.0, + top_p = 1.0, + max_tokens = 256, + stop = [f"S{i}" for i in range(10)], + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert len(body.get("stop", [])) == 5, body + + # ── Local OpenAI passthrough forwards new sampling fields ────────────── diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index 271dfe22af..07a6e2ccf2 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -86,6 +86,7 @@ import { EXTERNAL_MAX_OUTPUT_TOKENS, type ProviderCapabilities, getExternalMinOutputTokens, + getProviderStopMax, getServiceTierOptions, providerSupportsBuiltinCodeExecution, } from "./provider-capabilities"; @@ -427,12 +428,10 @@ export function ChatSettingsPanel({ isExternalModel && Boolean(providerCapabilities?.serviceTier); const showParallelToolCalls = !isExternalModel || Boolean(providerCapabilities?.parallelToolCalls); - // OpenAI Chat caps `stop` at 4; Anthropic, DeepSeek, Mistral, and - // local llama.cpp / vLLM / ollama backends accept more. Use 16 as - // the UI ceiling for everything that is not OpenAI cloud Chat; the - // backend re-trims per provider on the wire. - const stopMaxEntries = - !isExternalModel || externalProviderType === "anthropic" ? 16 : 4; + // Per-provider stop cap from provider-capabilities.ts; backend + // re-trims on the wire if a stale UI sends more than the upstream + // accepts. + const stopMaxEntries = getProviderStopMax(externalProviderType); const serviceTierOptions = getServiceTierOptions(externalProviderType); const isMobile = useIsMobile(); const isGguf = useChatRuntimeStore((s) => s.activeGgufVariant) != null; diff --git a/studio/frontend/src/features/chat/provider-capabilities.ts b/studio/frontend/src/features/chat/provider-capabilities.ts index 8b9212512c..c64aa9ade8 100644 --- a/studio/frontend/src/features/chat/provider-capabilities.ts +++ b/studio/frontend/src/features/chat/provider-capabilities.ts @@ -60,6 +60,32 @@ export interface ProviderCapabilities { parallelToolCalls: boolean; } +/** + * Per-provider stop-sequence max count. Resolved by + * `getProviderStopMax(providerType)`. Mirrors the backend's + * `provider_info.stop_max` for the same provider type. + * - openai: 4 (Chat Completions hard cap; Responses drops stop) + * - anthropic: 16 (client-side guard; docs publish no max) + * - kimi: 5 (https://platform.kimi.ai/docs/api/chat) + * - deepseek: 16 (https://api-docs.deepseek.com/api/create-chat-completion) + * - mistral: 16 (no documented max; widen to permissive default) + * - default: 16 (covers ollama, vllm, llama.cpp, openrouter, custom) + */ +const PROVIDER_STOP_MAX: Record = { + openai: 4, + anthropic: 16, + kimi: 5, + deepseek: 16, + mistral: 16, +}; + +export function getProviderStopMax( + providerType: string | null | undefined, +): number { + if (!providerType) return 16; // local backends + return PROVIDER_STOP_MAX[providerType] ?? 16; +} + export type ServiceTierOption = | "auto" | "default"