diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index df06098156..86ea6ea800 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -27,6 +27,52 @@ import structlog logger = structlog.get_logger(__name__) +def _normalize_stop_for_provider( + stop: Optional[Union[str, list[str]]], + provider_info: dict[str, Any], +) -> Optional[Union[str, list[str]]]: + """Apply per-provider stop_max / stop_max_bytes caps and dedup. + + Returns None when nothing survives the filter so callers can omit + the field. Single strings are returned verbatim when they fit. + """ + if not stop: + return None + + stop_max = int(provider_info.get("stop_max", 16)) + stop_max_bytes_raw = provider_info.get("stop_max_bytes") + stop_max_bytes = ( + int(stop_max_bytes_raw) if stop_max_bytes_raw is not None else None + ) + + def allowed(s: str) -> bool: + if not s: + return False + if stop_max_bytes is not None and len(s.encode("utf-8")) > stop_max_bytes: + logger.warning( + "dropping stop sequence longer than %d bytes", + stop_max_bytes, + ) + return False + return True + + if isinstance(stop, str): + return stop if allowed(stop) else None + if isinstance(stop, list): + sequences = list( + dict.fromkeys(s for s in stop if isinstance(s, str) and allowed(s)) + ) + if len(sequences) > stop_max: + logger.warning( + "stop sequences truncated to %d entries (received %d)", + stop_max, + len(sequences), + ) + sequences = sequences[:stop_max] + return sequences or None + return None + + # Claude 4.7 (Opus/Sonnet/Haiku) removed temperature, top_p, and top_k — # the API returns 400 " is deprecated for this model" if any of # them is set to a non-default value. The "Sampling parameters removed" @@ -472,25 +518,9 @@ class ExternalProviderClient: # Mistral renames `seed` to `random_seed` on /v1/chat/completions. seed_field = provider_info.get("seed_field", "seed") body[seed_field] = seed - if stop: - # Stop cap is provider-specific. OpenAI Chat = 4, Anthropic - # = 16 (client guard), DeepSeek = 16, others = 16 by default. - stop_max = int(provider_info.get("stop_max", 16)) - if isinstance(stop, str): - body["stop"] = stop - elif isinstance(stop, list): - sequences = list( - dict.fromkeys(s for s in stop if isinstance(s, str) and s) - ) - if len(sequences) > stop_max: - logger.warning( - "stop sequences truncated to %d entries (received %d)", - stop_max, - len(sequences), - ) - body["stop"] = sequences[:stop_max] - elif sequences: - body["stop"] = sequences + normalized_stop = _normalize_stop_for_provider(stop, provider_info) + if normalized_stop: + body["stop"] = normalized_stop if service_tier is not None: body["service_tier"] = service_tier if parallel_tool_calls is not None: @@ -908,23 +938,9 @@ class ExternalProviderClient: if seed is not None: seed_field = provider_info.get("seed_field", "seed") body[seed_field] = seed - if stop: - stop_max = int(provider_info.get("stop_max", 16)) - if isinstance(stop, str): - body["stop"] = stop - elif isinstance(stop, list): - sequences = list( - dict.fromkeys(s for s in stop if isinstance(s, str) and s) - ) - if len(sequences) > stop_max: - logger.warning( - "stop sequences truncated to %d entries (received %d)", - stop_max, - len(sequences), - ) - body["stop"] = sequences[:stop_max] - elif sequences: - body["stop"] = sequences + normalized_stop = _normalize_stop_for_provider(stop, provider_info) + if normalized_stop: + body["stop"] = normalized_stop if parallel_tool_calls is not None: body["parallel_tool_calls"] = parallel_tool_calls diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index 4a47e90850..6701b27ba9 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -165,6 +165,7 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { # Kimi accepts at most 5 stop strings (each <= 32 bytes) per # https://platform.kimi.ai/docs/api/chat "stop_max": 5, + "stop_max_bytes": 32, }, "qwen": { "display_name": "Qwen", diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index 012c2266f2..99cb3a8993 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -607,6 +607,61 @@ def test_kimi_web_search_uses_kimi_stop_cap_5(monkeypatch): assert body["stop"] == ["S0", "S1", "S2", "S3", "S4"], body +def test_kimi_drops_stop_strings_over_32_bytes(monkeypatch): + """Kimi limits each stop string to <= 32 bytes per + https://platform.kimi.ai/docs/api/chat. Drop overlong entries + client-side so a stale UI cannot 400 the request.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "kimi", + base_url = "https://api.moonshot.ai/v1", + api_key = "kimi-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "kimi-k2.6", + temperature = 1.0, + top_p = 1.0, + max_tokens = 256, + stop = ["END", "x" * 33, "DONE"], + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert body.get("stop") == ["END", "DONE"], body + + +def test_kimi_web_search_drops_stop_strings_over_32_bytes(monkeypatch): + """Same byte cap applies to the Kimi web-search bypass.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "kimi", + base_url = "https://api.moonshot.ai/v1", + api_key = "kimi-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "kimi-k2.6", + temperature = 1.0, + top_p = 1.0, + max_tokens = 256, + enabled_tools = ["web_search"], + stop = ["END", "x" * 40, "DONE"], + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert body.get("stop") == ["END", "DONE"], body + + def test_kimi_default_path_uses_kimi_stop_cap_5(monkeypatch): """The normal Kimi path must also honour the documented 5-cap.""" captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())