ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Kimi 32-byte per-stop cap; extract _normalize_stop_for_provider helper

Browse source 
Kimi documents max 5 stop strings AND <= 32 bytes per string at
https://platform.kimi.ai/docs/api/chat. The previous code capped
count but forwarded oversize entries, which can produce upstream
400s. Add stop_max_bytes=32 on the Kimi registry entry and apply
both checks in a new _normalize_stop_for_provider helper shared
between the default OAI-compat path and the Kimi web-search bypass.

Tests pin the byte-cap drop on both Kimi paths.
This commit is contained in:
Daniel Han 2026-05-24 15:06:15 +00:00
commit 1cc52465f3
3 changed files with 108 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

88
studio/backend/core/inference/external_provider.py
View file

@ -27,6 +27,52 @@ import structlog
logger = structlog.get_logger(__name__)
def _normalize_stop_for_provider(
stop: Optional[Union[str, list[str]]],
provider_info: dict[str, Any],
) -> Optional[Union[str, list[str]]]:
"""Apply per-provider stop_max / stop_max_bytes caps and dedup.
Returns None when nothing survives the filter so callers can omit
the field. Single strings are returned verbatim when they fit.
"""
if not stop:
return None
stop_max = int(provider_info.get("stop_max", 16))
stop_max_bytes_raw = provider_info.get("stop_max_bytes")
stop_max_bytes = (
int(stop_max_bytes_raw) if stop_max_bytes_raw is not None else None
)
def allowed(s: str) -> bool:
if not s:
return False
if stop_max_bytes is not None and len(s.encode("utf-8")) > stop_max_bytes:
logger.warning(
"dropping stop sequence longer than %d bytes",
stop_max_bytes,
)
return False
return True
if isinstance(stop, str):
return stop if allowed(stop) else None
if isinstance(stop, list):
sequences = list(
dict.fromkeys(s for s in stop if isinstance(s, str) and allowed(s))
)
if len(sequences) > stop_max:
logger.warning(
"stop sequences truncated to %d entries (received %d)",
stop_max,
len(sequences),
)
sequences = sequences[:stop_max]
return sequences or None
return None
# Claude 4.7 (Opus/Sonnet/Haiku) removed temperature, top_p, and top_k —
# the API returns 400 "<param> is deprecated for this model" if any of
# them is set to a non-default value. The "Sampling parameters removed"
@ -472,25 +518,9 @@ class ExternalProviderClient:
# Mistral renames `seed` to `random_seed` on /v1/chat/completions.
seed_field = provider_info.get("seed_field", "seed")
body[seed_field] = seed
if stop:
# Stop cap is provider-specific. OpenAI Chat = 4, Anthropic
# = 16 (client guard), DeepSeek = 16, others = 16 by default.
stop_max = int(provider_info.get("stop_max", 16))
if isinstance(stop, str):
body["stop"] = stop
elif isinstance(stop, list):
sequences = list(
dict.fromkeys(s for s in stop if isinstance(s, str) and s)
)
if len(sequences) > stop_max:
logger.warning(
"stop sequences truncated to %d entries (received %d)",
stop_max,
len(sequences),
)
body["stop"] = sequences[:stop_max]
elif sequences:
body["stop"] = sequences
normalized_stop = _normalize_stop_for_provider(stop, provider_info)
if normalized_stop:
body["stop"] = normalized_stop
if service_tier is not None:
body["service_tier"] = service_tier
if parallel_tool_calls is not None:
@ -908,23 +938,9 @@ class ExternalProviderClient:
if seed is not None:
seed_field = provider_info.get("seed_field", "seed")
body[seed_field] = seed
if stop:
stop_max = int(provider_info.get("stop_max", 16))
if isinstance(stop, str):
body["stop"] = stop
elif isinstance(stop, list):
sequences = list(
dict.fromkeys(s for s in stop if isinstance(s, str) and s)
)
if len(sequences) > stop_max:
logger.warning(
"stop sequences truncated to %d entries (received %d)",
stop_max,
len(sequences),
)
body["stop"] = sequences[:stop_max]
elif sequences:
body["stop"] = sequences
normalized_stop = _normalize_stop_for_provider(stop, provider_info)
if normalized_stop:
body["stop"] = normalized_stop
if parallel_tool_calls is not None:
body["parallel_tool_calls"] = parallel_tool_calls

1
studio/backend/core/inference/providers.py
View file

@ -165,6 +165,7 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = {
# Kimi accepts at most 5 stop strings (each <= 32 bytes) per
# https://platform.kimi.ai/docs/api/chat
"stop_max": 5,
"stop_max_bytes": 32,
},
"qwen": {
"display_name": "Qwen",

55
studio/backend/tests/test_sampling_params_routing.py
View file

@ -607,6 +607,61 @@ def test_kimi_web_search_uses_kimi_stop_cap_5(monkeypatch):
assert body["stop"] == ["S0", "S1", "S2", "S3", "S4"], body
def test_kimi_drops_stop_strings_over_32_bytes(monkeypatch):
"""Kimi limits each stop string to <= 32 bytes per
https://platform.kimi.ai/docs/api/chat. Drop overlong entries
client-side so a stale UI cannot 400 the request."""
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
async def run():
client = ExternalProviderClient(
provider_type = "kimi",
base_url = "https://api.moonshot.ai/v1",
api_key = "kimi-test",
)
async for _ in client.stream_chat_completion(
messages = [{"role": "user", "content": "hi"}],
model = "kimi-k2.6",
temperature = 1.0,
top_p = 1.0,
max_tokens = 256,
stop = ["END", "x" * 33, "DONE"],
):
pass
await client.close()
_drive(run())
body = captured["body"]
assert body.get("stop") == ["END", "DONE"], body
def test_kimi_web_search_drops_stop_strings_over_32_bytes(monkeypatch):
"""Same byte cap applies to the Kimi web-search bypass."""
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
async def run():
client = ExternalProviderClient(
provider_type = "kimi",
base_url = "https://api.moonshot.ai/v1",
api_key = "kimi-test",
)
async for _ in client.stream_chat_completion(
messages = [{"role": "user", "content": "hi"}],
model = "kimi-k2.6",
temperature = 1.0,
top_p = 1.0,
max_tokens = 256,
enabled_tools = ["web_search"],
stop = ["END", "x" * 40, "DONE"],
):
pass
await client.close()
_drive(run())
body = captured["body"]
assert body.get("stop") == ["END", "DONE"], body
def test_kimi_default_path_uses_kimi_stop_cap_5(monkeypatch):
"""The normal Kimi path must also honour the documented 5-cap."""
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())