From 91d04741ffb4b7cc1243dcb5dd2d1485f29b37cb Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 22 May 2026 15:01:37 +0000 Subject: [PATCH 01/56] Studio: expose Anthropic / OpenAI sampling knobs per provider Adds the missing sampling parameters that the upstream APIs accept and that Studio's chat UI previously hid. Each knob is gated per provider so the picker never offers a field the upstream would 400 on, and the per-provider stream functions translate / drop fields to match each API's naming. New `InferenceParams` fields (round-trip through PersistedInferenceParams and the chat-settings server store automatically): - frequencyPenalty (-2..2): OpenAI Chat Completions only. - seed (int | null): OpenAI Chat + OpenAI-compat local backends. - stop (string[]): all OpenAI Chat + Anthropic Messages. Backend truncates to 4 entries on OpenAI Chat per docs and renames to `stop_sequences` on Anthropic. - serviceTier (auto|default|flex|priority|scale|standard_only): per-provider enum sets resolved by getServiceTierOptions. - parallelToolCalls (bool, default true): forwarded as `parallel_tool_calls` on both OpenAI APIs and inverted into `disable_parallel_tool_use` on Anthropic. OpenAI Responses (gpt-5.x / o3) explicitly drops frequencyPenalty / seed / stop alongside the existing temperature / top_p drop, since the upstream 400s on all of them. service_tier on Responses accepts a subset (no `scale`) which the dispatch already enforces. UI rows land in the existing Sampling section of the chat settings sheet using ParamSlider (frequency penalty), a numeric Input (seed), a new chips editor `StopSequencesInput` (stop), Select (service tier), and Switch (parallel tool calls). Each row's visibility follows the new ProviderCapabilities flag. Tests pin the gating contract: stop_sequences renamed on Anthropic, 4-entry truncation on OpenAI Chat, every Responses-rejected field dropped, schema-level validation for the service_tier Literal and frequency_penalty range. Plan: plans/hashed-riding-porcupine.md --- .../core/inference/external_provider.py | 89 +++- studio/backend/models/inference.py | 42 ++ studio/backend/routes/inference.py | 5 + .../tests/test_sampling_params_routing.py | 385 ++++++++++++++++++ .../components/ui/stop-sequences-input.tsx | 117 ++++++ .../src/features/chat/api/chat-adapter.ts | 35 ++ .../src/features/chat/chat-settings-sheet.tsx | 157 ++++++- .../features/chat/provider-capabilities.ts | 95 ++++- .../chat/stores/chat-runtime-store.ts | 5 + .../frontend/src/features/chat/types/api.ts | 35 ++ .../src/features/chat/types/runtime.ts | 41 ++ 11 files changed, 1002 insertions(+), 4 deletions(-) create mode 100644 studio/backend/tests/test_sampling_params_routing.py create mode 100644 studio/frontend/src/components/ui/stop-sequences-input.tsx diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index 25e1725337..6565d8e643 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -11,7 +11,7 @@ Anthropic uses native Messages API with translation in this client. import json as _json import re import time -from typing import Any, AsyncGenerator, Literal, NamedTuple, Optional +from typing import Any, AsyncGenerator, Literal, NamedTuple, Optional, Union from urllib.parse import urlparse import httpx @@ -348,6 +348,11 @@ class ExternalProviderClient: anthropic_code_exec_container_id: Optional[str] = None, prompt_cache_ttl: Optional[str] = None, compaction_threshold: Optional[int] = None, + frequency_penalty: Optional[float] = None, + seed: Optional[int] = None, + stop: Optional[Union[str, list[str]]] = None, + service_tier: Optional[str] = None, + parallel_tool_calls: Optional[bool] = None, stream: bool = True, ) -> AsyncGenerator[str, None]: """ @@ -360,6 +365,12 @@ class ExternalProviderClient: supplies a value the provider accepts — the frontend's provider-capability map already filters these per provider, so we treat them as opt-in here. + + ``frequency_penalty``, ``seed``, ``stop``, ``service_tier``, + ``parallel_tool_calls`` follow the same rule: the per-provider + stream helpers silently drop fields the upstream API does not + accept (e.g. Responses rejects all of seed / frequency / stop; + Anthropic does not implement seed / frequency / logprobs). """ if not self._is_openai_compatible(): async for line in self._stream_anthropic( @@ -376,6 +387,9 @@ class ExternalProviderClient: anthropic_code_exec_container_id, prompt_cache_ttl, compaction_threshold, + stop = stop, + service_tier = service_tier, + parallel_tool_calls = parallel_tool_calls, ): yield line return @@ -398,6 +412,8 @@ class ExternalProviderClient: enable_prompt_caching, openai_code_exec_container_id, compaction_threshold, + service_tier = service_tier, + parallel_tool_calls = parallel_tool_calls, ): yield line return @@ -438,6 +454,38 @@ class ExternalProviderClient: else: body["max_tokens"] = max_tokens + # Optional sampling extensions (added in #5XXX). Only forwarded + # when the caller passed a value. Each upstream provider that + # 400s on the field appears in `body_omit` (see providers.py) + # so the registry-driven drop loop below removes them before + # the request hits the wire. The Responses path + # (_stream_openai_responses) drops these explicitly because it + # never reaches this body construction. + if frequency_penalty is not None: + body["frequency_penalty"] = frequency_penalty + if seed is not None: + body["seed"] = seed + if stop is not None: + # OpenAI Chat caps the list at 4 entries. Truncate with a + # warning rather than letting the upstream 400; users editing + # chips in the picker shouldn't get a cryptic API error. + if isinstance(stop, list) and len(stop) > 4: + logger.warning( + "stop sequences truncated to 4 entries " + "(received %d, OpenAI's hard cap is 4)", + len(stop), + ) + body["stop"] = stop[:4] + elif isinstance(stop, list) and len(stop) == 0: + # Empty list = unset; don't ship `stop: []`. + pass + else: + body["stop"] = stop + if service_tier is not None: + body["service_tier"] = service_tier + if parallel_tool_calls is not None: + body["parallel_tool_calls"] = parallel_tool_calls + # Strip body fields a provider's registry entry declares unusable — # reasoning-class models that lock these to fixed defaults (e.g. # Kimi k2.5/k2.6 only accept temperature=1, top_p=1) 400 otherwise. @@ -1186,6 +1234,10 @@ class ExternalProviderClient: anthropic_code_exec_container_id: Optional[str] = None, prompt_cache_ttl: Optional[str] = None, compaction_threshold: Optional[int] = None, + *, + stop: Optional[Union[str, list[str]]] = None, + service_tier: Optional[str] = None, + parallel_tool_calls: Optional[bool] = None, ) -> AsyncGenerator[str, None]: """ Call the Anthropic Messages API and translate its SSE to OpenAI format. @@ -1347,6 +1399,29 @@ class ExternalProviderClient: body["temperature"] = temperature if top_k is not None and top_k > 0 and not sampling_removed: body["top_k"] = top_k + + # Optional sampling extensions. Anthropic has no + # frequency_penalty / seed / logprobs equivalents, so those are + # silently dropped by virtue of not being forwarded from + # stream_chat_completion. The three knobs Anthropic does accept + # land here: + # stop → stop_sequences (renamed) + # service_tier → service_tier (auto|standard_only only) + # parallel_tool_calls → disable_parallel_tool_use (inverted) + if stop is not None: + sequences: list[str] + if isinstance(stop, str): + sequences = [stop] if stop else [] + else: + sequences = [s for s in stop if isinstance(s, str) and s] + if sequences: + body["stop_sequences"] = sequences + if service_tier in ("auto", "standard_only"): + body["service_tier"] = service_tier + if parallel_tool_calls is False: + # Default upstream behavior is parallel-allowed; only + # forward when the user explicitly disabled it. + body["disable_parallel_tool_use"] = True # Anthropic only caches a prefix when at least one cache_control # marker is attached to it — the frontend defaults # enable_prompt_caching to True for Anthropic, so treat `None` the @@ -2558,6 +2633,9 @@ class ExternalProviderClient: enable_prompt_caching: Optional[bool] = None, openai_code_exec_container_id: Optional[str] = None, compaction_threshold: Optional[int] = None, + *, + service_tier: Optional[str] = None, + parallel_tool_calls: Optional[bool] = None, ) -> AsyncGenerator[str, None]: """ Call OpenAI's /v1/responses endpoint and translate its SSE stream back @@ -2665,6 +2743,15 @@ class ExternalProviderClient: "input": input_items, "stream": True, } + # Responses accepts service_tier on the same enum set as Chat + # Completions minus `scale`. parallel_tool_calls follows the + # same shape (default true). The frontend capability gate + # (provider-capabilities.ts) already filters the option lists + # per provider, so we just forward what we got. + if service_tier in ("auto", "default", "flex", "priority"): + body["service_tier"] = service_tier + if parallel_tool_calls is not None: + body["parallel_tool_calls"] = bool(parallel_tool_calls) # `summary: "auto"` is what makes /v1/responses emit reasoning # summary events — without it OpenAI returns no thinking text on # most reasoning models, the SSE handler has no … diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index b5626951c4..0c1c3f814b 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -786,6 +786,48 @@ class ChatCompletionRequest(BaseModel): "to auto-create." ), ) + frequency_penalty: Optional[float] = Field( + None, + ge = -2.0, + le = 2.0, + description = ( + "OpenAI Chat Completions frequency penalty (-2.0 to 2.0). " + "Forwarded only on providers that accept it; Anthropic " + "Messages and the OpenAI Responses family silently drop it." + ), + ) + seed: Optional[int] = Field( + None, + description = ( + "Best-effort determinism seed. Forwarded to OpenAI Chat " + "Completions and OpenAI-compatible local backends. The " + "Responses family rejects it server-side and Anthropic does " + "not implement it, so it is silently dropped on those routes." + ), + ) + service_tier: Optional[ + Literal["auto", "default", "flex", "priority", "scale", "standard_only"] + ] = Field( + None, + description = ( + "Provider service tier. Anthropic accepts only `auto` and " + "`standard_only`; OpenAI Chat accepts " + "`auto|default|flex|priority|scale`; OpenAI Responses accepts " + "`auto|default|flex|priority`. Unsupported values per provider " + "are dropped in the per-provider stream helper instead of " + "422'ing here so a stale frontend never breaks a request." + ), + ) + parallel_tool_calls: Optional[bool] = Field( + None, + description = ( + "Whether the provider may dispatch tool calls in parallel. " + "OpenAI: forwarded as `parallel_tool_calls`. Anthropic: " + "inverted into `disable_parallel_tool_use` on the Messages " + "body. Default `None` preserves each provider's upstream " + "default (which is `true` everywhere today)." + ), + ) @model_validator(mode = "after") def _resolve_missing_tool_call_ids(self) -> "ChatCompletionRequest": diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 02270ab405..b6e4c4e370 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1868,6 +1868,11 @@ async def _proxy_to_external_provider( anthropic_code_exec_container_id = payload.anthropic_code_exec_container_id, prompt_cache_ttl = payload.prompt_cache_ttl, compaction_threshold = payload.compaction_threshold, + frequency_penalty = payload.frequency_penalty, + seed = payload.seed, + stop = payload.stop, + service_tier = payload.service_tier, + parallel_tool_calls = payload.parallel_tool_calls, stream = payload.stream, ) try: diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py new file mode 100644 index 0000000000..f00b1b12f4 --- /dev/null +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -0,0 +1,385 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""End-to-end routing tests for the new sampling parameters. + +Pins the per-provider gating contract added by the +expose-sampling-params PR: each of `frequency_penalty`, `seed`, `stop` +/ `stop_sequences`, `service_tier`, `parallel_tool_calls` only appears +on the outbound body when the upstream provider actually accepts it. + +The provider matrix is captured per docs: +- Anthropic Messages: accepts stop_sequences, service_tier + (auto|standard_only), disable_parallel_tool_use (inverted). REJECTS + frequency_penalty, seed, logprobs (silently dropped client-side). +- OpenAI Chat Completions (default OAI-compat branch): accepts every + field; OpenAI cloud uses `max_completion_tokens` rather than + `max_tokens`. +- OpenAI Responses (gpt-5.x / o3): rejects temperature, top_p, + frequency_penalty, seed, stop, logprobs. Accepts service_tier + (auto|default|flex|priority) and parallel_tool_calls. +""" + +import asyncio +import json + +import httpx +import pytest + +from core.inference import external_provider as ep_mod +from core.inference.external_provider import ExternalProviderClient + + +def _drive(coro): + return asyncio.new_event_loop().run_until_complete(coro) + + +def _install_mock(monkeypatch, *, sse_payload: bytes | None = None) -> dict: + captured: dict = {} + + def handler(request: httpx.Request) -> httpx.Response: + try: + captured["body"] = json.loads(request.content.decode("utf-8")) + except json.JSONDecodeError: + captured["body"] = None + captured["url"] = str(request.url) + captured["headers"] = dict(request.headers) + return httpx.Response( + 200, + content = sse_payload + or (b"event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"), + headers = {"content-type": "text/event-stream"}, + ) + + monkeypatch.setattr( + ep_mod, + "_http_client", + httpx.AsyncClient(transport = httpx.MockTransport(handler)), + ) + return captured + + +# ── Anthropic ────────────────────────────────────────────────────────── + + +def _drive_anthropic(captured, **kwargs) -> dict: + async def run(): + client = ExternalProviderClient( + provider_type = "anthropic", + base_url = "https://api.anthropic.com/v1", + api_key = "sk-ant-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "claude-opus-4-7", + temperature = 0.7, + top_p = 0.95, + max_tokens = 64, + **kwargs, + ): + pass + await client.close() + + _drive(run()) + return captured["body"] + + +def test_anthropic_stop_sequences_forwarded_as_renamed_field(monkeypatch): + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, stop = ["END", "DONE"]) + assert body.get("stop_sequences") == ["END", "DONE"], body + # Anthropic does not have a `stop` field; the unrenamed key must not appear. + assert "stop" not in body, body + + +def test_anthropic_single_string_stop_is_wrapped(monkeypatch): + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, stop = "STOPHERE") + assert body.get("stop_sequences") == ["STOPHERE"], body + + +def test_anthropic_empty_stop_omitted(monkeypatch): + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, stop = []) + assert "stop_sequences" not in body, body + assert "stop" not in body, body + + +def test_anthropic_service_tier_forwarded_when_valid(monkeypatch): + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, service_tier = "standard_only") + assert body.get("service_tier") == "standard_only", body + + +@pytest.mark.parametrize("bogus", ["flex", "priority", "scale", "default", "", "auto-foo"]) +def test_anthropic_service_tier_unsupported_values_dropped(monkeypatch, bogus): + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, service_tier = bogus) + assert "service_tier" not in body, body + + +def test_anthropic_disable_parallel_tool_use_only_when_false(monkeypatch): + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, parallel_tool_calls = False) + assert body.get("disable_parallel_tool_use") is True, body + # Anthropic has no `parallel_tool_calls` field. + assert "parallel_tool_calls" not in body, body + + +def test_anthropic_parallel_tool_calls_default_not_sent(monkeypatch): + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, parallel_tool_calls = True) + # True is the upstream default; do not surface + # `disable_parallel_tool_use: false` which would over-specify the request. + assert "disable_parallel_tool_use" not in body, body + assert "parallel_tool_calls" not in body, body + + +def test_anthropic_rejects_openai_only_knobs(monkeypatch): + """frequency_penalty / seed are dropped at the dispatch layer. + + Anthropic has no equivalent; the keyword args are not even forwarded + from stream_chat_completion to _stream_anthropic. This test pins + that no such field reaches the Messages body. + """ + captured = _install_mock(monkeypatch) + body = _drive_anthropic( + captured, + frequency_penalty = 1.5, + seed = 42, + ) + assert "frequency_penalty" not in body, body + assert "seed" not in body, body + + +# ── OpenAI Chat Completions (default OAI-compat) ───────────────────────── + + +def _drive_openai_compat(captured, **kwargs) -> dict: + """Send through the default OAI-compat branch (NOT /v1/responses). + + Use a non-OpenAI provider_type so the dispatcher takes the default + branch at the bottom of stream_chat_completion rather than the + Responses translator path that routes provider_type=="openai". + """ + + async def run(): + client = ExternalProviderClient( + provider_type = "mistral", + base_url = "https://api.mistral.ai/v1", + api_key = "test-key", + ) + # mistral's OpenAI-compat /v1/chat/completions returns OpenAI + # SSE; a single DONE frame is enough to drain the stream. + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "mistral-small-latest", + temperature = 0.5, + top_p = 0.9, + max_tokens = 64, + **kwargs, + ): + pass + await client.close() + + _drive(run()) + return captured["body"] + + +def _oai_done_payload() -> bytes: + return b"data: [DONE]\n\n" + + +def test_openai_compat_forwards_frequency_penalty(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured, frequency_penalty = 1.25) + assert body.get("frequency_penalty") == 1.25, body + + +def test_openai_compat_forwards_seed(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured, seed = 12345) + assert body.get("seed") == 12345, body + + +def test_openai_compat_forwards_stop_array(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured, stop = ["END", "DONE"]) + assert body.get("stop") == ["END", "DONE"], body + # The default OAI-compat branch does not rename to stop_sequences. + assert "stop_sequences" not in body, body + + +def test_openai_compat_truncates_stop_to_four(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat( + captured, stop = ["a", "b", "c", "d", "e", "f"] + ) + assert body.get("stop") == ["a", "b", "c", "d"], body + + +def test_openai_compat_empty_stop_omitted(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured, stop = []) + assert "stop" not in body, body + + +def test_openai_compat_forwards_service_tier(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured, service_tier = "flex") + assert body.get("service_tier") == "flex", body + + +def test_openai_compat_forwards_parallel_tool_calls(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured, parallel_tool_calls = False) + assert body.get("parallel_tool_calls") is False, body + + +def test_openai_compat_omits_unset_optionals(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured) + # Optional knobs default to None / unset -> never appear. + assert "frequency_penalty" not in body, body + assert "seed" not in body, body + assert "stop" not in body, body + assert "service_tier" not in body, body + assert "parallel_tool_calls" not in body, body + + +# ── OpenAI Responses (gpt-5.x via /v1/responses) ───────────────────────── + + +def _responses_done_payload() -> bytes: + return ( + b"event: response.completed\n" + b"data: {\"type\":\"response.completed\",\"response\":{\"usage\":{}}}\n\n" + ) + + +def _drive_openai_responses(captured, **kwargs) -> dict: + async def run(): + client = ExternalProviderClient( + provider_type = "openai", + base_url = "https://api.openai.com/v1", + api_key = "sk-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "gpt-5.5", + temperature = 1.0, + top_p = 1.0, + max_tokens = 64, + **kwargs, + ): + pass + await client.close() + + _drive(run()) + return captured["body"] + + +def test_openai_responses_drops_temperature_top_p(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) + body = _drive_openai_responses(captured) + assert "temperature" not in body, body + assert "top_p" not in body, body + + +def test_openai_responses_drops_frequency_penalty_seed_stop(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) + body = _drive_openai_responses( + captured, + frequency_penalty = 1.5, + seed = 99, + stop = ["END"], + ) + # Responses 400s on any of these; the dispatch must drop them + # before they hit the wire. + assert "frequency_penalty" not in body, body + assert "seed" not in body, body + assert "stop" not in body, body + assert "stop_sequences" not in body, body + + +def test_openai_responses_forwards_service_tier(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) + body = _drive_openai_responses(captured, service_tier = "priority") + assert body.get("service_tier") == "priority", body + + +def test_openai_responses_rejects_chat_only_service_tier(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) + body = _drive_openai_responses(captured, service_tier = "scale") + # Responses only accepts auto|default|flex|priority -- `scale` is + # silently dropped so a stale frontend cannot 400 the request. + assert "service_tier" not in body, body + + +def test_openai_responses_forwards_parallel_tool_calls(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) + body = _drive_openai_responses(captured, parallel_tool_calls = False) + assert body.get("parallel_tool_calls") is False, body + + +def test_openai_responses_omits_unset_optionals(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) + body = _drive_openai_responses(captured) + assert "service_tier" not in body, body + assert "parallel_tool_calls" not in body, body + + +# ── Schema-level smoke tests ───────────────────────────────────────────── + + +def test_chat_completion_request_accepts_new_sampling_fields(): + from models.inference import ChatCompletionRequest + + payload = ChatCompletionRequest.model_validate( + { + "messages": [{"role": "user", "content": "hi"}], + "frequency_penalty": -1.0, + "seed": 0, + "stop": ["END"], + "service_tier": "auto", + "parallel_tool_calls": True, + } + ) + assert payload.frequency_penalty == -1.0 + assert payload.seed == 0 + assert payload.stop == ["END"] + assert payload.service_tier == "auto" + assert payload.parallel_tool_calls is True + + +def test_chat_completion_request_rejects_bad_service_tier(): + import pydantic + from models.inference import ChatCompletionRequest + + with pytest.raises(pydantic.ValidationError): + ChatCompletionRequest.model_validate( + { + "messages": [{"role": "user", "content": "hi"}], + "service_tier": "bogus", + } + ) + + +def test_chat_completion_request_clamps_frequency_penalty_range(): + import pydantic + from models.inference import ChatCompletionRequest + + with pytest.raises(pydantic.ValidationError): + ChatCompletionRequest.model_validate( + { + "messages": [{"role": "user", "content": "hi"}], + "frequency_penalty": 3.0, + } + ) + with pytest.raises(pydantic.ValidationError): + ChatCompletionRequest.model_validate( + { + "messages": [{"role": "user", "content": "hi"}], + "frequency_penalty": -3.0, + } + ) diff --git a/studio/frontend/src/components/ui/stop-sequences-input.tsx b/studio/frontend/src/components/ui/stop-sequences-input.tsx new file mode 100644 index 0000000000..223c91c16d --- /dev/null +++ b/studio/frontend/src/components/ui/stop-sequences-input.tsx @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { cn } from "@/lib/utils"; +import { XIcon } from "lucide-react"; +import { type KeyboardEvent, useState } from "react"; + +/** + * Chips editor for the `stop` / `stop_sequences` array. + * + * Commit a chip with Enter or comma; press Backspace on an empty input + * to delete the most recent chip. Caps at `maxEntries` -- OpenAI Chat + * Completions documents a hard cap of 4 stop sequences; Anthropic + * Messages accepts arbitrarily many. Pass `Infinity` to disable the cap. + */ +export interface StopSequencesInputProps { + value: string[]; + onChange: (next: string[]) => void; + maxEntries?: number; + disabled?: boolean; + placeholder?: string; + className?: string; + "aria-label"?: string; +} + +export function StopSequencesInput({ + value, + onChange, + maxEntries = 4, + disabled, + placeholder = "Add stop sequence", + className, + "aria-label": ariaLabel, +}: StopSequencesInputProps) { + const [draft, setDraft] = useState(""); + const atCap = value.length >= maxEntries; + + function commitDraft() { + const trimmed = draft.trim(); + if (!trimmed) return; + if (atCap) return; + if (value.includes(trimmed)) { + setDraft(""); + return; + } + onChange([...value, trimmed]); + setDraft(""); + } + + function removeChip(index: number) { + if (disabled) return; + onChange(value.filter((_, i) => i !== index)); + } + + function handleKeyDown(event: KeyboardEvent) { + if (disabled) return; + if (event.key === "Enter" || event.key === ",") { + event.preventDefault(); + commitDraft(); + return; + } + if (event.key === "Backspace" && !draft && value.length > 0) { + event.preventDefault(); + onChange(value.slice(0, -1)); + } + } + + return ( +
+ {value.map((entry, index) => ( + + {entry} + {!disabled ? ( + + ) : null} + + ))} + setDraft(event.target.value)} + onKeyDown={handleKeyDown} + onBlur={commitDraft} + placeholder={atCap ? `Max ${maxEntries} stops` : placeholder} + disabled={disabled || atCap} + className={cn( + "h-6 min-w-[8ch] flex-1 border-0 bg-transparent p-0 text-sm shadow-none", + "focus-visible:ring-0 focus-visible:border-0", + )} + /> +
+ ); +} diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index 0c557f1b01..6f01c0becd 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -1403,6 +1403,29 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { ...(externalCapabilities?.presencePenalty ? { presence_penalty: params.presencePenalty } : {}), + // Optional sampling extensions. Each gate is per-provider + // (see provider-capabilities.ts); the backend additionally + // drops fields the upstream API does not accept, so a + // stale frontend cannot 400 the request. + ...(externalCapabilities?.frequencyPenalty + ? { frequency_penalty: params.frequencyPenalty } + : {}), + ...(externalCapabilities?.seed && params.seed !== null + ? { seed: params.seed } + : {}), + ...(externalCapabilities?.stop && params.stop.length > 0 + ? { stop: params.stop } + : {}), + ...(externalCapabilities?.serviceTier && params.serviceTier + ? { service_tier: params.serviceTier } + : {}), + // Forward parallel_tool_calls when the user explicitly + // turned it off (the upstream default is `true` on every + // provider we ship, so default true is a no-op). + ...(externalCapabilities?.parallelToolCalls && + params.parallelToolCalls === false + ? { parallel_tool_calls: false } + : {}), // Built-in tools: Search pill maps to provider-side // web_search (currently OpenAI / Anthropic / OpenRouter / // Kimi); Code pill maps to Anthropic's server-side @@ -1503,6 +1526,18 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { min_p: params.minP, repetition_penalty: params.repetitionPenalty, presence_penalty: params.presencePenalty, + // Optional sampling extensions; local llama-server already + // accepts `stop` / `seed` / `frequency_penalty` via + // _build_passthrough_payload (routes/inference.py:4884) and + // silently ignores fields it does not recognise. + ...(params.frequencyPenalty !== 0 + ? { frequency_penalty: params.frequencyPenalty } + : {}), + ...(params.seed !== null ? { seed: params.seed } : {}), + ...(params.stop.length > 0 ? { stop: params.stop } : {}), + ...(params.parallelToolCalls === false + ? { parallel_tool_calls: false } + : {}), image_base64: imageBase64, audio_base64: audioBase64, cancel_id: cancelId, diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index 9cd4db2705..5c0c531142 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -86,10 +86,13 @@ import { EXTERNAL_MAX_OUTPUT_TOKENS, type ProviderCapabilities, getExternalMinOutputTokens, + getServiceTierOptions, providerSupportsBuiltinCodeExecution, } from "./provider-capabilities"; +import { StopSequencesInput } from "@/components/ui/stop-sequences-input"; import { useChatRuntimeStore } from "./stores/chat-runtime-store"; -import type { InferenceParams } from "./types/runtime"; +import type { InferenceParams, ServiceTier } from "./types/runtime"; +import { Input } from "@/components/ui/input"; export { defaultInferenceParams, type Preset } from "./presets/preset-policy"; export type { InferenceParams } from "./types/runtime"; @@ -416,6 +419,19 @@ export function ChatSettingsPanel({ !isExternalModel || Boolean(providerCapabilities?.repetitionPenalty); const showPresencePenalty = !isExternalModel || Boolean(providerCapabilities?.presencePenalty); + const showFrequencyPenalty = + !isExternalModel || Boolean(providerCapabilities?.frequencyPenalty); + const showSeed = !isExternalModel || Boolean(providerCapabilities?.seed); + const showStop = !isExternalModel || Boolean(providerCapabilities?.stop); + const showServiceTier = + isExternalModel && Boolean(providerCapabilities?.serviceTier); + const showParallelToolCalls = + !isExternalModel || Boolean(providerCapabilities?.parallelToolCalls); + // OpenAI Chat docs cap `stop` at 4 entries; Anthropic accepts more. + // Pick the right ceiling per active connection so the chips editor's + // placeholder doesn't lie. + const stopMaxEntries = externalProviderType === "anthropic" ? 16 : 4; + const serviceTierOptions = getServiceTierOptions(externalProviderType); const isMobile = useIsMobile(); const isGguf = useChatRuntimeStore((s) => s.activeGgufVariant) != null; const hasModelContent = @@ -1259,6 +1275,145 @@ export function ChatSettingsPanel({ info="Penalizes any token that has already appeared at least once, encouraging the model to introduce new topics. 0 = off." /> ) : null} + {showFrequencyPenalty ? ( + + ) : null} + {showSeed ? ( +
+
+ + Seed + + + Best-effort determinism seed. Same seed + prompt = + same output (approximately). OpenAI Chat Completions + and OpenAI-compat local backends honor it; OpenAI + Responses and Anthropic silently drop it. + +
+ { + const raw = event.target.value; + if (raw === "") { + set("seed")(null); + return; + } + const parsed = Number.parseInt(raw, 10); + if (Number.isFinite(parsed)) { + set("seed")(parsed); + } + }} + className="h-8 w-[124px] shrink-0 text-right font-mono text-xs" + aria-label="Seed" + /> +
+ ) : null} + {showStop ? ( +
+
+ + Stop sequences + + + Strings that halt generation as soon as the model + emits them. Enter a value and press Enter or comma + to commit a chip. Backend translates to + `stop_sequences` on Anthropic and `stop` on OpenAI + Chat (capped at 4 entries). + +
+ +
+ ) : null} + {showServiceTier ? ( +
+
+ + Service tier + + + Provider routing tier. `auto` (default) lets the + provider choose. `flex` / `priority` / `scale` route + to higher-latency-tolerant or premium queues on + OpenAI; `standard_only` opts out of Anthropic's + priority tier. + +
+ +
+ ) : null} + {showParallelToolCalls ? ( +
+
+ + Parallel tool calls + + + When on, the model may dispatch multiple tool calls + in a single turn (default). Turn off to force one + tool call at a time. Anthropic implements this as + `disable_parallel_tool_use`; OpenAI as + `parallel_tool_calls`. + +
+ +
+ ) : null} {!isExternalModel && !isGguf && ( = { @@ -302,6 +368,8 @@ const PROVIDER_CAPABILITIES: Record = { // models served via /v1/responses, which rejects temperature, top_p, and // presence/frequency penalty. See backend // external_provider._stream_openai_responses for the proxy. + // service_tier and parallel_tool_calls are accepted on /v1/responses; + // seed / stop / frequency_penalty are 400'd alongside temperature/top_p. openai: { temperature: false, topP: false, @@ -309,14 +377,22 @@ const PROVIDER_CAPABILITIES: Record = { minP: false, repetitionPenalty: false, presencePenalty: false, + frequencyPenalty: false, + seed: false, + stop: false, + serviceTier: true, + parallelToolCalls: true, }, // Anthropic's Messages API accepts top_k on 3.x and 4.5/4.6, but Claude // 4.7 (Opus/Sonnet/Haiku) deprecated it and returns 400 if it is set. // We surface top_k in the panel for all Anthropic providers and let the // backend strip it per-model — see _stream_anthropic in // studio/backend/core/inference/external_provider.py. - // Presence/frequency penalty is not part of the Messages API on any - // Claude generation. + // Presence/frequency penalty / seed / logprobs are not part of the + // Messages API on any Claude generation. stop_sequences (Anthropic name + // for `stop`), service_tier (auto|standard_only), and + // disable_parallel_tool_use (inverse of parallel_tool_calls) ARE + // supported. anthropic: { temperature: true, topP: true, @@ -324,6 +400,11 @@ const PROVIDER_CAPABILITIES: Record = { minP: false, repetitionPenalty: false, presencePenalty: false, + frequencyPenalty: false, + seed: false, + stop: true, + serviceTier: true, + parallelToolCalls: true, }, mistral: OPENAI_COMPAT_BASE, gemini: OPENAI_COMPAT_BASE, @@ -340,6 +421,11 @@ const PROVIDER_CAPABILITIES: Record = { minP: false, repetitionPenalty: false, presencePenalty: true, + frequencyPenalty: true, + seed: true, + stop: true, + serviceTier: false, + parallelToolCalls: true, }, // DeepSeek deprecated presence/frequency penalty in their current docs. deepseek: { @@ -349,6 +435,11 @@ const PROVIDER_CAPABILITIES: Record = { minP: false, repetitionPenalty: false, presencePenalty: false, + frequencyPenalty: false, + seed: true, + stop: true, + serviceTier: false, + parallelToolCalls: true, }, qwen: OPENAI_COMPAT_BASE, huggingface: OPENAI_COMPAT_BASE, diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts index a00b53a44c..dd26006228 100644 --- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts +++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts @@ -373,6 +373,11 @@ const PERSISTED_INFERENCE_PARAM_KEYS = [ "minP", "repetitionPenalty", "presencePenalty", + "frequencyPenalty", + "seed", + "stop", + "serviceTier", + "parallelToolCalls", "maxSeqLength", "maxTokens", "systemPrompt", diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index 1e6bcf8b87..764e571bfa 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -262,6 +262,41 @@ export interface OpenAIChatCompletionsRequest { * the Anthropic provider with `code_execution` in `enabled_tools`. */ anthropic_code_exec_container_id?: string | null; + /** + * OpenAI Chat Completions only; rejected by the Responses family and + * silently dropped by Anthropic. Range -2.0 .. 2.0. + */ + frequency_penalty?: number; + /** + * Best-effort determinism seed. OpenAI Chat / OpenAI-compat backends + * forward it; Responses + Anthropic drop it server-side. + */ + seed?: number; + /** + * Custom stop sequences. Backend translates to `stop_sequences` for + * Anthropic; OpenAI Chat caps at 4 entries (server-side truncates + * with a warning). Empty arrays are omitted. + */ + stop?: string[]; + /** + * Provider service tier. Anthropic accepts `auto|standard_only`; + * OpenAI Chat accepts `auto|default|flex|priority|scale`; OpenAI + * Responses accepts `auto|default|flex|priority`. + */ + service_tier?: + | "auto" + | "default" + | "flex" + | "priority" + | "scale" + | "standard_only"; + /** + * Whether the provider may dispatch tool calls in parallel. + * OpenAI: forwarded as `parallel_tool_calls`. Anthropic: inverted + * into `disable_parallel_tool_use` server-side. Default `undefined` + * keeps each provider's upstream default. + */ + parallel_tool_calls?: boolean; } export interface OpenAIChatDelta { diff --git a/studio/frontend/src/features/chat/types/runtime.ts b/studio/frontend/src/features/chat/types/runtime.ts index 2967584653..49c7c291e7 100644 --- a/studio/frontend/src/features/chat/types/runtime.ts +++ b/studio/frontend/src/features/chat/types/runtime.ts @@ -1,6 +1,14 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 +export type ServiceTier = + | "auto" + | "default" + | "flex" + | "priority" + | "scale" + | "standard_only"; + export interface InferenceParams { temperature: number; topP: number; @@ -8,6 +16,34 @@ export interface InferenceParams { minP: number; repetitionPenalty: number; presencePenalty: number; + /** OpenAI Chat Completions only; rejected by Responses + Anthropic. */ + frequencyPenalty: number; + /** + * Best-effort determinism seed. OpenAI Chat Completions only; the + * Responses family and Anthropic reject it (silently dropped server-side). + * `null` = unset (no `seed` field on the wire). + */ + seed: number | null; + /** + * Custom stop sequences. Maps to `stop` on OpenAI Chat Completions and + * `stop_sequences` on Anthropic Messages. OpenAI caps the array at 4 + * entries; backend truncates with a warning. Empty array = unset. + */ + stop: string[]; + /** + * Provider service tier. Each provider accepts a different enum set; + * `getServiceTierOptions(providerType)` resolves the legal values. `null` + * means "let the provider pick its default" and is the safe choice on + * provider switch. + */ + serviceTier: ServiceTier | null; + /** + * Whether the provider may dispatch tool calls in parallel. Maps to + * `parallel_tool_calls` on both OpenAI APIs and is inverted into + * `disable_parallel_tool_use` for Anthropic. Default true matches the + * upstream defaults across all three. + */ + parallelToolCalls: boolean; maxSeqLength: number; maxTokens: number; systemPrompt: string; @@ -23,6 +59,11 @@ export const DEFAULT_INFERENCE_PARAMS: InferenceParams = { minP: 0.01, repetitionPenalty: 1.0, presencePenalty: 0.0, + frequencyPenalty: 0.0, + seed: null, + stop: [], + serviceTier: null, + parallelToolCalls: true, maxSeqLength: 4096, maxTokens: 8192, systemPrompt: "", From 3cd3a640887156b318b230b10f4c2efeea615344 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 15:03:09 +0000 Subject: [PATCH 02/56] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/tests/test_sampling_params_routing.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index f00b1b12f4..30eb64e1fe 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -47,7 +47,7 @@ def _install_mock(monkeypatch, *, sse_payload: bytes | None = None) -> dict: return httpx.Response( 200, content = sse_payload - or (b"event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"), + or (b'event: message_stop\ndata: {"type":"message_stop"}\n\n'), headers = {"content-type": "text/event-stream"}, ) @@ -111,7 +111,9 @@ def test_anthropic_service_tier_forwarded_when_valid(monkeypatch): assert body.get("service_tier") == "standard_only", body -@pytest.mark.parametrize("bogus", ["flex", "priority", "scale", "default", "", "auto-foo"]) +@pytest.mark.parametrize( + "bogus", ["flex", "priority", "scale", "default", "", "auto-foo"] +) def test_anthropic_service_tier_unsupported_values_dropped(monkeypatch, bogus): captured = _install_mock(monkeypatch) body = _drive_anthropic(captured, service_tier = bogus) @@ -212,9 +214,7 @@ def test_openai_compat_forwards_stop_array(monkeypatch): def test_openai_compat_truncates_stop_to_four(monkeypatch): captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) - body = _drive_openai_compat( - captured, stop = ["a", "b", "c", "d", "e", "f"] - ) + body = _drive_openai_compat(captured, stop = ["a", "b", "c", "d", "e", "f"]) assert body.get("stop") == ["a", "b", "c", "d"], body @@ -253,7 +253,7 @@ def test_openai_compat_omits_unset_optionals(monkeypatch): def _responses_done_payload() -> bytes: return ( b"event: response.completed\n" - b"data: {\"type\":\"response.completed\",\"response\":{\"usage\":{}}}\n\n" + b'data: {"type":"response.completed","response":{"usage":{}}}\n\n' ) From 807165810fb41c3cdb13029d456a966eb716589d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 22 May 2026 15:46:52 +0000 Subject: [PATCH 03/56] Address review feedback on sampling-params knobs - Drop `scale` from the OpenAI service-tier picker (frontend types and picker option list). OpenAI in Studio routes through `/v1/responses`, which does not accept `scale`; offering it in the UI silently dropped the value at the backend and misled users into thinking their selection was applied. Backend Literal still accepts it on input so stale clients are not 422'd, and `_stream_openai_responses` continues to drop it from the wire body. - Dedupe + drop empty entries for OpenAI Chat `stop` and Anthropic `stop_sequences` before forwarding so whitespace chips or accidental repeats do not waste the 4-entry OpenAI cap or the 16-entry Anthropic cap. Anthropic over-cap now logs and truncates, matching the OpenAI path. - Static `aria-label="Parallel tool calls"` on the Switch; screen readers already announce checked / unchecked state, so the dynamic Enable/Disable label was redundant. - Forward an `aria-label` onto the inner Input inside `StopSequencesInput` so screen-reader users can identify the field. - Regression tests covering the new dedup, truncation, and the preserved silent-drop of `scale` on Responses. --- .../core/inference/external_provider.py | 51 ++++++++++++------- .../tests/test_sampling_params_routing.py | 30 +++++++++++ .../components/ui/stop-sequences-input.tsx | 1 + .../src/features/chat/chat-settings-sheet.tsx | 6 +-- .../features/chat/provider-capabilities.ts | 10 ++-- .../frontend/src/features/chat/types/api.ts | 1 - .../src/features/chat/types/runtime.ts | 1 - 7 files changed, 71 insertions(+), 29 deletions(-) diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index 6565d8e643..fa9eb0f13d 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -465,22 +465,26 @@ class ExternalProviderClient: body["frequency_penalty"] = frequency_penalty if seed is not None: body["seed"] = seed - if stop is not None: - # OpenAI Chat caps the list at 4 entries. Truncate with a - # warning rather than letting the upstream 400; users editing - # chips in the picker shouldn't get a cryptic API error. - if isinstance(stop, list) and len(stop) > 4: - logger.warning( - "stop sequences truncated to 4 entries " - "(received %d, OpenAI's hard cap is 4)", - len(stop), - ) - body["stop"] = stop[:4] - elif isinstance(stop, list) and len(stop) == 0: - # Empty list = unset; don't ship `stop: []`. - pass - else: + if stop: + # OpenAI Chat caps the list at 4 entries. Dedupe + drop + # empties first so users entering chips with whitespace or + # accidental repeats don't waste budget against the cap or + # trip a 400. + if isinstance(stop, str): body["stop"] = stop + elif isinstance(stop, list): + sequences = list( + dict.fromkeys(s for s in stop if isinstance(s, str) and s) + ) + if len(sequences) > 4: + logger.warning( + "stop sequences truncated to 4 entries " + "(received %d, OpenAI's hard cap is 4)", + len(sequences), + ) + body["stop"] = sequences[:4] + elif sequences: + body["stop"] = sequences if service_tier is not None: body["service_tier"] = service_tier if parallel_tool_calls is not None: @@ -1408,12 +1412,23 @@ class ExternalProviderClient: # stop → stop_sequences (renamed) # service_tier → service_tier (auto|standard_only only) # parallel_tool_calls → disable_parallel_tool_use (inverted) - if stop is not None: + if stop: sequences: list[str] if isinstance(stop, str): - sequences = [stop] if stop else [] + sequences = [stop] else: - sequences = [s for s in stop if isinstance(s, str) and s] + # Dedupe + drop empties first; Anthropic 400s on empty + # entries and the docs cap stop_sequences at 16 entries. + sequences = list( + dict.fromkeys(s for s in stop if isinstance(s, str) and s) + ) + if len(sequences) > 16: + logger.warning( + "stop_sequences truncated to 16 entries " + "(received %d, Anthropic's hard cap is 16)", + len(sequences), + ) + sequences = sequences[:16] if sequences: body["stop_sequences"] = sequences if service_tier in ("auto", "standard_only"): diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index 30eb64e1fe..e5a16e03ad 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -105,6 +105,27 @@ def test_anthropic_empty_stop_omitted(monkeypatch): assert "stop" not in body, body +def test_anthropic_stop_sequences_dedup_and_drop_empties(monkeypatch): + """Whitespace-only / duplicate chips shouldn't reach the wire and + waste budget against Anthropic's 16-entry cap.""" + captured = _install_mock(monkeypatch) + body = _drive_anthropic( + captured, stop = ["END", "", "END", "DONE", " ", "END"] + ) + # Order preserved on first sight, duplicates and empties dropped. + assert body.get("stop_sequences") == ["END", "DONE", " "], body + + +def test_anthropic_stop_sequences_truncated_to_16(monkeypatch): + captured = _install_mock(monkeypatch) + body = _drive_anthropic( + captured, stop = [f"S{i}" for i in range(20)] + ) + assert len(body.get("stop_sequences", [])) == 16, body + assert body["stop_sequences"][0] == "S0" + assert body["stop_sequences"][-1] == "S15" + + def test_anthropic_service_tier_forwarded_when_valid(monkeypatch): captured = _install_mock(monkeypatch) body = _drive_anthropic(captured, service_tier = "standard_only") @@ -218,6 +239,15 @@ def test_openai_compat_truncates_stop_to_four(monkeypatch): assert body.get("stop") == ["a", "b", "c", "d"], body +def test_openai_compat_stop_dedup_and_drop_empties(monkeypatch): + """Duplicates and empties shouldn't eat into the 4-entry cap.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat( + captured, stop = ["END", "", "END", "DONE", "FIN", "END"] + ) + assert body.get("stop") == ["END", "DONE", "FIN"], body + + def test_openai_compat_empty_stop_omitted(monkeypatch): captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) body = _drive_openai_compat(captured, stop = []) diff --git a/studio/frontend/src/components/ui/stop-sequences-input.tsx b/studio/frontend/src/components/ui/stop-sequences-input.tsx index 223c91c16d..e3c2d8a561 100644 --- a/studio/frontend/src/components/ui/stop-sequences-input.tsx +++ b/studio/frontend/src/components/ui/stop-sequences-input.tsx @@ -107,6 +107,7 @@ export function StopSequencesInput({ onBlur={commitDraft} placeholder={atCap ? `Max ${maxEntries} stops` : placeholder} disabled={disabled || atCap} + aria-label={ariaLabel || placeholder} className={cn( "h-6 min-w-[8ch] flex-1 border-0 bg-transparent p-0 text-sm shadow-none", "focus-visible:ring-0 focus-visible:border-0", diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index 5c0c531142..b568916c05 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -1406,11 +1406,7 @@ export function ChatSettingsPanel({ className="panel-switch shrink-0" checked={params.parallelToolCalls} onCheckedChange={set("parallelToolCalls")} - aria-label={ - params.parallelToolCalls - ? "Disable parallel tool calls" - : "Enable parallel tool calls" - } + aria-label="Parallel tool calls" /> ) : null} diff --git a/studio/frontend/src/features/chat/provider-capabilities.ts b/studio/frontend/src/features/chat/provider-capabilities.ts index 4971101eb0..0c63fed5bc 100644 --- a/studio/frontend/src/features/chat/provider-capabilities.ts +++ b/studio/frontend/src/features/chat/provider-capabilities.ts @@ -65,13 +65,15 @@ export type ServiceTierOption = | "default" | "flex" | "priority" - | "scale" | "standard_only"; /** * Legal `service_tier` values per provider, sourced from each upstream's - * docs. Anthropic exposes only `auto` and `standard_only`; OpenAI Chat - * Completions adds `scale` on top of the Responses-family set. Other + * docs. Anthropic exposes only `auto` and `standard_only`. OpenAI in + * Studio is routed through `/v1/responses` (not Chat Completions), and + * the Responses endpoint only accepts `auto` / `default` / `flex` / + * `priority` — `scale` is Chat-only and would be silently dropped here, + * so the option list omits it to avoid misleading the user. Other * providers fall through to a permissive `auto` / `default` pair so the * picker stays usable for OpenAI-compat backends. */ @@ -82,7 +84,7 @@ export function getServiceTierOptions( return ["auto", "standard_only"] as const; } if (providerType === "openai") { - return ["auto", "default", "flex", "priority", "scale"] as const; + return ["auto", "default", "flex", "priority"] as const; } return ["auto", "default"] as const; } diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index 764e571bfa..e94f9dca7c 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -288,7 +288,6 @@ export interface OpenAIChatCompletionsRequest { | "default" | "flex" | "priority" - | "scale" | "standard_only"; /** * Whether the provider may dispatch tool calls in parallel. diff --git a/studio/frontend/src/features/chat/types/runtime.ts b/studio/frontend/src/features/chat/types/runtime.ts index 49c7c291e7..8efd5bab2c 100644 --- a/studio/frontend/src/features/chat/types/runtime.ts +++ b/studio/frontend/src/features/chat/types/runtime.ts @@ -6,7 +6,6 @@ export type ServiceTier = | "default" | "flex" | "priority" - | "scale" | "standard_only"; export interface InferenceParams { From 093f4656200b690044234167c0e096dd9dfb81a2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 15:48:13 +0000 Subject: [PATCH 04/56] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/tests/test_sampling_params_routing.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index e5a16e03ad..e7da1684cd 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -109,18 +109,14 @@ def test_anthropic_stop_sequences_dedup_and_drop_empties(monkeypatch): """Whitespace-only / duplicate chips shouldn't reach the wire and waste budget against Anthropic's 16-entry cap.""" captured = _install_mock(monkeypatch) - body = _drive_anthropic( - captured, stop = ["END", "", "END", "DONE", " ", "END"] - ) + body = _drive_anthropic(captured, stop = ["END", "", "END", "DONE", " ", "END"]) # Order preserved on first sight, duplicates and empties dropped. assert body.get("stop_sequences") == ["END", "DONE", " "], body def test_anthropic_stop_sequences_truncated_to_16(monkeypatch): captured = _install_mock(monkeypatch) - body = _drive_anthropic( - captured, stop = [f"S{i}" for i in range(20)] - ) + body = _drive_anthropic(captured, stop = [f"S{i}" for i in range(20)]) assert len(body.get("stop_sequences", [])) == 16, body assert body["stop_sequences"][0] == "S0" assert body["stop_sequences"][-1] == "S15" @@ -242,9 +238,7 @@ def test_openai_compat_truncates_stop_to_four(monkeypatch): def test_openai_compat_stop_dedup_and_drop_empties(monkeypatch): """Duplicates and empties shouldn't eat into the 4-entry cap.""" captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) - body = _drive_openai_compat( - captured, stop = ["END", "", "END", "DONE", "FIN", "END"] - ) + body = _drive_openai_compat(captured, stop = ["END", "", "END", "DONE", "FIN", "END"]) assert body.get("stop") == ["END", "DONE", "FIN"], body From eefe40a6bf75b8a6a4ac94fc2335efba445ed126 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 22 May 2026 20:34:10 +0000 Subject: [PATCH 05/56] Studio: assert promoted fields on attribute path in test_extra_fields_accepted This PR promoted frequency_penalty and seed from undeclared chat-completion extras into explicit ChatCompletionRequest fields, so they ride the attribute path now, not model_extra. The test still asserted both via model_extra and failed on Linux Python 3.10-3.13 with 'assert None == 0.5'. response_format stays in model_extra (still undeclared) so the extra='allow' contract is covered by that branch. --- .../backend/tests/test_openai_tool_passthrough.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py index 84f3e41998..331605b998 100644 --- a/studio/backend/tests/test_openai_tool_passthrough.py +++ b/studio/backend/tests/test_openai_tool_passthrough.py @@ -275,18 +275,19 @@ class TestChatCompletionRequestToolFields: assert req.stop is None def test_extra_fields_accepted(self): - # `frequency_penalty`, `seed`, `response_format` are not yet - # explicitly declared but must survive Pydantic parsing now that - # extra="allow" is set. + # ``response_format`` is still an undeclared OpenAI-side field; + # it must survive Pydantic parsing because extra="allow" is set. + # ``frequency_penalty`` and ``seed`` were promoted to explicit + # ChatCompletionRequest fields in the sampling-params PR, so + # they now ride the attribute path, not model_extra. req = self._make( frequency_penalty = 0.5, seed = 42, response_format = {"type": "json_object"}, ) - # Extras land in model_extra + assert req.frequency_penalty == 0.5 + assert req.seed == 42 assert req.model_extra is not None - assert req.model_extra.get("frequency_penalty") == 0.5 - assert req.model_extra.get("seed") == 42 assert req.model_extra.get("response_format") == {"type": "json_object"} def test_unsloth_extensions_still_work(self): From ffda6bbc71037e6172aa364b9e56b0f112beff4b Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 23 May 2026 16:37:51 +0000 Subject: [PATCH 06/56] Studio: persist new sampling keys through settings sanitizer Codex P1: the runtime store added frequencyPenalty, seed, stop, serviceTier, parallelToolCalls but the save/load path went through sanitizeInferenceParams, which only whitelisted the older numeric set plus systemPrompt / trustRemoteCode. The new keys were silently stripped on save and dropped on reload. Extend the whitelist: - frequencyPenalty added to the numeric finite-number set. - seed: integer or explicit null (null = "no seed field on the wire"). - stop: string array, capped at 4 entries per OpenAI's limit. - serviceTier: nullable enum (auto/default/flex/priority/scale). - parallelToolCalls: boolean. --- .../chat/utils/chat-settings-storage.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/studio/frontend/src/features/chat/utils/chat-settings-storage.ts b/studio/frontend/src/features/chat/utils/chat-settings-storage.ts index e07e1ddb1d..f57f44f883 100644 --- a/studio/frontend/src/features/chat/utils/chat-settings-storage.ts +++ b/studio/frontend/src/features/chat/utils/chat-settings-storage.ts @@ -40,10 +40,21 @@ const NUMERIC_INFERENCE_FIELDS = [ "minP", "repetitionPenalty", "presencePenalty", + "frequencyPenalty", "maxSeqLength", "maxTokens", ] as const satisfies readonly (keyof PersistedInferenceParams)[]; +// `seed` is numeric but nullable (null = "no seed field on the wire") so +// it can't go through the NUMERIC_INFERENCE_FIELDS Finite-number filter. +const VALID_SERVICE_TIERS = new Set([ + "auto", + "default", + "flex", + "priority", + "scale", +]); + const CHAT_PRESET_SOURCES = new Set([ "builtin-default", "custom", @@ -140,6 +151,29 @@ function sanitizeInferenceParams( if (typeof value.trustRemoteCode === "boolean") { params.trustRemoteCode = value.trustRemoteCode; } + // seed: nullable integer (null = no seed on the wire). + if (value.seed === null) { + params.seed = null; + } else if (typeof value.seed === "number" && Number.isInteger(value.seed)) { + params.seed = value.seed; + } + // stop: capped string array per OpenAI's max-4 rule. + if (Array.isArray(value.stop)) { + const stops = value.stop.filter((s): s is string => typeof s === "string"); + if (stops.length > 0) params.stop = stops.slice(0, 4); + } + // serviceTier: nullable enum string. + if (value.serviceTier === null) { + params.serviceTier = null; + } else if ( + typeof value.serviceTier === "string" && + VALID_SERVICE_TIERS.has(value.serviceTier) + ) { + params.serviceTier = value.serviceTier as PersistedInferenceParams["serviceTier"]; + } + if (typeof value.parallelToolCalls === "boolean") { + params.parallelToolCalls = value.parallelToolCalls; + } return hasKeys(params) ? params : undefined; } From febadebefacd58e379f71b3bb9a741c78e51da89 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 23 May 2026 17:34:07 +0000 Subject: [PATCH 07/56] ci: re-trigger after transient actions/checkout git auth flake From 5316c29588d17c7b2c710cf6887b269460f47b84 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 23 May 2026 18:35:18 +0000 Subject: [PATCH 08/56] ci: re-trigger after transient GitHub API HTTP flake (checkout + ggml-org release fetch) From aa9d8e21807fc74e9f167ffc4e63011f56bd7382 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 23 May 2026 19:34:47 +0000 Subject: [PATCH 09/56] ci: re-trigger after transient infra flake on Windows prebuilt / actions/checkout From d6b4c36e0ad8177d2591b50b3fc83180045790e5 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 12:37:28 +0000 Subject: [PATCH 10/56] Studio: nest disable_parallel_tool_use, drop ws-only stops, fix persistence Anthropic Messages API rejects `disable_parallel_tool_use` as a top-level field; it is only accepted as a property on the `tool_choice` object. Move the inversion into a tool_choice merge that defaults to `{type:"auto"}` when no choice is supplied, and skip the field entirely when no tools are defined (it is a no-op without tools). The same path also dropped `stop` chips that contain only whitespace, because Anthropic 400s with `each stop sequence must contain non-whitespace` on entries like " ", "\n", and "\n\n". The previous filter only dropped truly empty strings; switch to `s.strip()` so the common newline-stop defaults are also filtered out client-side. Frontend persistence had three round-trip data-loss bugs: - `VALID_SERVICE_TIERS` was missing `standard_only`, so any Anthropic user who picked that tier lost it on the next reload. - The settings sanitizer truncated `stop` to 4 entries on save, which defeated the Anthropic UI cap of 16. Use 16 here and let the per-provider stream helper cap to the wire's allowed length. - The chat-settings sheet's `stopMaxEntries` capped local backends (llama.cpp / vLLM / ollama / generic OpenAI-compat) at 4 even though those backends happily accept more. Match Anthropic's 16 for the local path. Preset policy now carries `frequencyPenalty` and `stop` so a saved preset can fix a user's preferred decoding style. `seed`, `serviceTier`, and `parallelToolCalls` stay out of presets because they are per-request determinism / per-provider account / per-tool state, not reusable preset values. Drops the test that pinned the buggy top-level placement of `disable_parallel_tool_use` and adds two tests for the nested shape plus the without-tools skip path, plus a test pinning the whitespace-stop filter against the documented Anthropic error. --- .../core/inference/external_provider.py | 53 +++++++++---- .../tests/test_sampling_params_routing.py | 77 ++++++++++++++++--- .../src/features/chat/chat-settings-sheet.tsx | 10 ++- .../features/chat/presets/preset-policy.ts | 23 ++++++ .../chat/utils/chat-settings-storage.ts | 18 ++++- 5 files changed, 152 insertions(+), 29 deletions(-) diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index fa9eb0f13d..16e65e89cc 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -1407,25 +1407,39 @@ class ExternalProviderClient: # Optional sampling extensions. Anthropic has no # frequency_penalty / seed / logprobs equivalents, so those are # silently dropped by virtue of not being forwarded from - # stream_chat_completion. The three knobs Anthropic does accept - # land here: - # stop → stop_sequences (renamed) + # stream_chat_completion. The two body-level knobs Anthropic + # does accept land here: + # stop → stop_sequences (renamed, ws-stripped) # service_tier → service_tier (auto|standard_only only) - # parallel_tool_calls → disable_parallel_tool_use (inverted) + # parallel_tool_calls inversion is applied AFTER the tools + # wiring below, because Anthropic requires it nested under + # tool_choice (top-level placement is rejected with + # `extraneous key [disable_parallel_tool_use] is not permitted`). if stop: sequences: list[str] if isinstance(stop, str): - sequences = [stop] + sequences = [stop] if stop.strip() else [] else: - # Dedupe + drop empties first; Anthropic 400s on empty - # entries and the docs cap stop_sequences at 16 entries. + # Dedupe + drop whitespace-only entries. Anthropic 400s + # on any sequence that contains no non-whitespace char: + # `stop_sequences: each stop sequence must contain + # non-whitespace`. That rejects empty strings, " ", and + # — critically — common defaults like "\n" / "\n\n". + # The truncation cap below (16) is a client-side guard; + # the Anthropic Messages API does not publish a max + # array length but every SDK we have inspected treats + # 16 as a sane ceiling (Bedrock's hard cap is 8191, so + # this only matters when callers paste pathologically + # long lists by accident). sequences = list( - dict.fromkeys(s for s in stop if isinstance(s, str) and s) + dict.fromkeys( + s for s in stop if isinstance(s, str) and s.strip() + ) ) if len(sequences) > 16: logger.warning( "stop_sequences truncated to 16 entries " - "(received %d, Anthropic's hard cap is 16)", + "(received %d, client-side guard ceiling)", len(sequences), ) sequences = sequences[:16] @@ -1433,10 +1447,6 @@ class ExternalProviderClient: body["stop_sequences"] = sequences if service_tier in ("auto", "standard_only"): body["service_tier"] = service_tier - if parallel_tool_calls is False: - # Default upstream behavior is parallel-allowed; only - # forward when the user explicitly disabled it. - body["disable_parallel_tool_use"] = True # Anthropic only caches a prefix when at least one cache_control # marker is attached to it — the frontend defaults # enable_prompt_caching to True for Anthropic, so treat `None` the @@ -1667,6 +1677,23 @@ class ExternalProviderClient: if anthropic_code_exec_container_id: body["container"] = anthropic_code_exec_container_id + # parallel_tool_calls=false → disable_parallel_tool_use=true, + # nested under tool_choice (NOT top-level). The Anthropic + # Messages API only accepts `disable_parallel_tool_use` as a + # property on the `tool_choice` object (ToolChoiceAuto / + # ToolChoiceAny / ToolChoiceTool). Top-level placement is + # rejected with `extraneous key [disable_parallel_tool_use] + # is not permitted`. Without any tools the flag is also a + # no-op upstream — skip it to keep the request body minimal. + # See + # https://platform.claude.com/docs/en/agents-and-tools/tool-use/implement-tool-use + if parallel_tool_calls is False and body.get("tools"): + tc = body.get("tool_choice") + if not isinstance(tc, dict): + tc = {"type": "auto"} + tc["disable_parallel_tool_use"] = True + body["tool_choice"] = tc + # Server-side context compaction — see # https://platform.claude.com/docs/en/build-with-claude/compaction # Beta as of `compact-2026-01-12`. When `compaction_threshold` is diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index e7da1684cd..699d4d4b0e 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -105,13 +105,29 @@ def test_anthropic_empty_stop_omitted(monkeypatch): assert "stop" not in body, body -def test_anthropic_stop_sequences_dedup_and_drop_empties(monkeypatch): - """Whitespace-only / duplicate chips shouldn't reach the wire and - waste budget against Anthropic's 16-entry cap.""" +def test_anthropic_stop_sequences_dedup_and_drop_whitespace(monkeypatch): + """Anthropic 400s on any stop sequence that contains no non- + whitespace character (`stop_sequences: each stop sequence must + contain non-whitespace`). Empty strings, " ", "\\n", "\\n\\n", and + other whitespace-only chips are filtered out client-side so the + request reaches the wire. Duplicates are deduped to avoid wasting + slots against the cap. + """ captured = _install_mock(monkeypatch) - body = _drive_anthropic(captured, stop = ["END", "", "END", "DONE", " ", "END"]) - # Order preserved on first sight, duplicates and empties dropped. - assert body.get("stop_sequences") == ["END", "DONE", " "], body + body = _drive_anthropic( + captured, + stop = ["END", "", "END", "DONE", " ", "END", "\n\n", "\t"], + ) + # Order preserved on first sight, duplicates + every whitespace-only + # entry dropped. + assert body.get("stop_sequences") == ["END", "DONE"], body + + +def test_anthropic_single_whitespace_stop_string_dropped(monkeypatch): + """Single-string stop="\\n\\n" must not reach the wire either.""" + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, stop = "\n\n") + assert "stop_sequences" not in body, body def test_anthropic_stop_sequences_truncated_to_16(monkeypatch): @@ -137,21 +153,58 @@ def test_anthropic_service_tier_unsupported_values_dropped(monkeypatch, bogus): assert "service_tier" not in body, body -def test_anthropic_disable_parallel_tool_use_only_when_false(monkeypatch): +def _drive_anthropic_with_tools(captured, **kwargs) -> dict: + """Same as `_drive_anthropic` but enables a server-side tool + (`web_search`) so the request body carries `tools`. Needed to + exercise the `disable_parallel_tool_use` nesting path, which only + fires when there is at least one tool defined. + """ + enabled_tools = kwargs.pop("enabled_tools", None) or ["web_search"] + return _drive_anthropic(captured, enabled_tools = enabled_tools, **kwargs) + + +def test_anthropic_disable_parallel_tool_use_nested_under_tool_choice(monkeypatch): + """`disable_parallel_tool_use` must be a property of `tool_choice`, + NOT a top-level body field. Top-level placement is rejected with + `extraneous key [disable_parallel_tool_use] is not permitted`. See + https://platform.claude.com/docs/en/agents-and-tools/tool-use/implement-tool-use. + """ + captured = _install_mock(monkeypatch) + body = _drive_anthropic_with_tools(captured, parallel_tool_calls = False) + # Top-level fields must not carry the flag — Anthropic 400s otherwise. + assert "disable_parallel_tool_use" not in body, body + assert "parallel_tool_calls" not in body, body + # The flag is set on tool_choice. Default type is "auto" when the + # user didn't pick one explicitly. + tc = body.get("tool_choice") + assert isinstance(tc, dict), body + assert tc.get("disable_parallel_tool_use") is True, body + assert tc.get("type") == "auto", body + + +def test_anthropic_disable_parallel_tool_use_skipped_without_tools(monkeypatch): + """Without any tools defined, `disable_parallel_tool_use` is a + no-op upstream — skip it so the request body stays minimal and the + flag never lands at top level either. + """ captured = _install_mock(monkeypatch) body = _drive_anthropic(captured, parallel_tool_calls = False) - assert body.get("disable_parallel_tool_use") is True, body - # Anthropic has no `parallel_tool_calls` field. + assert "disable_parallel_tool_use" not in body, body assert "parallel_tool_calls" not in body, body + assert "tool_choice" not in body, body def test_anthropic_parallel_tool_calls_default_not_sent(monkeypatch): captured = _install_mock(monkeypatch) - body = _drive_anthropic(captured, parallel_tool_calls = True) - # True is the upstream default; do not surface - # `disable_parallel_tool_use: false` which would over-specify the request. + body = _drive_anthropic_with_tools(captured, parallel_tool_calls = True) + # True is the upstream default; do not surface a tool_choice we + # would otherwise not have set, and definitely no top-level + # `disable_parallel_tool_use`. assert "disable_parallel_tool_use" not in body, body assert "parallel_tool_calls" not in body, body + tc = body.get("tool_choice") + if isinstance(tc, dict): + assert "disable_parallel_tool_use" not in tc, body def test_anthropic_rejects_openai_only_knobs(monkeypatch): diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index b568916c05..d02369b4ef 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -429,8 +429,14 @@ export function ChatSettingsPanel({ !isExternalModel || Boolean(providerCapabilities?.parallelToolCalls); // OpenAI Chat docs cap `stop` at 4 entries; Anthropic accepts more. // Pick the right ceiling per active connection so the chips editor's - // placeholder doesn't lie. - const stopMaxEntries = externalProviderType === "anthropic" ? 16 : 4; + // placeholder doesn't lie. Local backends (llama.cpp / vLLM / ollama + // / generic OpenAI-compat connections) accept many more — match the + // Anthropic cap there so we do not block users from using stop + // sequences the backend would happily accept. The wire-side + // truncation in `_stream_openai_compat` will still trim to OpenAI's + // 4-entry hard cap when a cloud OpenAI endpoint receives the body. + const stopMaxEntries = + !isExternalModel || externalProviderType === "anthropic" ? 16 : 4; const serviceTierOptions = getServiceTierOptions(externalProviderType); const isMobile = useIsMobile(); const isGguf = useChatRuntimeStore((s) => s.activeGgufVariant) != null; diff --git a/studio/frontend/src/features/chat/presets/preset-policy.ts b/studio/frontend/src/features/chat/presets/preset-policy.ts index ae8c1f41ae..7cd5c76048 100644 --- a/studio/frontend/src/features/chat/presets/preset-policy.ts +++ b/studio/frontend/src/features/chat/presets/preset-policy.ts @@ -13,6 +13,14 @@ export interface Preset { params: InferenceParams; } +// Fields that belong to a preset. Sampling knobs are included so a +// user can save a preset that fixes their preferred decoding style and +// re-apply it on any model. Operational knobs (`serviceTier`, which is +// account-level and per-provider, and `parallelToolCalls`, which is +// tool-level state) are intentionally excluded so switching presets +// does not silently change request routing for the active provider. +// `seed` is also excluded — it is per-request determinism state, not a +// reusable preset value. export type PresetOwnedParams = Pick< InferenceParams, | "temperature" @@ -21,6 +29,8 @@ export type PresetOwnedParams = Pick< | "minP" | "repetitionPenalty" | "presencePenalty" + | "frequencyPenalty" + | "stop" | "maxTokens" | "systemPrompt" >; @@ -103,11 +113,22 @@ export function getPresetOwnedParams( minP: params.minP, repetitionPenalty: params.repetitionPenalty, presencePenalty: params.presencePenalty, + frequencyPenalty: params.frequencyPenalty, + stop: params.stop, maxTokens: params.maxTokens, systemPrompt: params.systemPrompt, }; } +function stopArraysEqual(a: string[], b: string[]): boolean { + if (a === b) return true; + if (a.length !== b.length) return false; + for (let i = 0; i < a.length; i += 1) { + if (a[i] !== b[i]) return false; + } + return true; +} + export function isSamePresetConfig( a: InferenceParams, b: InferenceParams, @@ -121,6 +142,8 @@ export function isSamePresetConfig( left.minP === right.minP && left.repetitionPenalty === right.repetitionPenalty && left.presencePenalty === right.presencePenalty && + left.frequencyPenalty === right.frequencyPenalty && + stopArraysEqual(left.stop, right.stop) && left.maxTokens === right.maxTokens && left.systemPrompt === right.systemPrompt ); diff --git a/studio/frontend/src/features/chat/utils/chat-settings-storage.ts b/studio/frontend/src/features/chat/utils/chat-settings-storage.ts index f57f44f883..7446aa961b 100644 --- a/studio/frontend/src/features/chat/utils/chat-settings-storage.ts +++ b/studio/frontend/src/features/chat/utils/chat-settings-storage.ts @@ -47,12 +47,20 @@ const NUMERIC_INFERENCE_FIELDS = [ // `seed` is numeric but nullable (null = "no seed field on the wire") so // it can't go through the NUMERIC_INFERENCE_FIELDS Finite-number filter. +// Keep this set in sync with `ServiceTier` in ../types/runtime.ts and with +// `getServiceTierOptions` in ../provider-capabilities.ts. `standard_only` +// is Anthropic-only and was missing here — dropping it on save erased the +// user's Anthropic tier choice on reload. `scale` was removed from the UI +// (Codex review feedback) but accepting it on load is harmless for stale +// persisted state, so it stays in the allowlist; the resolver no longer +// surfaces it. const VALID_SERVICE_TIERS = new Set([ "auto", "default", "flex", "priority", "scale", + "standard_only", ]); const CHAT_PRESET_SOURCES = new Set([ @@ -157,10 +165,16 @@ function sanitizeInferenceParams( } else if (typeof value.seed === "number" && Number.isInteger(value.seed)) { params.seed = value.seed; } - // stop: capped string array per OpenAI's max-4 rule. + // stop: capped string array. Use Anthropic's 16-entry max so the + // sanitizer never silently discards entries the user actually typed + // for an Anthropic session. The backend's per-provider stream helper + // re-truncates to the wire cap (4 for OpenAI Chat, 16 for Anthropic, + // dropped entirely for OpenAI Responses) before the request hits the + // network. Capping to 4 here would defeat Anthropic's UI cap of 16 + // for users who switch providers between sessions. if (Array.isArray(value.stop)) { const stops = value.stop.filter((s): s is string => typeof s === "string"); - if (stops.length > 0) params.stop = stops.slice(0, 4); + if (stops.length > 0) params.stop = stops.slice(0, 16); } // serviceTier: nullable enum string. if (value.serviceTier === null) { From 3ef64c2d6525e7588c6b043912de4e3bd433ba7f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 24 May 2026 12:37:49 +0000 Subject: [PATCH 11/56] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/core/inference/external_provider.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index 16e65e89cc..c78c60e6ad 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -1432,9 +1432,7 @@ class ExternalProviderClient: # this only matters when callers paste pathologically # long lists by accident). sequences = list( - dict.fromkeys( - s for s in stop if isinstance(s, str) and s.strip() - ) + dict.fromkeys(s for s in stop if isinstance(s, str) and s.strip()) ) if len(sequences) > 16: logger.warning( From d6765fddcee211380e9e8b3b48a2e8243265355a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 12:44:38 +0000 Subject: [PATCH 12/56] Studio: thread sampling extensions through local + Kimi-search paths Round-2 round of review-feedback fixes for the sampling-knobs PR: - studio/backend/routes/chat_history.py: ChatInferenceSettings still had the pre-PR field list with extra="forbid", so every settings save the new frontend issued would 422 on the new keys (frequencyPenalty, seed, stop, serviceTier, parallelToolCalls). Add the fields with the same range / enum constraints the chat-completions schema uses, so the settings-persistence path round-trips cleanly. - studio/backend/routes/inference.py: _build_passthrough_payload and _build_openai_passthrough_body now thread frequency_penalty, seed, and parallel_tool_calls through to llama-server. The frontend exposes these knobs for local backends; without the forwarding the UI was a decoration. Each field is gated on `is not None` so 0 / False / "0" still reach the body. - studio/backend/core/inference/external_provider.py: the Kimi $web_search bypass takes an early return into _stream_kimi_web_search before the default OAI-compat body builder runs, so the new sampling fields never landed on Kimi-with-search. Forward them through the helper, with the same dedupe / truncate behavior the main path applies to `stop`. Also extend the OpenAI Responses service_tier allowlist to include `scale` per the live openai-python SDK (response_create_params.py declares Literal["auto","default","flex","scale","priority"]). - studio/frontend/src/features/chat/provider-capabilities.ts + types/runtime.ts: add `scale` to ServiceTier / ServiceTierOption and surface it on the OpenAI Responses options so the UI matches the upstream enum. - studio/backend/tests/test_sampling_params_routing.py: add tests for every gap above: Kimi web-search bypass forwarding, local OpenAI passthrough forwarding, ChatSettingsPayload round-trip, and the full Responses service_tier enum (parametrized over the five accepted values plus a drop check for the Anthropic-only standard_only). --- .../core/inference/external_provider.py | 60 ++++++++- studio/backend/routes/chat_history.py | 12 ++ studio/backend/routes/inference.py | 18 +++ .../tests/test_sampling_params_routing.py | 124 +++++++++++++++++- .../features/chat/provider-capabilities.ts | 16 ++- .../src/features/chat/types/runtime.ts | 1 + 6 files changed, 214 insertions(+), 17 deletions(-) diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index c78c60e6ad..002e2548c6 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -435,6 +435,11 @@ class ExternalProviderClient: messages, model, max_tokens, + frequency_penalty = frequency_penalty, + seed = seed, + stop = stop, + parallel_tool_calls = parallel_tool_calls, + presence_penalty = presence_penalty, ): yield line return @@ -850,6 +855,12 @@ class ExternalProviderClient: messages: list[dict[str, Any]], model: str, max_tokens: Optional[int], + *, + frequency_penalty: Optional[float] = None, + seed: Optional[int] = None, + stop: Optional[Union[str, list[str]]] = None, + parallel_tool_calls: Optional[bool] = None, + presence_penalty: Optional[float] = None, ) -> AsyncGenerator[str, None]: """ Kimi $web_search round-trip. @@ -889,6 +900,37 @@ class ExternalProviderClient: if max_tokens is not None: body["max_tokens"] = max_tokens + # Forward the new optional sampling extensions (#5711) on the + # web-search bypass too. The default OAI-compat body construction + # (which adds these) is skipped because this helper returns + # early; forwarding here ensures kimi-with-search honours the + # same sampling controls as kimi-without-search. + if presence_penalty is not None: + body["presence_penalty"] = presence_penalty + if frequency_penalty is not None: + body["frequency_penalty"] = frequency_penalty + if seed is not None: + body["seed"] = seed + if stop: + if isinstance(stop, str): + if stop.strip(): + body["stop"] = stop + elif isinstance(stop, list): + sequences = list( + dict.fromkeys(s for s in stop if isinstance(s, str) and s) + ) + if len(sequences) > 4: + logger.warning( + "stop sequences truncated to 4 entries " + "(received %d, OpenAI's hard cap is 4)", + len(sequences), + ) + body["stop"] = sequences[:4] + elif sequences: + body["stop"] = sequences + if parallel_tool_calls is not None: + body["parallel_tool_calls"] = parallel_tool_calls + # Strip body fields the Kimi registry declares unusable # (temperature/top_p — see body_omit in providers.py). from core.inference.providers import get_provider_info @@ -2783,12 +2825,18 @@ class ExternalProviderClient: "input": input_items, "stream": True, } - # Responses accepts service_tier on the same enum set as Chat - # Completions minus `scale`. parallel_tool_calls follows the - # same shape (default true). The frontend capability gate - # (provider-capabilities.ts) already filters the option lists - # per provider, so we just forward what we got. - if service_tier in ("auto", "default", "flex", "priority"): + # Responses accepts the same service_tier enum set as Chat + # Completions (auto|default|flex|scale|priority) per the live + # `openai-python` SDK + # (`src/openai/types/responses/response_create_params.py` + # declares `Optional[Literal["auto", "default", "flex", + # "scale", "priority"]]`). parallel_tool_calls follows the same + # shape (default true). The frontend capability gate + # (provider-capabilities.ts) already filters per-provider, so + # we just forward whatever value the dispatcher hands us, with + # `standard_only` (Anthropic-only) being the one value Responses + # has never accepted. + if service_tier in ("auto", "default", "flex", "scale", "priority"): body["service_tier"] = service_tier if parallel_tool_calls is not None: body["parallel_tool_calls"] = bool(parallel_tool_calls) diff --git a/studio/backend/routes/chat_history.py b/studio/backend/routes/chat_history.py index ed808040d2..6f7efb6eb5 100644 --- a/studio/backend/routes/chat_history.py +++ b/studio/backend/routes/chat_history.py @@ -107,6 +107,18 @@ class ChatInferenceSettings(BaseModel): minP: Optional[float] = None repetitionPenalty: Optional[float] = None presencePenalty: Optional[float] = None + # New per-provider sampling knobs. `extra="forbid"` would 422 any + # settings save from a frontend on the new code if these were not + # listed here, breaking the entire chat-settings persistence path. + # Keep these aligned with `InferenceParams` in + # studio/frontend/src/features/chat/types/runtime.ts. + frequencyPenalty: Optional[float] = Field(default = None, ge = -2.0, le = 2.0) + seed: Optional[int] = None + stop: Optional[list[str]] = None + serviceTier: Optional[ + Literal["auto", "default", "flex", "priority", "scale", "standard_only"] + ] = None + parallelToolCalls: Optional[bool] = None maxSeqLength: Optional[float] = None maxTokens: Optional[float] = None systemPrompt: Optional[str] = None diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index b6e4c4e370..1ee20c6459 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -4898,6 +4898,9 @@ def _build_passthrough_payload( min_p = None, repetition_penalty = None, presence_penalty = None, + frequency_penalty = None, + seed = None, + parallel_tool_calls = None, tool_choice = "auto", response_format = None, chat_template_kwargs = None, @@ -4929,6 +4932,18 @@ def _build_passthrough_payload( body["repeat_penalty"] = repetition_penalty if presence_penalty is not None: body["presence_penalty"] = presence_penalty + # New per-provider sampling extensions (PR #5711). llama-server's + # /v1/chat/completions endpoint accepts the standard OpenAI fields, + # so forward them straight through. parallel_tool_calls is a no-op + # on llama-server today (the upstream always dispatches sequentially) + # but forward it anyway so a future llama-server release that + # implements it picks up the user's preference automatically. + if frequency_penalty is not None: + body["frequency_penalty"] = frequency_penalty + if seed is not None: + body["seed"] = seed + if parallel_tool_calls is not None: + body["parallel_tool_calls"] = parallel_tool_calls if response_format is not None: # llama-server applies a GBNF grammar derived from the JSON schema # when response_format is present. Field is documented flat at the @@ -5347,6 +5362,9 @@ def _build_openai_passthrough_body(payload, backend_ctx = None) -> dict: min_p = payload.min_p, repetition_penalty = payload.repetition_penalty, presence_penalty = payload.presence_penalty, + frequency_penalty = payload.frequency_penalty, + seed = payload.seed, + parallel_tool_calls = payload.parallel_tool_calls, tool_choice = tool_choice, response_format = _extract_response_format(payload), chat_template_kwargs = tpl_kwargs, diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index 699d4d4b0e..27f430f8d3 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -385,11 +385,24 @@ def test_openai_responses_forwards_service_tier(monkeypatch): assert body.get("service_tier") == "priority", body -def test_openai_responses_rejects_chat_only_service_tier(monkeypatch): +@pytest.mark.parametrize( + "value", ["auto", "default", "flex", "scale", "priority"] +) +def test_openai_responses_accepts_full_service_tier_enum(monkeypatch, value): + """`openai-python`'s ResponseCreateParams declares + `Optional[Literal["auto", "default", "flex", "scale", "priority"]]` + so every value in that set forwards untouched.""" captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) - body = _drive_openai_responses(captured, service_tier = "scale") - # Responses only accepts auto|default|flex|priority -- `scale` is - # silently dropped so a stale frontend cannot 400 the request. + body = _drive_openai_responses(captured, service_tier = value) + assert body.get("service_tier") == value, body + + +def test_openai_responses_drops_anthropic_only_service_tier(monkeypatch): + """`standard_only` is Anthropic-only and Responses has never accepted + it. Drop it client-side so a stale frontend cannot 400 the request. + """ + captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) + body = _drive_openai_responses(captured, service_tier = "standard_only") assert "service_tier" not in body, body @@ -460,3 +473,106 @@ def test_chat_completion_request_clamps_frequency_penalty_range(): "frequency_penalty": -3.0, } ) + + +# ── Kimi web-search bypass forwards new sampling fields ──────────────── + + +def test_kimi_web_search_bypass_forwards_new_sampling_fields(monkeypatch): + """The Kimi `enabled_tools=["web_search"]` path takes an early + return into `_stream_kimi_web_search` BEFORE the default OAI-compat + body builder runs. PR #5711 added new sampling fields to the + default builder; this test pins that the web-search bypass also + forwards them so Kimi-with-search and Kimi-without-search behave + consistently.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "kimi", + base_url = "https://api.moonshot.ai/v1", + api_key = "kimi-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "kimi-k2.6", + temperature = 1.0, + top_p = 1.0, + max_tokens = 256, + enabled_tools = ["web_search"], + presence_penalty = 0.5, + frequency_penalty = 1.25, + seed = 7, + stop = ["END"], + parallel_tool_calls = False, + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert body.get("frequency_penalty") == 1.25, body + assert body.get("seed") == 7, body + assert body.get("stop") == ["END"], body + assert body.get("parallel_tool_calls") is False, body + assert body.get("presence_penalty") == 0.5, body + # body_omit still strips temperature / top_p for Kimi. + assert "temperature" not in body, body + assert "top_p" not in body, body + + +# ── Local OpenAI passthrough forwards new sampling fields ────────────── + + +def test_local_openai_passthrough_forwards_new_sampling_fields(): + """Round 1 reviewers (10/20) flagged that + `_build_openai_passthrough_body` dropped frequency_penalty / seed / + parallel_tool_calls when forwarding to llama-server. Pin the + extended contract.""" + from models.inference import ChatCompletionRequest + from routes.inference import _build_openai_passthrough_body + + payload = ChatCompletionRequest.model_validate( + { + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + "frequency_penalty": 1.25, + "seed": 123, + "stop": ["END"], + "parallel_tool_calls": False, + } + ) + body = _build_openai_passthrough_body(payload, backend_ctx = 4096) + assert body["frequency_penalty"] == 1.25, body + assert body["seed"] == 123, body + assert body["stop"] == ["END"], body + assert body["parallel_tool_calls"] is False, body + + +# ── Backend ChatInferenceSettings schema accepts new fields ──────────── + + +def test_chat_settings_payload_accepts_new_sampling_keys(): + """Round 1 reviewers flagged that `ChatSettingsPayload.extra="forbid"` + with the old field list 422'd every settings save that contained + any of the new keys. Pin that the new keys round-trip.""" + from routes.chat_history import ChatSettingsPayload + + parsed = ChatSettingsPayload.model_validate( + { + "inferenceParams": { + "frequencyPenalty": 0.7, + "seed": 42, + "stop": ["END"], + "serviceTier": "standard_only", + "parallelToolCalls": False, + } + } + ) + ip = parsed.inferenceParams + assert ip is not None + assert ip.frequencyPenalty == 0.7 + assert ip.seed == 42 + assert ip.stop == ["END"] + assert ip.serviceTier == "standard_only" + assert ip.parallelToolCalls is False diff --git a/studio/frontend/src/features/chat/provider-capabilities.ts b/studio/frontend/src/features/chat/provider-capabilities.ts index 0c63fed5bc..fb1c2ea0f9 100644 --- a/studio/frontend/src/features/chat/provider-capabilities.ts +++ b/studio/frontend/src/features/chat/provider-capabilities.ts @@ -65,17 +65,19 @@ export type ServiceTierOption = | "default" | "flex" | "priority" + | "scale" | "standard_only"; /** * Legal `service_tier` values per provider, sourced from each upstream's * docs. Anthropic exposes only `auto` and `standard_only`. OpenAI in - * Studio is routed through `/v1/responses` (not Chat Completions), and - * the Responses endpoint only accepts `auto` / `default` / `flex` / - * `priority` — `scale` is Chat-only and would be silently dropped here, - * so the option list omits it to avoid misleading the user. Other - * providers fall through to a permissive `auto` / `default` pair so the - * picker stays usable for OpenAI-compat backends. + * Studio is routed through `/v1/responses` (not Chat Completions); the + * live `openai-python` SDK declares the Responses-side service_tier as + * `Literal["auto", "default", "flex", "scale", "priority"]` + * (`src/openai/types/responses/response_create_params.py`), so the + * full set is exposed. Other providers fall through to a permissive + * `auto` / `default` pair so the picker stays usable for + * OpenAI-compat backends. */ export function getServiceTierOptions( providerType: string | null | undefined, @@ -84,7 +86,7 @@ export function getServiceTierOptions( return ["auto", "standard_only"] as const; } if (providerType === "openai") { - return ["auto", "default", "flex", "priority"] as const; + return ["auto", "default", "flex", "scale", "priority"] as const; } return ["auto", "default"] as const; } diff --git a/studio/frontend/src/features/chat/types/runtime.ts b/studio/frontend/src/features/chat/types/runtime.ts index 8efd5bab2c..49c7c291e7 100644 --- a/studio/frontend/src/features/chat/types/runtime.ts +++ b/studio/frontend/src/features/chat/types/runtime.ts @@ -6,6 +6,7 @@ export type ServiceTier = | "default" | "flex" | "priority" + | "scale" | "standard_only"; export interface InferenceParams { From fdf0be484ea765ad7b9820c29c303b6e73b70f47 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 24 May 2026 12:45:33 +0000 Subject: [PATCH 13/56] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/tests/test_sampling_params_routing.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index 27f430f8d3..b0d0c5c3b4 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -385,9 +385,7 @@ def test_openai_responses_forwards_service_tier(monkeypatch): assert body.get("service_tier") == "priority", body -@pytest.mark.parametrize( - "value", ["auto", "default", "flex", "scale", "priority"] -) +@pytest.mark.parametrize("value", ["auto", "default", "flex", "scale", "priority"]) def test_openai_responses_accepts_full_service_tier_enum(monkeypatch, value): """`openai-python`'s ResponseCreateParams declares `Optional[Literal["auto", "default", "flex", "scale", "priority"]]` From d8a4627355eb8b108e842534296b48c57c59a837 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 13:44:38 +0000 Subject: [PATCH 14/56] Studio: widen scale type, preserve significant ws in stops, Kimi parity Round 2 reviewer feedback: - studio/frontend/src/features/chat/types/api.ts: `OpenAIChatCompletionsRequest.service_tier` did not include `"scale"`, so the request builder in chat-adapter.ts failed typecheck after the runtime ServiceTier union widened (`Type 'ServiceTier | undefined' is not assignable...`). Widen the type to match the SDK and keep the typecheck green. - studio/frontend/src/components/ui/stop-sequences-input.tsx: the chip editor used `draft.trim()` for storage, which silently mutated semantically meaningful stops like " End", "### ", and "\n\n". Keep the whitespace-only rejection (Anthropic 400s on those, OpenAI silently drops them) but persist the raw draft so leading/trailing whitespace inside otherwise-meaningful stops survives. - studio/backend/core/inference/external_provider.py: the Kimi web-search bypass dropped a single string `stop="\n\n"` via `stop.strip()` while the normal default OAI-compat path forwards it verbatim. Mirror the default path's behavior here so kimi-with-search and kimi-without-search apply the same rules (asymmetric provider-path fix flagged in round-2 review). --- studio/backend/core/inference/external_provider.py | 10 ++++++++-- .../src/components/ui/stop-sequences-input.tsx | 14 ++++++++++---- studio/frontend/src/features/chat/types/api.ts | 12 ++++++++++-- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index 002e2548c6..d55f132487 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -912,9 +912,15 @@ class ExternalProviderClient: if seed is not None: body["seed"] = seed if stop: + # Mirror the default OAI-compat path's stop handling exactly + # so Kimi-with-search and Kimi-without-search apply the + # same rules — a single string is forwarded verbatim and + # lists are deduped + truncated to OpenAI's 4-entry cap. + # Earlier the bypass dropped whitespace-only strings here + # while the normal path forwarded them, which was an + # asymmetric provider-path fix. if isinstance(stop, str): - if stop.strip(): - body["stop"] = stop + body["stop"] = stop elif isinstance(stop, list): sequences = list( dict.fromkeys(s for s in stop if isinstance(s, str) and s) diff --git a/studio/frontend/src/components/ui/stop-sequences-input.tsx b/studio/frontend/src/components/ui/stop-sequences-input.tsx index e3c2d8a561..5d65fdd36a 100644 --- a/studio/frontend/src/components/ui/stop-sequences-input.tsx +++ b/studio/frontend/src/components/ui/stop-sequences-input.tsx @@ -38,14 +38,20 @@ export function StopSequencesInput({ const atCap = value.length >= maxEntries; function commitDraft() { - const trimmed = draft.trim(); - if (!trimmed) return; + // Reject chips that are empty or contain ONLY whitespace + // (Anthropic 400s on those and OpenAI silently drops them), but + // preserve significant leading/trailing whitespace inside otherwise + // -meaningful stops like " END", "### ", or "\n\n" — stop matching + // is exact, so stripping would silently change the semantics. The + // backend re-validates per-provider before the request hits the + // wire. + if (!draft || !draft.trim()) return; if (atCap) return; - if (value.includes(trimmed)) { + if (value.includes(draft)) { setDraft(""); return; } - onChange([...value, trimmed]); + onChange([...value, draft]); setDraft(""); } diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index e94f9dca7c..7c5b5bf99f 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -280,14 +280,22 @@ export interface OpenAIChatCompletionsRequest { stop?: string[]; /** * Provider service tier. Anthropic accepts `auto|standard_only`; - * OpenAI Chat accepts `auto|default|flex|priority|scale`; OpenAI - * Responses accepts `auto|default|flex|priority`. + * OpenAI Chat + Responses both accept + * `auto|default|flex|scale|priority` per the live `openai-python` + * SDK (`src/openai/types/responses/response_create_params.py` + * declares `Optional[Literal["auto", "default", "flex", "scale", + * "priority"]]`). The wire-side helper in + * `studio/backend/core/inference/external_provider.py` drops values + * that a given provider does not accept; this union stays permissive + * so the request-builder typechecks against + * `InferenceParams.serviceTier` without per-provider narrowing. */ service_tier?: | "auto" | "default" | "flex" | "priority" + | "scale" | "standard_only"; /** * Whether the provider may dispatch tool calls in parallel. From b8cef29b50d7ce06362dac074bf384f49f9f2a27 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 13:57:46 +0000 Subject: [PATCH 15/56] Studio: forward parallel_tool_calls through /v1/responses bridge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 3 reviewer feedback: - studio/backend/routes/inference.py: _build_chat_request (the /v1/responses → /v1/chat/completions translator) was dropping parallel_tool_calls on the floor. A Responses-API caller that set `parallel_tool_calls=false` saw the flag accepted at the schema layer but never reach llama-server because the translated ChatCompletionRequest had no first-class field for it. Now that parallel_tool_calls IS a first-class field on ChatCompletionRequest (added by this PR's earlier commits), translate it through the bridge so the preference actually fires. - studio/frontend/src/features/chat/utils/chat-settings-storage.ts: the stop sanitizer silently dropped `stop: []` instead of persisting the empty array. That meant a user could not clear the last chip — on reload, the previously-persisted stops came back. Persist empty arrays explicitly so the cleared state round-trips. - studio/backend/tests/test_sampling_params_routing.py: pin both with the raw reproductions reviewers cited. --- studio/backend/routes/inference.py | 16 ++++--- .../tests/test_sampling_params_routing.py | 45 +++++++++++++++++++ .../chat/utils/chat-settings-storage.ts | 7 ++- 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 1ee20c6459..8c7267c0e2 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -3780,6 +3780,14 @@ def _build_chat_request( chat_kwargs["top_p"] = payload.top_p if payload.max_output_tokens is not None: chat_kwargs["max_tokens"] = payload.max_output_tokens + # `parallel_tool_calls` is now a first-class field on + # ChatCompletionRequest (PR #5711) and the OpenAI-compat + # passthrough builder forwards it. Translate it here so a Responses + # API caller (e.g. OpenAI Codex SDK) that sets + # `parallel_tool_calls=false` actually sees the preference reach + # llama-server instead of getting silently dropped at the bridge. + if payload.parallel_tool_calls is not None: + chat_kwargs["parallel_tool_calls"] = payload.parallel_tool_calls chat_tools = _translate_responses_tools_to_chat(payload.tools) if chat_tools is not None: @@ -3789,13 +3797,7 @@ def _build_chat_request( if chat_tool_choice is not None: chat_kwargs["tool_choice"] = chat_tool_choice - req = ChatCompletionRequest(**chat_kwargs) - # `parallel_tool_calls` is not a first-class field on ChatCompletionRequest, - # but the model allows extras and _build_openai_passthrough_body forwards - # only explicitly-known fields. Llama-server does not currently implement - # parallel_tool_calls semantics, so we accept-and-ignore it on the - # Responses side to avoid breaking SDK clients that always send it. - return req + return ChatCompletionRequest(**chat_kwargs) def _chat_tool_calls_to_responses_output(tool_calls: list[dict]) -> list[dict]: diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index b0d0c5c3b4..c27d782c62 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -547,6 +547,51 @@ def test_local_openai_passthrough_forwards_new_sampling_fields(): assert body["parallel_tool_calls"] is False, body +# ── Responses → ChatCompletions bridge preserves parallel_tool_calls ── + + +def test_responses_to_chat_bridge_preserves_parallel_tool_calls(): + """Round 3 reviewers flagged that `_build_chat_request` (the + /v1/responses → /v1/chat/completions translator) dropped + `parallel_tool_calls`, so a Responses-API caller that set + `parallel_tool_calls=false` never saw the flag reach llama-server. + Pin the translation.""" + from models.inference import ChatMessage, ResponsesRequest + from routes.inference import _build_chat_request, _build_openai_passthrough_body + + payload = ResponsesRequest( + input = "hi", + stream = True, + parallel_tool_calls = False, + ) + chat_req = _build_chat_request( + payload, + [ChatMessage(role = "user", content = "hi")], + stream = True, + ) + assert chat_req.parallel_tool_calls is False, chat_req + body = _build_openai_passthrough_body(chat_req, backend_ctx = 4096) + assert body["parallel_tool_calls"] is False, body + + +def test_responses_to_chat_bridge_omits_unset_parallel_tool_calls(): + """Unset `parallel_tool_calls` (None) must not appear on the + translated body — the upstream default is `true` everywhere, so + forwarding `parallel_tool_calls=None` would over-specify.""" + from models.inference import ChatMessage, ResponsesRequest + from routes.inference import _build_chat_request, _build_openai_passthrough_body + + payload = ResponsesRequest(input = "hi", stream = True) + chat_req = _build_chat_request( + payload, + [ChatMessage(role = "user", content = "hi")], + stream = True, + ) + assert chat_req.parallel_tool_calls is None, chat_req + body = _build_openai_passthrough_body(chat_req, backend_ctx = 4096) + assert "parallel_tool_calls" not in body, body + + # ── Backend ChatInferenceSettings schema accepts new fields ──────────── diff --git a/studio/frontend/src/features/chat/utils/chat-settings-storage.ts b/studio/frontend/src/features/chat/utils/chat-settings-storage.ts index 7446aa961b..e529b792d1 100644 --- a/studio/frontend/src/features/chat/utils/chat-settings-storage.ts +++ b/studio/frontend/src/features/chat/utils/chat-settings-storage.ts @@ -171,10 +171,13 @@ function sanitizeInferenceParams( // re-truncates to the wire cap (4 for OpenAI Chat, 16 for Anthropic, // dropped entirely for OpenAI Responses) before the request hits the // network. Capping to 4 here would defeat Anthropic's UI cap of 16 - // for users who switch providers between sessions. + // for users who switch providers between sessions. An EMPTY array + // must persist as an empty array (not be sanitized away) so the user + // can clear the last chip and have the change saved — otherwise the + // previously stored stops come back on reload. if (Array.isArray(value.stop)) { const stops = value.stop.filter((s): s is string => typeof s === "string"); - if (stops.length > 0) params.stop = stops.slice(0, 16); + params.stop = stops.slice(0, 16); } // serviceTier: nullable enum string. if (value.serviceTier === null) { From 30d6ce201e5a6f75991dfb44b8f60765350209ee Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 14:12:13 +0000 Subject: [PATCH 16/56] Studio: drop service_tier=scale on OpenAI Responses path Round 4 reviewer consensus (~9/20 independent reviewers) flagged service_tier=scale as a 400 risk on /v1/responses. The earlier commit added scale based on the openai-python SDK literal, but the live OpenAI Responses API reference, the PR's own provider matrix, and the 9-reviewer round-4 consensus all agree the documented Responses enum is auto|default|flex|priority only. Drop scale on this path to remove the risk. Keeps scale on the Chat Completions / OAI-compat path where the SDK enum is honored and where users who want Scale Tier can still select it. The widened TypeScript ServiceTier / ServiceTierOption / api.ts union and the storage sanitizer allowlist remain permissive so legacy persisted "scale" values do not get silently dropped on reload; the runtime per-provider gate makes the routing decision. Tests are updated to pin the restricted Responses enum and the explicit drop of scale + standard_only + bogus values. --- .../core/inference/external_provider.py | 24 +++++++++---------- .../tests/test_sampling_params_routing.py | 23 ++++++++++-------- .../features/chat/provider-capabilities.ts | 16 ++++++------- 3 files changed, 33 insertions(+), 30 deletions(-) diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index d55f132487..c703129652 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -2831,18 +2831,18 @@ class ExternalProviderClient: "input": input_items, "stream": True, } - # Responses accepts the same service_tier enum set as Chat - # Completions (auto|default|flex|scale|priority) per the live - # `openai-python` SDK - # (`src/openai/types/responses/response_create_params.py` - # declares `Optional[Literal["auto", "default", "flex", - # "scale", "priority"]]`). parallel_tool_calls follows the same - # shape (default true). The frontend capability gate - # (provider-capabilities.ts) already filters per-provider, so - # we just forward whatever value the dispatcher hands us, with - # `standard_only` (Anthropic-only) being the one value Responses - # has never accepted. - if service_tier in ("auto", "default", "flex", "scale", "priority"): + # Responses accepts service_tier on the Chat Completions enum + # MINUS `scale`. The `openai-python` SDK type + # (`src/openai/types/responses/response_create_params.py`) + # technically includes `scale`, but the live OpenAI Responses + # API reference and the PR's own provider matrix only list + # `auto|default|flex|priority` for /v1/responses, and an + # independent round of 20 codex reviewers reached the same + # conclusion. Drop `scale` here to prevent the 400 risk — + # users who want Scale Tier can still pick it on a Chat + # Completions-compat provider where the SDK enum is honored. + # parallel_tool_calls follows the same shape (default true). + if service_tier in ("auto", "default", "flex", "priority"): body["service_tier"] = service_tier if parallel_tool_calls is not None: body["parallel_tool_calls"] = bool(parallel_tool_calls) diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index c27d782c62..9777d5ef73 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -385,22 +385,25 @@ def test_openai_responses_forwards_service_tier(monkeypatch): assert body.get("service_tier") == "priority", body -@pytest.mark.parametrize("value", ["auto", "default", "flex", "scale", "priority"]) -def test_openai_responses_accepts_full_service_tier_enum(monkeypatch, value): - """`openai-python`'s ResponseCreateParams declares - `Optional[Literal["auto", "default", "flex", "scale", "priority"]]` - so every value in that set forwards untouched.""" +@pytest.mark.parametrize("value", ["auto", "default", "flex", "priority"]) +def test_openai_responses_forwards_documented_service_tiers(monkeypatch, value): + """The live OpenAI Responses API reference lists `service_tier` as + `auto|default|flex|priority` for /v1/responses. Pin that every value + in the documented enum forwards untouched.""" captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) body = _drive_openai_responses(captured, service_tier = value) assert body.get("service_tier") == value, body -def test_openai_responses_drops_anthropic_only_service_tier(monkeypatch): - """`standard_only` is Anthropic-only and Responses has never accepted - it. Drop it client-side so a stale frontend cannot 400 the request. - """ +@pytest.mark.parametrize("bogus", ["scale", "standard_only", "bogus", ""]) +def test_openai_responses_drops_undocumented_service_tier(monkeypatch, bogus): + """`scale` and `standard_only` are not in the documented Responses + request enum. Drop them client-side so a stale frontend cannot 400 + the request. Round 4 consensus (~9/20 reviewers) flagged the + earlier permissive list as a 400 risk; this test pins the + restricted form.""" captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) - body = _drive_openai_responses(captured, service_tier = "standard_only") + body = _drive_openai_responses(captured, service_tier = bogus) assert "service_tier" not in body, body diff --git a/studio/frontend/src/features/chat/provider-capabilities.ts b/studio/frontend/src/features/chat/provider-capabilities.ts index fb1c2ea0f9..1c628fb199 100644 --- a/studio/frontend/src/features/chat/provider-capabilities.ts +++ b/studio/frontend/src/features/chat/provider-capabilities.ts @@ -71,13 +71,13 @@ export type ServiceTierOption = /** * Legal `service_tier` values per provider, sourced from each upstream's * docs. Anthropic exposes only `auto` and `standard_only`. OpenAI in - * Studio is routed through `/v1/responses` (not Chat Completions); the - * live `openai-python` SDK declares the Responses-side service_tier as - * `Literal["auto", "default", "flex", "scale", "priority"]` - * (`src/openai/types/responses/response_create_params.py`), so the - * full set is exposed. Other providers fall through to a permissive - * `auto` / `default` pair so the picker stays usable for - * OpenAI-compat backends. + * Studio is routed through `/v1/responses` (not Chat Completions). The + * live OpenAI Responses API reference and the PR contract both list + * only `auto|default|flex|priority` for /v1/responses, so `scale` is + * deliberately excluded here even though the openai-python SDK type + * is wider — sending an undocumented Responses value is a 400 risk. + * Other providers fall through to a permissive `auto` / `default` + * pair so the picker stays usable for OpenAI-compat backends. */ export function getServiceTierOptions( providerType: string | null | undefined, @@ -86,7 +86,7 @@ export function getServiceTierOptions( return ["auto", "standard_only"] as const; } if (providerType === "openai") { - return ["auto", "default", "flex", "scale", "priority"] as const; + return ["auto", "default", "flex", "priority"] as const; } return ["auto", "default"] as const; } From b48d68f8bf039a1ecdf784b99c94112297ba0703 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 14:32:36 +0000 Subject: [PATCH 17/56] Fix Mistral seed mapping, raise default OAI-compat stop cap, thread sampling through GGUF direct path Mistral chat completions uses random_seed not seed; map the field via a new seed_field on the provider registry so the new seed control actually works on Mistral. Default for other providers stays seed. DeepSeek and Mistral both accept up to 16 stop sequences but the default OAI-compat branch was hard-capping at 4 (the OpenAI Chat limit). Studio routes the openai provider through /v1/responses not /v1/chat/completions so the 4-cap only applies if we explicitly added an openai entry. Raise the default to 16 and let per-provider stop_max overrides tighten if needed. The local GGUF direct chat path (gguf_generate / gguf_generate_with_tools) bypassed _build_openai_passthrough_body and therefore dropped frequency_penalty, seed, stop, and parallel_tool_calls on the floor for users on the default no-tools and with-tools paths. Thread the new fields through LlamaCppBackend.generate_chat_completion and generate_chat_completion_with_tools and the two callsites that invoke them. Also tighten comments to drop review-process narration that crept in and to remove the em dashes I had introduced in this PR's earlier commits. Tests pin the Mistral random_seed rename, the DeepSeek 16-cap, and confirm the openai-compat default cap is 16. --- .../core/inference/external_provider.py | 128 +++++++----------- studio/backend/core/inference/llama_cpp.py | 21 +++ studio/backend/core/inference/providers.py | 3 + studio/backend/routes/chat_history.py | 7 +- studio/backend/routes/inference.py | 30 ++-- .../tests/test_sampling_params_routing.py | 113 ++++++++++++---- .../components/ui/stop-sequences-input.tsx | 10 +- .../src/features/chat/chat-settings-sheet.tsx | 12 +- .../features/chat/presets/preset-policy.ts | 11 +- .../features/chat/provider-capabilities.ts | 16 +-- .../chat/utils/chat-settings-storage.ts | 29 ++-- 11 files changed, 206 insertions(+), 174 deletions(-) diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index c703129652..904380ac5d 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -459,35 +459,36 @@ class ExternalProviderClient: else: body["max_tokens"] = max_tokens - # Optional sampling extensions (added in #5XXX). Only forwarded - # when the caller passed a value. Each upstream provider that - # 400s on the field appears in `body_omit` (see providers.py) - # so the registry-driven drop loop below removes them before - # the request hits the wire. The Responses path - # (_stream_openai_responses) drops these explicitly because it - # never reaches this body construction. + # Optional sampling extensions. Only forwarded when the caller + # passed a value. Per-provider rename / cap is applied via + # `seed_field` and `stop_max` on the provider registry below, + # and body_omit strips fields the upstream rejects. + from core.inference.providers import get_provider_info + + provider_info = get_provider_info(self.provider_type) or {} if frequency_penalty is not None: body["frequency_penalty"] = frequency_penalty if seed is not None: - body["seed"] = seed + # Mistral renames `seed` to `random_seed` on /v1/chat/completions. + seed_field = provider_info.get("seed_field", "seed") + body[seed_field] = seed if stop: - # OpenAI Chat caps the list at 4 entries. Dedupe + drop - # empties first so users entering chips with whitespace or - # accidental repeats don't waste budget against the cap or - # trip a 400. + # Stop cap is provider-specific. OpenAI Chat = 4, Anthropic + # = 16 (client guard), DeepSeek = 16, others = 16 by default. + stop_max = int(provider_info.get("stop_max", 16)) if isinstance(stop, str): body["stop"] = stop elif isinstance(stop, list): sequences = list( dict.fromkeys(s for s in stop if isinstance(s, str) and s) ) - if len(sequences) > 4: + if len(sequences) > stop_max: logger.warning( - "stop sequences truncated to 4 entries " - "(received %d, OpenAI's hard cap is 4)", + "stop sequences truncated to %d entries (received %d)", + stop_max, len(sequences), ) - body["stop"] = sequences[:4] + body["stop"] = sequences[:stop_max] elif sequences: body["stop"] = sequences if service_tier is not None: @@ -495,15 +496,8 @@ class ExternalProviderClient: if parallel_tool_calls is not None: body["parallel_tool_calls"] = parallel_tool_calls - # Strip body fields a provider's registry entry declares unusable — - # reasoning-class models that lock these to fixed defaults (e.g. - # Kimi k2.5/k2.6 only accept temperature=1, top_p=1) 400 otherwise. - # The frontend capability map already hides the matching sliders; - # this is the matching guard for the pydantic default that the - # route layer would otherwise still fill in. - from core.inference.providers import get_provider_info - - provider_info = get_provider_info(self.provider_type) or {} + # Drop body fields the provider's registry entry locks down + # (e.g. Kimi k2.5/k2.6 only accept temperature=1, top_p=1). for field in provider_info.get("body_omit", ()): body.pop(field, None) @@ -900,11 +894,10 @@ class ExternalProviderClient: if max_tokens is not None: body["max_tokens"] = max_tokens - # Forward the new optional sampling extensions (#5711) on the - # web-search bypass too. The default OAI-compat body construction - # (which adds these) is skipped because this helper returns - # early; forwarding here ensures kimi-with-search honours the - # same sampling controls as kimi-without-search. + # The default OAI-compat body construction is skipped because + # this helper returns early. Forward the optional sampling + # extensions here so kimi-with-search behaves the same as + # kimi-without-search. if presence_penalty is not None: body["presence_penalty"] = presence_penalty if frequency_penalty is not None: @@ -912,13 +905,8 @@ class ExternalProviderClient: if seed is not None: body["seed"] = seed if stop: - # Mirror the default OAI-compat path's stop handling exactly - # so Kimi-with-search and Kimi-without-search apply the - # same rules — a single string is forwarded verbatim and - # lists are deduped + truncated to OpenAI's 4-entry cap. - # Earlier the bypass dropped whitespace-only strings here - # while the normal path forwarded them, which was an - # asymmetric provider-path fix. + # Match the default OAI-compat path: forward a single string + # verbatim, dedupe + cap lists to 4 (OpenAI hard limit). if isinstance(stop, str): body["stop"] = stop elif isinstance(stop, list): @@ -1453,32 +1441,26 @@ class ExternalProviderClient: body["top_k"] = top_k # Optional sampling extensions. Anthropic has no - # frequency_penalty / seed / logprobs equivalents, so those are - # silently dropped by virtue of not being forwarded from - # stream_chat_completion. The two body-level knobs Anthropic - # does accept land here: - # stop → stop_sequences (renamed, ws-stripped) - # service_tier → service_tier (auto|standard_only only) - # parallel_tool_calls inversion is applied AFTER the tools - # wiring below, because Anthropic requires it nested under - # tool_choice (top-level placement is rejected with - # `extraneous key [disable_parallel_tool_use] is not permitted`). + # frequency_penalty / seed / logprobs equivalents so they are + # never forwarded here. The two body-level knobs Anthropic + # accepts land here: + # stop -> stop_sequences (renamed, ws-stripped) + # service_tier -> service_tier (auto|standard_only only) + # parallel_tool_calls inversion is applied after the tools + # wiring below because Anthropic requires it nested under + # tool_choice. if stop: sequences: list[str] if isinstance(stop, str): sequences = [stop] if stop.strip() else [] else: - # Dedupe + drop whitespace-only entries. Anthropic 400s - # on any sequence that contains no non-whitespace char: - # `stop_sequences: each stop sequence must contain - # non-whitespace`. That rejects empty strings, " ", and - # — critically — common defaults like "\n" / "\n\n". - # The truncation cap below (16) is a client-side guard; - # the Anthropic Messages API does not publish a max - # array length but every SDK we have inspected treats - # 16 as a sane ceiling (Bedrock's hard cap is 8191, so - # this only matters when callers paste pathologically - # long lists by accident). + # Dedupe + drop whitespace-only entries. Anthropic + # rejects any sequence with no non-whitespace char + # ("stop_sequences: each stop sequence must contain + # non-whitespace"), so "", " ", "\n", "\n\n" are all + # filtered. The 16-cap is a client-side guard; the + # docs do not publish a max, but every SDK treats 16 + # as a sane ceiling (Bedrock is the outlier at 8191). sequences = list( dict.fromkeys(s for s in stop if isinstance(s, str) and s.strip()) ) @@ -1723,15 +1705,11 @@ class ExternalProviderClient: if anthropic_code_exec_container_id: body["container"] = anthropic_code_exec_container_id - # parallel_tool_calls=false → disable_parallel_tool_use=true, - # nested under tool_choice (NOT top-level). The Anthropic - # Messages API only accepts `disable_parallel_tool_use` as a - # property on the `tool_choice` object (ToolChoiceAuto / - # ToolChoiceAny / ToolChoiceTool). Top-level placement is - # rejected with `extraneous key [disable_parallel_tool_use] - # is not permitted`. Without any tools the flag is also a - # no-op upstream — skip it to keep the request body minimal. - # See + # parallel_tool_calls=False maps to disable_parallel_tool_use= + # True nested under tool_choice. Top-level placement is + # rejected with "extraneous key [disable_parallel_tool_use] + # is not permitted". Without tools the flag is a no-op so the + # block is skipped to keep the body minimal. See # https://platform.claude.com/docs/en/agents-and-tools/tool-use/implement-tool-use if parallel_tool_calls is False and body.get("tools"): tc = body.get("tool_choice") @@ -2831,17 +2809,11 @@ class ExternalProviderClient: "input": input_items, "stream": True, } - # Responses accepts service_tier on the Chat Completions enum - # MINUS `scale`. The `openai-python` SDK type - # (`src/openai/types/responses/response_create_params.py`) - # technically includes `scale`, but the live OpenAI Responses - # API reference and the PR's own provider matrix only list - # `auto|default|flex|priority` for /v1/responses, and an - # independent round of 20 codex reviewers reached the same - # conclusion. Drop `scale` here to prevent the 400 risk — - # users who want Scale Tier can still pick it on a Chat - # Completions-compat provider where the SDK enum is honored. - # parallel_tool_calls follows the same shape (default true). + # Responses accepts auto|default|flex|priority per the live + # docs. The openai-python SDK type happens to include "scale" + # too but the public Responses reference does not, so drop it + # here to avoid a 400. Scale Tier is still selectable on Chat + # Completions backends. parallel_tool_calls default is true. if service_tier in ("auto", "default", "flex", "priority"): body["service_tier"] = service_tier if parallel_tool_calls is not None: diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index bf8a3c04df..e227053118 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -4245,6 +4245,9 @@ class LlamaCppBackend: enable_thinking: Optional[bool] = None, reasoning_effort: Optional[str] = None, preserve_thinking: Optional[bool] = None, + frequency_penalty: Optional[float] = None, + seed: Optional[int] = None, + parallel_tool_calls: Optional[bool] = None, ) -> Generator[str | dict, None, None]: """ Send a chat completion request to llama-server and stream tokens back. @@ -4286,6 +4289,14 @@ class LlamaCppBackend: payload["t_max_predict_ms"] = _DEFAULT_T_MAX_PREDICT_MS if stop: payload["stop"] = stop + # Optional sampling extensions, gated on `is not None` so 0, + # 0.0, and False all reach the wire. + if frequency_penalty is not None: + payload["frequency_penalty"] = frequency_penalty + if seed is not None: + payload["seed"] = seed + if parallel_tool_calls is not None: + payload["parallel_tool_calls"] = parallel_tool_calls payload["stream_options"] = {"include_usage": True} url = f"{self.base_url}/v1/chat/completions" @@ -4428,6 +4439,9 @@ class LlamaCppBackend: auto_heal_tool_calls: bool = True, tool_call_timeout: int = 300, session_id: Optional[str] = None, + frequency_penalty: Optional[float] = None, + seed: Optional[int] = None, + parallel_tool_calls: Optional[bool] = None, ) -> Generator[dict, None, None]: """ Agentic loop: let the model call tools, execute them, and continue. @@ -4512,6 +4526,13 @@ class LlamaCppBackend: payload["t_max_predict_ms"] = _DEFAULT_T_MAX_PREDICT_MS if stop: payload["stop"] = stop + # Optional sampling extensions; gated on `is not None`. + if frequency_penalty is not None: + payload["frequency_penalty"] = frequency_penalty + if seed is not None: + payload["seed"] = seed + if parallel_tool_calls is not None: + payload["parallel_tool_calls"] = parallel_tool_calls try: _auth_headers = ( diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index fef9ba3e12..98c8abe53b 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -131,6 +131,9 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { r"mistral-(?:large|medium|small|tiny)-latest|" r"mistral-vibe-cli-latest)$" ), + # Mistral renames OpenAI's `seed` to `random_seed` on + # /v1/chat/completions. https://docs.mistral.ai/api/endpoint/chat + "seed_field": "random_seed", }, "kimi": { "display_name": "Kimi", diff --git a/studio/backend/routes/chat_history.py b/studio/backend/routes/chat_history.py index 6f7efb6eb5..800873afba 100644 --- a/studio/backend/routes/chat_history.py +++ b/studio/backend/routes/chat_history.py @@ -107,10 +107,9 @@ class ChatInferenceSettings(BaseModel): minP: Optional[float] = None repetitionPenalty: Optional[float] = None presencePenalty: Optional[float] = None - # New per-provider sampling knobs. `extra="forbid"` would 422 any - # settings save from a frontend on the new code if these were not - # listed here, breaking the entire chat-settings persistence path. - # Keep these aligned with `InferenceParams` in + # New per-provider sampling knobs. extra="forbid" requires these + # to be listed; otherwise every save from the new frontend 422s. + # Keep aligned with InferenceParams in # studio/frontend/src/features/chat/types/runtime.ts. frequencyPenalty: Optional[float] = Field(default = None, ge = -2.0, le = 2.0) seed: Optional[int] = None diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 8c7267c0e2..5ee25e9922 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -2474,6 +2474,9 @@ async def openai_chat_completions( max_tokens = payload.max_tokens, repetition_penalty = payload.repetition_penalty, presence_penalty = payload.presence_penalty, + stop = payload.stop if isinstance(payload.stop, list) else ( + [payload.stop] if isinstance(payload.stop, str) and payload.stop else None + ), cancel_event = cancel_event, enable_thinking = payload.enable_thinking, reasoning_effort = payload.reasoning_effort, @@ -2488,6 +2491,9 @@ async def openai_chat_completions( if payload.tool_call_timeout is not None else 300, session_id = payload.session_id, + frequency_penalty = payload.frequency_penalty, + seed = payload.seed, + parallel_tool_calls = payload.parallel_tool_calls, ) _tool_sentinel = object() @@ -2653,10 +2659,16 @@ async def openai_chat_completions( max_tokens = payload.max_tokens, repetition_penalty = payload.repetition_penalty, presence_penalty = payload.presence_penalty, + stop = payload.stop if isinstance(payload.stop, list) else ( + [payload.stop] if isinstance(payload.stop, str) and payload.stop else None + ), cancel_event = cancel_event, enable_thinking = payload.enable_thinking, reasoning_effort = payload.reasoning_effort, preserve_thinking = payload.preserve_thinking, + frequency_penalty = payload.frequency_penalty, + seed = payload.seed, + parallel_tool_calls = payload.parallel_tool_calls, ) _gguf_sentinel = object() @@ -3780,12 +3792,9 @@ def _build_chat_request( chat_kwargs["top_p"] = payload.top_p if payload.max_output_tokens is not None: chat_kwargs["max_tokens"] = payload.max_output_tokens - # `parallel_tool_calls` is now a first-class field on - # ChatCompletionRequest (PR #5711) and the OpenAI-compat - # passthrough builder forwards it. Translate it here so a Responses - # API caller (e.g. OpenAI Codex SDK) that sets - # `parallel_tool_calls=false` actually sees the preference reach - # llama-server instead of getting silently dropped at the bridge. + # parallel_tool_calls is first-class on ChatCompletionRequest and + # the OpenAI-compat passthrough builder forwards it. Translate it + # so a Responses API caller's preference reaches llama-server. if payload.parallel_tool_calls is not None: chat_kwargs["parallel_tool_calls"] = payload.parallel_tool_calls @@ -4934,12 +4943,9 @@ def _build_passthrough_payload( body["repeat_penalty"] = repetition_penalty if presence_penalty is not None: body["presence_penalty"] = presence_penalty - # New per-provider sampling extensions (PR #5711). llama-server's - # /v1/chat/completions endpoint accepts the standard OpenAI fields, - # so forward them straight through. parallel_tool_calls is a no-op - # on llama-server today (the upstream always dispatches sequentially) - # but forward it anyway so a future llama-server release that - # implements it picks up the user's preference automatically. + # llama-server's /v1/chat/completions accepts the standard OpenAI + # fields. parallel_tool_calls is a no-op on llama-server today but + # is forwarded so a future release picks it up automatically. if frequency_penalty is not None: body["frequency_penalty"] = frequency_penalty if seed is not None: diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index 9777d5ef73..8d8b199d78 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -171,11 +171,10 @@ def test_anthropic_disable_parallel_tool_use_nested_under_tool_choice(monkeypatc """ captured = _install_mock(monkeypatch) body = _drive_anthropic_with_tools(captured, parallel_tool_calls = False) - # Top-level fields must not carry the flag — Anthropic 400s otherwise. + # Top-level placement is rejected with 400. assert "disable_parallel_tool_use" not in body, body assert "parallel_tool_calls" not in body, body - # The flag is set on tool_choice. Default type is "auto" when the - # user didn't pick one explicitly. + # Flag lives on tool_choice; default type is "auto". tc = body.get("tool_choice") assert isinstance(tc, dict), body assert tc.get("disable_parallel_tool_use") is True, body @@ -183,9 +182,8 @@ def test_anthropic_disable_parallel_tool_use_nested_under_tool_choice(monkeypatc def test_anthropic_disable_parallel_tool_use_skipped_without_tools(monkeypatch): - """Without any tools defined, `disable_parallel_tool_use` is a - no-op upstream — skip it so the request body stays minimal and the - flag never lands at top level either. + """Without tools the flag is a no-op upstream; keep the body + minimal and never emit it at top level either. """ captured = _install_mock(monkeypatch) body = _drive_anthropic(captured, parallel_tool_calls = False) @@ -271,7 +269,64 @@ def test_openai_compat_forwards_frequency_penalty(monkeypatch): def test_openai_compat_forwards_seed(monkeypatch): captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) body = _drive_openai_compat(captured, seed = 12345) - assert body.get("seed") == 12345, body + # Default OAI-compat provider (mistral here) renames seed to + # random_seed via provider registry's seed_field. + assert body.get("random_seed") == 12345, body + assert "seed" not in body, body + + +def test_openai_compat_seed_field_default_is_seed(monkeypatch): + """Providers without a seed_field override get the OpenAI default.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "deepseek", + base_url = "https://api.deepseek.com/v1", + api_key = "ds-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "deepseek-chat", + temperature = 0.5, + top_p = 0.9, + max_tokens = 64, + seed = 7, + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert body.get("seed") == 7, body + assert "random_seed" not in body, body + + +def test_openai_compat_deepseek_stop_cap_is_16(monkeypatch): + """DeepSeek docs allow up to 16 stop sequences; the previous + 4-cap silently truncated valid configs.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "deepseek", + base_url = "https://api.deepseek.com/v1", + api_key = "ds-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "deepseek-chat", + temperature = 0.5, + top_p = 0.9, + max_tokens = 64, + stop = [f"S{i}" for i in range(20)], + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert len(body.get("stop", [])) == 16, body def test_openai_compat_forwards_stop_array(monkeypatch): @@ -282,14 +337,18 @@ def test_openai_compat_forwards_stop_array(monkeypatch): assert "stop_sequences" not in body, body -def test_openai_compat_truncates_stop_to_four(monkeypatch): +def test_openai_compat_truncates_stop_to_default_cap(monkeypatch): + """Default OAI-compat cap is 16 (DeepSeek and Mistral both accept + that many); only OpenAI Chat has a tighter 4-entry hard limit.""" captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) - body = _drive_openai_compat(captured, stop = ["a", "b", "c", "d", "e", "f"]) - assert body.get("stop") == ["a", "b", "c", "d"], body + body = _drive_openai_compat(captured, stop = [f"s{i}" for i in range(20)]) + assert len(body.get("stop", [])) == 16, body + assert body["stop"][0] == "s0" + assert body["stop"][-1] == "s15" def test_openai_compat_stop_dedup_and_drop_empties(monkeypatch): - """Duplicates and empties shouldn't eat into the 4-entry cap.""" + """Duplicates and empties shouldn't eat into the cap.""" captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) body = _drive_openai_compat(captured, stop = ["END", "", "END", "DONE", "FIN", "END"]) assert body.get("stop") == ["END", "DONE", "FIN"], body @@ -398,10 +457,8 @@ def test_openai_responses_forwards_documented_service_tiers(monkeypatch, value): @pytest.mark.parametrize("bogus", ["scale", "standard_only", "bogus", ""]) def test_openai_responses_drops_undocumented_service_tier(monkeypatch, bogus): """`scale` and `standard_only` are not in the documented Responses - request enum. Drop them client-side so a stale frontend cannot 400 - the request. Round 4 consensus (~9/20 reviewers) flagged the - earlier permissive list as a 400 risk; this test pins the - restricted form.""" + request enum; drop them client-side so a stale frontend never + sends an upstream-rejected value.""" captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) body = _drive_openai_responses(captured, service_tier = bogus) assert "service_tier" not in body, body @@ -526,10 +583,8 @@ def test_kimi_web_search_bypass_forwards_new_sampling_fields(monkeypatch): def test_local_openai_passthrough_forwards_new_sampling_fields(): - """Round 1 reviewers (10/20) flagged that - `_build_openai_passthrough_body` dropped frequency_penalty / seed / - parallel_tool_calls when forwarding to llama-server. Pin the - extended contract.""" + """`_build_openai_passthrough_body` forwards frequency_penalty, + seed, stop, and parallel_tool_calls to llama-server.""" from models.inference import ChatCompletionRequest from routes.inference import _build_openai_passthrough_body @@ -554,11 +609,9 @@ def test_local_openai_passthrough_forwards_new_sampling_fields(): def test_responses_to_chat_bridge_preserves_parallel_tool_calls(): - """Round 3 reviewers flagged that `_build_chat_request` (the - /v1/responses → /v1/chat/completions translator) dropped - `parallel_tool_calls`, so a Responses-API caller that set - `parallel_tool_calls=false` never saw the flag reach llama-server. - Pin the translation.""" + """`_build_chat_request` (the /v1/responses to /v1/chat/completions + translator) must forward parallel_tool_calls so a Responses-API + caller's preference reaches llama-server.""" from models.inference import ChatMessage, ResponsesRequest from routes.inference import _build_chat_request, _build_openai_passthrough_body @@ -578,9 +631,9 @@ def test_responses_to_chat_bridge_preserves_parallel_tool_calls(): def test_responses_to_chat_bridge_omits_unset_parallel_tool_calls(): - """Unset `parallel_tool_calls` (None) must not appear on the - translated body — the upstream default is `true` everywhere, so - forwarding `parallel_tool_calls=None` would over-specify.""" + """Unset parallel_tool_calls (None) must not appear on the + translated body; the upstream default is true everywhere so + forwarding None would over-specify.""" from models.inference import ChatMessage, ResponsesRequest from routes.inference import _build_chat_request, _build_openai_passthrough_body @@ -599,9 +652,9 @@ def test_responses_to_chat_bridge_omits_unset_parallel_tool_calls(): def test_chat_settings_payload_accepts_new_sampling_keys(): - """Round 1 reviewers flagged that `ChatSettingsPayload.extra="forbid"` - with the old field list 422'd every settings save that contained - any of the new keys. Pin that the new keys round-trip.""" + """ChatSettingsPayload has extra="forbid" so the new keys must be + listed explicitly; otherwise every settings save with any of them + 422s. Pin the round-trip.""" from routes.chat_history import ChatSettingsPayload parsed = ChatSettingsPayload.model_validate( diff --git a/studio/frontend/src/components/ui/stop-sequences-input.tsx b/studio/frontend/src/components/ui/stop-sequences-input.tsx index 5d65fdd36a..2e647de01e 100644 --- a/studio/frontend/src/components/ui/stop-sequences-input.tsx +++ b/studio/frontend/src/components/ui/stop-sequences-input.tsx @@ -38,13 +38,9 @@ export function StopSequencesInput({ const atCap = value.length >= maxEntries; function commitDraft() { - // Reject chips that are empty or contain ONLY whitespace - // (Anthropic 400s on those and OpenAI silently drops them), but - // preserve significant leading/trailing whitespace inside otherwise - // -meaningful stops like " END", "### ", or "\n\n" — stop matching - // is exact, so stripping would silently change the semantics. The - // backend re-validates per-provider before the request hits the - // wire. + // Reject empty / whitespace-only chips but preserve significant + // leading/trailing whitespace (stop matching is exact). Backend + // re-validates per provider. if (!draft || !draft.trim()) return; if (atCap) return; if (value.includes(draft)) { diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index d02369b4ef..271dfe22af 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -427,14 +427,10 @@ export function ChatSettingsPanel({ isExternalModel && Boolean(providerCapabilities?.serviceTier); const showParallelToolCalls = !isExternalModel || Boolean(providerCapabilities?.parallelToolCalls); - // OpenAI Chat docs cap `stop` at 4 entries; Anthropic accepts more. - // Pick the right ceiling per active connection so the chips editor's - // placeholder doesn't lie. Local backends (llama.cpp / vLLM / ollama - // / generic OpenAI-compat connections) accept many more — match the - // Anthropic cap there so we do not block users from using stop - // sequences the backend would happily accept. The wire-side - // truncation in `_stream_openai_compat` will still trim to OpenAI's - // 4-entry hard cap when a cloud OpenAI endpoint receives the body. + // OpenAI Chat caps `stop` at 4; Anthropic, DeepSeek, Mistral, and + // local llama.cpp / vLLM / ollama backends accept more. Use 16 as + // the UI ceiling for everything that is not OpenAI cloud Chat; the + // backend re-trims per provider on the wire. const stopMaxEntries = !isExternalModel || externalProviderType === "anthropic" ? 16 : 4; const serviceTierOptions = getServiceTierOptions(externalProviderType); diff --git a/studio/frontend/src/features/chat/presets/preset-policy.ts b/studio/frontend/src/features/chat/presets/preset-policy.ts index 7cd5c76048..d853ef5073 100644 --- a/studio/frontend/src/features/chat/presets/preset-policy.ts +++ b/studio/frontend/src/features/chat/presets/preset-policy.ts @@ -14,13 +14,10 @@ export interface Preset { } // Fields that belong to a preset. Sampling knobs are included so a -// user can save a preset that fixes their preferred decoding style and -// re-apply it on any model. Operational knobs (`serviceTier`, which is -// account-level and per-provider, and `parallelToolCalls`, which is -// tool-level state) are intentionally excluded so switching presets -// does not silently change request routing for the active provider. -// `seed` is also excluded — it is per-request determinism state, not a -// reusable preset value. +// user can save a preset that fixes their preferred decoding style. +// Operational knobs (serviceTier, parallelToolCalls) and per-request +// determinism state (seed) are intentionally excluded so switching +// presets does not silently change request routing. export type PresetOwnedParams = Pick< InferenceParams, | "temperature" diff --git a/studio/frontend/src/features/chat/provider-capabilities.ts b/studio/frontend/src/features/chat/provider-capabilities.ts index 1c628fb199..8b9212512c 100644 --- a/studio/frontend/src/features/chat/provider-capabilities.ts +++ b/studio/frontend/src/features/chat/provider-capabilities.ts @@ -69,15 +69,13 @@ export type ServiceTierOption = | "standard_only"; /** - * Legal `service_tier` values per provider, sourced from each upstream's - * docs. Anthropic exposes only `auto` and `standard_only`. OpenAI in - * Studio is routed through `/v1/responses` (not Chat Completions). The - * live OpenAI Responses API reference and the PR contract both list - * only `auto|default|flex|priority` for /v1/responses, so `scale` is - * deliberately excluded here even though the openai-python SDK type - * is wider — sending an undocumented Responses value is a 400 risk. - * Other providers fall through to a permissive `auto` / `default` - * pair so the picker stays usable for OpenAI-compat backends. + * Legal `service_tier` values per provider. Anthropic exposes only + * `auto` and `standard_only`. OpenAI in Studio is routed through + * `/v1/responses`, which the live docs list as + * `auto|default|flex|priority`; `scale` is excluded here even though + * the openai-python SDK type happens to include it. Other providers + * fall through to a permissive `auto` / `default` pair so the picker + * stays usable for OpenAI-compat backends. */ export function getServiceTierOptions( providerType: string | null | undefined, diff --git a/studio/frontend/src/features/chat/utils/chat-settings-storage.ts b/studio/frontend/src/features/chat/utils/chat-settings-storage.ts index e529b792d1..e6f6b693f1 100644 --- a/studio/frontend/src/features/chat/utils/chat-settings-storage.ts +++ b/studio/frontend/src/features/chat/utils/chat-settings-storage.ts @@ -45,15 +45,12 @@ const NUMERIC_INFERENCE_FIELDS = [ "maxTokens", ] as const satisfies readonly (keyof PersistedInferenceParams)[]; -// `seed` is numeric but nullable (null = "no seed field on the wire") so -// it can't go through the NUMERIC_INFERENCE_FIELDS Finite-number filter. -// Keep this set in sync with `ServiceTier` in ../types/runtime.ts and with -// `getServiceTierOptions` in ../provider-capabilities.ts. `standard_only` -// is Anthropic-only and was missing here — dropping it on save erased the -// user's Anthropic tier choice on reload. `scale` was removed from the UI -// (Codex review feedback) but accepting it on load is harmless for stale -// persisted state, so it stays in the allowlist; the resolver no longer -// surfaces it. +// `seed` is numeric but nullable (null = no seed on the wire) so it +// can't go through the NUMERIC_INFERENCE_FIELDS finite-number filter. +// Keep this set in sync with `ServiceTier` in ../types/runtime.ts and +// `getServiceTierOptions` in ../provider-capabilities.ts. `scale` is +// kept in the allowlist for forward-compat with legacy persisted data +// even though the resolver no longer surfaces it for OpenAI Responses. const VALID_SERVICE_TIERS = new Set([ "auto", "default", @@ -165,16 +162,10 @@ function sanitizeInferenceParams( } else if (typeof value.seed === "number" && Number.isInteger(value.seed)) { params.seed = value.seed; } - // stop: capped string array. Use Anthropic's 16-entry max so the - // sanitizer never silently discards entries the user actually typed - // for an Anthropic session. The backend's per-provider stream helper - // re-truncates to the wire cap (4 for OpenAI Chat, 16 for Anthropic, - // dropped entirely for OpenAI Responses) before the request hits the - // network. Capping to 4 here would defeat Anthropic's UI cap of 16 - // for users who switch providers between sessions. An EMPTY array - // must persist as an empty array (not be sanitized away) so the user - // can clear the last chip and have the change saved — otherwise the - // previously stored stops come back on reload. + // stop: cap at 16 (Anthropic's widest supported). The backend's + // per-provider stream helper re-truncates to the wire cap. An EMPTY + // array must persist (not be sanitized away) so clearing the last + // chip actually saves; otherwise the old stops come back on reload. if (Array.isArray(value.stop)) { const stops = value.stop.filter((s): s is string => typeof s === "string"); params.stop = stops.slice(0, 16); From f200bc20c06f9227285d62a17cfa410fbb9d19ce Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 24 May 2026 14:33:38 +0000 Subject: [PATCH 18/56] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/routes/inference.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 5ee25e9922..9f5b07317b 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -2474,8 +2474,12 @@ async def openai_chat_completions( max_tokens = payload.max_tokens, repetition_penalty = payload.repetition_penalty, presence_penalty = payload.presence_penalty, - stop = payload.stop if isinstance(payload.stop, list) else ( - [payload.stop] if isinstance(payload.stop, str) and payload.stop else None + stop = payload.stop + if isinstance(payload.stop, list) + else ( + [payload.stop] + if isinstance(payload.stop, str) and payload.stop + else None ), cancel_event = cancel_event, enable_thinking = payload.enable_thinking, @@ -2659,8 +2663,12 @@ async def openai_chat_completions( max_tokens = payload.max_tokens, repetition_penalty = payload.repetition_penalty, presence_penalty = payload.presence_penalty, - stop = payload.stop if isinstance(payload.stop, list) else ( - [payload.stop] if isinstance(payload.stop, str) and payload.stop else None + stop = payload.stop + if isinstance(payload.stop, list) + else ( + [payload.stop] + if isinstance(payload.stop, str) and payload.stop + else None ), cancel_event = cancel_event, enable_thinking = payload.enable_thinking, From 95e143545f209eb07331b41a04fbd98c82d63bef Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 14:49:57 +0000 Subject: [PATCH 19/56] Per-provider stop cap on Kimi web-search bypass and frontend sheet Round 5 review flagged two asymmetries: 1. Kimi web-search bypass hard-capped stops at 4 while the default OAI-compat path honours provider_info["stop_max"]. Apply the same provider-aware logic in _stream_kimi_web_search so kimi-with-search and kimi-without-search match. Also add Kimi's documented 5-stop max (https://platform.kimi.ai/docs/api/chat) to the provider registry so the cap actually fires. 2. chat-settings-sheet.tsx caps every non-Anthropic external provider at 4 stops. Replace with a per-provider getProviderStopMax helper in provider-capabilities.ts so DeepSeek, Mistral, and local backends are not artificially restricted while OpenAI Chat still hits its 4-entry hard limit and Kimi hits its documented 5-entry cap. Tests pin the Kimi 5-cap on both Kimi paths. --- .../core/inference/external_provider.py | 27 ++++---- studio/backend/core/inference/providers.py | 3 + .../tests/test_sampling_params_routing.py | 66 +++++++++++++++++-- .../src/features/chat/chat-settings-sheet.tsx | 11 ++-- .../features/chat/provider-capabilities.ts | 26 ++++++++ 5 files changed, 107 insertions(+), 26 deletions(-) diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index 904380ac5d..df06098156 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -895,41 +895,40 @@ class ExternalProviderClient: body["max_tokens"] = max_tokens # The default OAI-compat body construction is skipped because - # this helper returns early. Forward the optional sampling - # extensions here so kimi-with-search behaves the same as + # this helper returns early. Apply the same provider-aware + # sampling / stop logic here so kimi-with-search matches # kimi-without-search. + from core.inference.providers import get_provider_info + + provider_info = get_provider_info(self.provider_type) or {} if presence_penalty is not None: body["presence_penalty"] = presence_penalty if frequency_penalty is not None: body["frequency_penalty"] = frequency_penalty if seed is not None: - body["seed"] = seed + seed_field = provider_info.get("seed_field", "seed") + body[seed_field] = seed if stop: - # Match the default OAI-compat path: forward a single string - # verbatim, dedupe + cap lists to 4 (OpenAI hard limit). + stop_max = int(provider_info.get("stop_max", 16)) if isinstance(stop, str): body["stop"] = stop elif isinstance(stop, list): sequences = list( dict.fromkeys(s for s in stop if isinstance(s, str) and s) ) - if len(sequences) > 4: + if len(sequences) > stop_max: logger.warning( - "stop sequences truncated to 4 entries " - "(received %d, OpenAI's hard cap is 4)", + "stop sequences truncated to %d entries (received %d)", + stop_max, len(sequences), ) - body["stop"] = sequences[:4] + body["stop"] = sequences[:stop_max] elif sequences: body["stop"] = sequences if parallel_tool_calls is not None: body["parallel_tool_calls"] = parallel_tool_calls - # Strip body fields the Kimi registry declares unusable - # (temperature/top_p — see body_omit in providers.py). - from core.inference.providers import get_provider_info - - provider_info = get_provider_info(self.provider_type) or {} + # Drop body fields the provider's registry entry locks down. for field in provider_info.get("body_omit", ()): body.pop(field, None) diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index 98c8abe53b..4a47e90850 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -162,6 +162,9 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { # (and the same shape for top_p). Strip both fields from the # outbound body so the server falls back to its required defaults. "body_omit": ("temperature", "top_p"), + # Kimi accepts at most 5 stop strings (each <= 32 bytes) per + # https://platform.kimi.ai/docs/api/chat + "stop_max": 5, }, "qwen": { "display_name": "Qwen", diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index 8d8b199d78..012c2266f2 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -537,12 +537,10 @@ def test_chat_completion_request_clamps_frequency_penalty_range(): def test_kimi_web_search_bypass_forwards_new_sampling_fields(monkeypatch): - """The Kimi `enabled_tools=["web_search"]` path takes an early - return into `_stream_kimi_web_search` BEFORE the default OAI-compat - body builder runs. PR #5711 added new sampling fields to the - default builder; this test pins that the web-search bypass also - forwards them so Kimi-with-search and Kimi-without-search behave - consistently.""" + """The Kimi $web_search path takes an early return into + `_stream_kimi_web_search` before the default OAI-compat body + builder runs; forwarding here keeps Kimi-with-search and + Kimi-without-search in lockstep.""" captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) async def run(): @@ -579,6 +577,62 @@ def test_kimi_web_search_bypass_forwards_new_sampling_fields(monkeypatch): assert "top_p" not in body, body +def test_kimi_web_search_uses_kimi_stop_cap_5(monkeypatch): + """Kimi documents a 5-stop max; the web-search bypass must honour + `provider_info["stop_max"]` rather than the OpenAI 4-cap or the + permissive default.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "kimi", + base_url = "https://api.moonshot.ai/v1", + api_key = "kimi-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "kimi-k2.6", + temperature = 1.0, + top_p = 1.0, + max_tokens = 256, + enabled_tools = ["web_search"], + stop = [f"S{i}" for i in range(10)], + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert len(body.get("stop", [])) == 5, body + assert body["stop"] == ["S0", "S1", "S2", "S3", "S4"], body + + +def test_kimi_default_path_uses_kimi_stop_cap_5(monkeypatch): + """The normal Kimi path must also honour the documented 5-cap.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "kimi", + base_url = "https://api.moonshot.ai/v1", + api_key = "kimi-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "kimi-k2.6", + temperature = 1.0, + top_p = 1.0, + max_tokens = 256, + stop = [f"S{i}" for i in range(10)], + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert len(body.get("stop", [])) == 5, body + + # ── Local OpenAI passthrough forwards new sampling fields ────────────── diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index 271dfe22af..07a6e2ccf2 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -86,6 +86,7 @@ import { EXTERNAL_MAX_OUTPUT_TOKENS, type ProviderCapabilities, getExternalMinOutputTokens, + getProviderStopMax, getServiceTierOptions, providerSupportsBuiltinCodeExecution, } from "./provider-capabilities"; @@ -427,12 +428,10 @@ export function ChatSettingsPanel({ isExternalModel && Boolean(providerCapabilities?.serviceTier); const showParallelToolCalls = !isExternalModel || Boolean(providerCapabilities?.parallelToolCalls); - // OpenAI Chat caps `stop` at 4; Anthropic, DeepSeek, Mistral, and - // local llama.cpp / vLLM / ollama backends accept more. Use 16 as - // the UI ceiling for everything that is not OpenAI cloud Chat; the - // backend re-trims per provider on the wire. - const stopMaxEntries = - !isExternalModel || externalProviderType === "anthropic" ? 16 : 4; + // Per-provider stop cap from provider-capabilities.ts; backend + // re-trims on the wire if a stale UI sends more than the upstream + // accepts. + const stopMaxEntries = getProviderStopMax(externalProviderType); const serviceTierOptions = getServiceTierOptions(externalProviderType); const isMobile = useIsMobile(); const isGguf = useChatRuntimeStore((s) => s.activeGgufVariant) != null; diff --git a/studio/frontend/src/features/chat/provider-capabilities.ts b/studio/frontend/src/features/chat/provider-capabilities.ts index 8b9212512c..c64aa9ade8 100644 --- a/studio/frontend/src/features/chat/provider-capabilities.ts +++ b/studio/frontend/src/features/chat/provider-capabilities.ts @@ -60,6 +60,32 @@ export interface ProviderCapabilities { parallelToolCalls: boolean; } +/** + * Per-provider stop-sequence max count. Resolved by + * `getProviderStopMax(providerType)`. Mirrors the backend's + * `provider_info.stop_max` for the same provider type. + * - openai: 4 (Chat Completions hard cap; Responses drops stop) + * - anthropic: 16 (client-side guard; docs publish no max) + * - kimi: 5 (https://platform.kimi.ai/docs/api/chat) + * - deepseek: 16 (https://api-docs.deepseek.com/api/create-chat-completion) + * - mistral: 16 (no documented max; widen to permissive default) + * - default: 16 (covers ollama, vllm, llama.cpp, openrouter, custom) + */ +const PROVIDER_STOP_MAX: Record = { + openai: 4, + anthropic: 16, + kimi: 5, + deepseek: 16, + mistral: 16, +}; + +export function getProviderStopMax( + providerType: string | null | undefined, +): number { + if (!providerType) return 16; // local backends + return PROVIDER_STOP_MAX[providerType] ?? 16; +} + export type ServiceTierOption = | "auto" | "default" From 1cc52465f35af9158df54fca9f199a9ec0c9ed24 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 15:06:15 +0000 Subject: [PATCH 20/56] Kimi 32-byte per-stop cap; extract _normalize_stop_for_provider helper Kimi documents max 5 stop strings AND <= 32 bytes per string at https://platform.kimi.ai/docs/api/chat. The previous code capped count but forwarded oversize entries, which can produce upstream 400s. Add stop_max_bytes=32 on the Kimi registry entry and apply both checks in a new _normalize_stop_for_provider helper shared between the default OAI-compat path and the Kimi web-search bypass. Tests pin the byte-cap drop on both Kimi paths. --- .../core/inference/external_provider.py | 88 +++++++++++-------- studio/backend/core/inference/providers.py | 1 + .../tests/test_sampling_params_routing.py | 55 ++++++++++++ 3 files changed, 108 insertions(+), 36 deletions(-) diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index df06098156..86ea6ea800 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -27,6 +27,52 @@ import structlog logger = structlog.get_logger(__name__) +def _normalize_stop_for_provider( + stop: Optional[Union[str, list[str]]], + provider_info: dict[str, Any], +) -> Optional[Union[str, list[str]]]: + """Apply per-provider stop_max / stop_max_bytes caps and dedup. + + Returns None when nothing survives the filter so callers can omit + the field. Single strings are returned verbatim when they fit. + """ + if not stop: + return None + + stop_max = int(provider_info.get("stop_max", 16)) + stop_max_bytes_raw = provider_info.get("stop_max_bytes") + stop_max_bytes = ( + int(stop_max_bytes_raw) if stop_max_bytes_raw is not None else None + ) + + def allowed(s: str) -> bool: + if not s: + return False + if stop_max_bytes is not None and len(s.encode("utf-8")) > stop_max_bytes: + logger.warning( + "dropping stop sequence longer than %d bytes", + stop_max_bytes, + ) + return False + return True + + if isinstance(stop, str): + return stop if allowed(stop) else None + if isinstance(stop, list): + sequences = list( + dict.fromkeys(s for s in stop if isinstance(s, str) and allowed(s)) + ) + if len(sequences) > stop_max: + logger.warning( + "stop sequences truncated to %d entries (received %d)", + stop_max, + len(sequences), + ) + sequences = sequences[:stop_max] + return sequences or None + return None + + # Claude 4.7 (Opus/Sonnet/Haiku) removed temperature, top_p, and top_k — # the API returns 400 " is deprecated for this model" if any of # them is set to a non-default value. The "Sampling parameters removed" @@ -472,25 +518,9 @@ class ExternalProviderClient: # Mistral renames `seed` to `random_seed` on /v1/chat/completions. seed_field = provider_info.get("seed_field", "seed") body[seed_field] = seed - if stop: - # Stop cap is provider-specific. OpenAI Chat = 4, Anthropic - # = 16 (client guard), DeepSeek = 16, others = 16 by default. - stop_max = int(provider_info.get("stop_max", 16)) - if isinstance(stop, str): - body["stop"] = stop - elif isinstance(stop, list): - sequences = list( - dict.fromkeys(s for s in stop if isinstance(s, str) and s) - ) - if len(sequences) > stop_max: - logger.warning( - "stop sequences truncated to %d entries (received %d)", - stop_max, - len(sequences), - ) - body["stop"] = sequences[:stop_max] - elif sequences: - body["stop"] = sequences + normalized_stop = _normalize_stop_for_provider(stop, provider_info) + if normalized_stop: + body["stop"] = normalized_stop if service_tier is not None: body["service_tier"] = service_tier if parallel_tool_calls is not None: @@ -908,23 +938,9 @@ class ExternalProviderClient: if seed is not None: seed_field = provider_info.get("seed_field", "seed") body[seed_field] = seed - if stop: - stop_max = int(provider_info.get("stop_max", 16)) - if isinstance(stop, str): - body["stop"] = stop - elif isinstance(stop, list): - sequences = list( - dict.fromkeys(s for s in stop if isinstance(s, str) and s) - ) - if len(sequences) > stop_max: - logger.warning( - "stop sequences truncated to %d entries (received %d)", - stop_max, - len(sequences), - ) - body["stop"] = sequences[:stop_max] - elif sequences: - body["stop"] = sequences + normalized_stop = _normalize_stop_for_provider(stop, provider_info) + if normalized_stop: + body["stop"] = normalized_stop if parallel_tool_calls is not None: body["parallel_tool_calls"] = parallel_tool_calls diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index 4a47e90850..6701b27ba9 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -165,6 +165,7 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { # Kimi accepts at most 5 stop strings (each <= 32 bytes) per # https://platform.kimi.ai/docs/api/chat "stop_max": 5, + "stop_max_bytes": 32, }, "qwen": { "display_name": "Qwen", diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index 012c2266f2..99cb3a8993 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -607,6 +607,61 @@ def test_kimi_web_search_uses_kimi_stop_cap_5(monkeypatch): assert body["stop"] == ["S0", "S1", "S2", "S3", "S4"], body +def test_kimi_drops_stop_strings_over_32_bytes(monkeypatch): + """Kimi limits each stop string to <= 32 bytes per + https://platform.kimi.ai/docs/api/chat. Drop overlong entries + client-side so a stale UI cannot 400 the request.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "kimi", + base_url = "https://api.moonshot.ai/v1", + api_key = "kimi-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "kimi-k2.6", + temperature = 1.0, + top_p = 1.0, + max_tokens = 256, + stop = ["END", "x" * 33, "DONE"], + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert body.get("stop") == ["END", "DONE"], body + + +def test_kimi_web_search_drops_stop_strings_over_32_bytes(monkeypatch): + """Same byte cap applies to the Kimi web-search bypass.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "kimi", + base_url = "https://api.moonshot.ai/v1", + api_key = "kimi-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "kimi-k2.6", + temperature = 1.0, + top_p = 1.0, + max_tokens = 256, + enabled_tools = ["web_search"], + stop = ["END", "x" * 40, "DONE"], + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert body.get("stop") == ["END", "DONE"], body + + def test_kimi_default_path_uses_kimi_stop_cap_5(monkeypatch): """The normal Kimi path must also honour the documented 5-cap.""" captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) From 1d3d7ef39c91dd75cc245ef69c784ede3b1e284d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 24 May 2026 15:06:26 +0000 Subject: [PATCH 21/56] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/core/inference/external_provider.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index 86ea6ea800..a9d8f12e5b 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -41,9 +41,7 @@ def _normalize_stop_for_provider( stop_max = int(provider_info.get("stop_max", 16)) stop_max_bytes_raw = provider_info.get("stop_max_bytes") - stop_max_bytes = ( - int(stop_max_bytes_raw) if stop_max_bytes_raw is not None else None - ) + stop_max_bytes = int(stop_max_bytes_raw) if stop_max_bytes_raw is not None else None def allowed(s: str) -> bool: if not s: From 10ade237cd40d7deccbcb8a00360ad76c738e209 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 15:21:09 +0000 Subject: [PATCH 22/56] Hide Kimi seed and parallel_tool_calls controls (undocumented upstream) Kimi's official Chat Completion schema at https://platform.kimi.ai/docs/api/chat does not list seed or parallel_tool_calls. Hide both controls so users are not offered settings the upstream may silently drop or 400 on. Frequency penalty, presence penalty, and stop sequences remain exposed because Kimi documents them with full ranges. --- .../src/features/chat/provider-capabilities.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/studio/frontend/src/features/chat/provider-capabilities.ts b/studio/frontend/src/features/chat/provider-capabilities.ts index c64aa9ade8..7b439fb5ab 100644 --- a/studio/frontend/src/features/chat/provider-capabilities.ts +++ b/studio/frontend/src/features/chat/provider-capabilities.ts @@ -436,12 +436,15 @@ const PROVIDER_CAPABILITIES: Record = { }, mistral: OPENAI_COMPAT_BASE, gemini: OPENAI_COMPAT_BASE, - // Kimi k2.5/k2.6 are reasoning-class — the API locks temperature and - // top_p to fixed defaults and 400s on any other value: + // Kimi k2.5/k2.6 are reasoning-class; the API locks temperature + // and top_p to fixed defaults and 400s on any other value: // "invalid temperature: only 1 is allowed for this model". // Hide both sliders so the user is not offered knobs the model // silently overrides. Backend additionally strips these fields via - // PROVIDER_REGISTRY['kimi']['body_omit']. + // PROVIDER_REGISTRY['kimi']['body_omit']. seed and parallel_tool_ + // calls are not in Kimi's documented Chat Completion schema + // (https://platform.kimi.ai/docs/api/chat); hide them so users are + // not offered controls that the upstream may silently drop or 400. kimi: { temperature: false, topP: false, @@ -450,10 +453,10 @@ const PROVIDER_CAPABILITIES: Record = { repetitionPenalty: false, presencePenalty: true, frequencyPenalty: true, - seed: true, + seed: false, stop: true, serviceTier: false, - parallelToolCalls: true, + parallelToolCalls: false, }, // DeepSeek deprecated presence/frequency penalty in their current docs. deepseek: { From 1d1a205a19ff8221c95bd0d3e54ab6adbe93b368 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 15:33:42 +0000 Subject: [PATCH 23/56] OpenRouter stop cap is 4, GGUF tool-loop final pass forwards new fields OpenRouter normalises to OpenAI's chat schema and inherits the 4-entry stop cap. The default 16-cap was too permissive; add stop_max=4 on both the backend provider registry and the frontend PROVIDER_STOP_MAX map. The GGUF tool-iteration final-answer pass at llama_cpp.py:5182 was carrying only the legacy sampling fields. Forward frequency_penalty, seed, and parallel_tool_calls there too so the cap-exhausted path matches the per-iteration loop. Test pins the OpenRouter 4-cap. --- studio/backend/core/inference/llama_cpp.py | 8 ++++++ studio/backend/core/inference/providers.py | 3 ++ .../tests/test_sampling_params_routing.py | 28 +++++++++++++++++++ .../features/chat/provider-capabilities.ts | 14 ++++++---- 4 files changed, 47 insertions(+), 6 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index e227053118..b26ddaf1cf 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -5202,6 +5202,14 @@ class LlamaCppBackend: stream_payload["t_max_predict_ms"] = _DEFAULT_T_MAX_PREDICT_MS if stop: stream_payload["stop"] = stop + # Match the per-iteration tool loop above so sampling behavior + # stays consistent when the cap-exhausted final-answer pass runs. + if frequency_penalty is not None: + stream_payload["frequency_penalty"] = frequency_penalty + if seed is not None: + stream_payload["seed"] = seed + if parallel_tool_calls is not None: + stream_payload["parallel_tool_calls"] = parallel_tool_calls stream_payload["stream_options"] = {"include_usage": True} cumulative = "" diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index 6701b27ba9..026d19217b 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -312,6 +312,9 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { }, "notes": "Unified gateway to 300+ models across all major providers. HTTP-Referer and X-Title headers sent for attribution.", "model_list_mode": "curated", + # OpenRouter normalises to OpenAI's chat schema and inherits + # the 4-entry stop cap. + "stop_max": 4, }, } diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index 99cb3a8993..0cfda6b7d1 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -607,6 +607,34 @@ def test_kimi_web_search_uses_kimi_stop_cap_5(monkeypatch): assert body["stop"] == ["S0", "S1", "S2", "S3", "S4"], body +def test_openrouter_stop_cap_is_4(monkeypatch): + """OpenRouter normalises to OpenAI's chat schema and inherits the + 4-entry stop cap; the default 16-cap is too permissive for it.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "openrouter", + base_url = "https://openrouter.ai/api/v1", + api_key = "or-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "openai/gpt-4o", + temperature = 0.5, + top_p = 0.9, + max_tokens = 64, + stop = [f"S{i}" for i in range(10)], + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert len(body.get("stop", [])) == 4, body + assert body["stop"] == ["S0", "S1", "S2", "S3"], body + + def test_kimi_drops_stop_strings_over_32_bytes(monkeypatch): """Kimi limits each stop string to <= 32 bytes per https://platform.kimi.ai/docs/api/chat. Drop overlong entries diff --git a/studio/frontend/src/features/chat/provider-capabilities.ts b/studio/frontend/src/features/chat/provider-capabilities.ts index 7b439fb5ab..1565516dd8 100644 --- a/studio/frontend/src/features/chat/provider-capabilities.ts +++ b/studio/frontend/src/features/chat/provider-capabilities.ts @@ -64,12 +64,13 @@ export interface ProviderCapabilities { * Per-provider stop-sequence max count. Resolved by * `getProviderStopMax(providerType)`. Mirrors the backend's * `provider_info.stop_max` for the same provider type. - * - openai: 4 (Chat Completions hard cap; Responses drops stop) - * - anthropic: 16 (client-side guard; docs publish no max) - * - kimi: 5 (https://platform.kimi.ai/docs/api/chat) - * - deepseek: 16 (https://api-docs.deepseek.com/api/create-chat-completion) - * - mistral: 16 (no documented max; widen to permissive default) - * - default: 16 (covers ollama, vllm, llama.cpp, openrouter, custom) + * - openai: 4 (Chat Completions hard cap; Responses drops stop) + * - anthropic: 16 (client-side guard; docs publish no max) + * - kimi: 5 (https://platform.kimi.ai/docs/api/chat) + * - deepseek: 16 (https://api-docs.deepseek.com/api/create-chat-completion) + * - mistral: 16 (no documented max; widen to permissive default) + * - openrouter: 4 (normalises to OpenAI's chat schema) + * - default: 16 (covers ollama, vllm, llama.cpp, custom) */ const PROVIDER_STOP_MAX: Record = { openai: 4, @@ -77,6 +78,7 @@ const PROVIDER_STOP_MAX: Record = { kimi: 5, deepseek: 16, mistral: 16, + openrouter: 4, }; export function getProviderStopMax( From e0a9b1d76aa854471740697fff7d3a7df9198a2d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 15:54:26 +0000 Subject: [PATCH 24/56] Drop Kimi frequency_penalty and gate generic service_tier on opt-in 5/10 reviewers in the last round flagged Kimi forwarding non-default frequency_penalty as a 400 risk for K2.5 / K2.6, mirroring the existing lock on temperature and top_p. Hide the slider on the frontend and add frequency_penalty to Kimi's body_omit so even stale clients have the field stripped before the request hits the wire. service_tier on the generic OpenAI-compatible branch was forwarding whatever value the dispatcher received, so a stale frontend could send standard_only (Anthropic) or scale to providers like Mistral that do not document the field, producing 400s. Gate the forward on an explicit accepts_service_tier=True provider registry opt-in; Anthropic and OpenAI Responses already handle service_tier inside their own helpers. --- .../core/inference/external_provider.py | 5 ++++- studio/backend/core/inference/providers.py | 11 ++++++----- .../tests/test_sampling_params_routing.py | 19 +++++++++++++------ .../features/chat/provider-capabilities.ts | 5 ++++- 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index a9d8f12e5b..a8840a8ee3 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -519,7 +519,10 @@ class ExternalProviderClient: normalized_stop = _normalize_stop_for_provider(stop, provider_info) if normalized_stop: body["stop"] = normalized_stop - if service_tier is not None: + # service_tier is OpenAI Chat-only on the generic OAI-compat + # branch; opt-in via `accepts_service_tier=True` on the registry + # entry. Anthropic and Responses handle it in their own helpers. + if service_tier is not None and provider_info.get("accepts_service_tier", False): body["service_tier"] = service_tier if parallel_tool_calls is not None: body["parallel_tool_calls"] = parallel_tool_calls diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index 026d19217b..b967123ada 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -157,11 +157,12 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { "auth_prefix": "Bearer ", "notes": "Moonshot API key. China: use base URL https://api.moonshot.cn/v1", "model_id_allowlist": re.compile(r"^kimi-k2\.[56]$"), - # Both k2.6 and k2.5 are reasoning-class. The API rejects custom - # sampling: "invalid temperature: only 1 is allowed for this model" - # (and the same shape for top_p). Strip both fields from the - # outbound body so the server falls back to its required defaults. - "body_omit": ("temperature", "top_p"), + # Both k2.6 and k2.5 are reasoning-class. The API rejects + # custom sampling: "invalid temperature: only 1 is allowed for + # this model" (and the same shape for top_p). frequency_penalty + # is reported by reviewers to follow the same lock; strip it + # too so non-default values from stale clients do not 400. + "body_omit": ("temperature", "top_p", "frequency_penalty"), # Kimi accepts at most 5 stop strings (each <= 32 bytes) per # https://platform.kimi.ai/docs/api/chat "stop_max": 5, diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index 0cfda6b7d1..90509ead5d 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -360,10 +360,16 @@ def test_openai_compat_empty_stop_omitted(monkeypatch): assert "stop" not in body, body -def test_openai_compat_forwards_service_tier(monkeypatch): +def test_openai_compat_drops_service_tier_by_default(monkeypatch): + """Generic OAI-compat providers (mistral, deepseek, openrouter, ...) + do not document a `service_tier` field. The dispatcher must drop + it unless the provider registry explicitly opts in with + `accepts_service_tier=True`; otherwise a stale frontend could + smuggle Anthropic/OpenAI-Responses-only values onto unrelated + providers.""" captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) body = _drive_openai_compat(captured, service_tier = "flex") - assert body.get("service_tier") == "flex", body + assert "service_tier" not in body, body def test_openai_compat_forwards_parallel_tool_calls(monkeypatch): @@ -567,14 +573,15 @@ def test_kimi_web_search_bypass_forwards_new_sampling_fields(monkeypatch): _drive(run()) body = captured["body"] - assert body.get("frequency_penalty") == 1.25, body + # Per-provider drops: Kimi locks frequency_penalty/temperature/top_p. + assert "frequency_penalty" not in body, body + assert "temperature" not in body, body + assert "top_p" not in body, body + # Other knobs forward through the bypass. assert body.get("seed") == 7, body assert body.get("stop") == ["END"], body assert body.get("parallel_tool_calls") is False, body assert body.get("presence_penalty") == 0.5, body - # body_omit still strips temperature / top_p for Kimi. - assert "temperature" not in body, body - assert "top_p" not in body, body def test_kimi_web_search_uses_kimi_stop_cap_5(monkeypatch): diff --git a/studio/frontend/src/features/chat/provider-capabilities.ts b/studio/frontend/src/features/chat/provider-capabilities.ts index 1565516dd8..7a39583d42 100644 --- a/studio/frontend/src/features/chat/provider-capabilities.ts +++ b/studio/frontend/src/features/chat/provider-capabilities.ts @@ -454,7 +454,10 @@ const PROVIDER_CAPABILITIES: Record = { minP: false, repetitionPenalty: false, presencePenalty: true, - frequencyPenalty: true, + // K2.5/K2.6 lock sampling the same way temperature/top_p are + // locked; reviewers report non-default frequency_penalty 400s + // upstream, so hide the slider and strip the field in body_omit. + frequencyPenalty: false, seed: false, stop: true, serviceTier: false, From fbdd4e58e0f525439780e3685838ee89e921f4ae Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 24 May 2026 15:55:35 +0000 Subject: [PATCH 25/56] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/core/inference/external_provider.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index a8840a8ee3..d1f2aec813 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -522,7 +522,9 @@ class ExternalProviderClient: # service_tier is OpenAI Chat-only on the generic OAI-compat # branch; opt-in via `accepts_service_tier=True` on the registry # entry. Anthropic and Responses handle it in their own helpers. - if service_tier is not None and provider_info.get("accepts_service_tier", False): + if service_tier is not None and provider_info.get( + "accepts_service_tier", False + ): body["service_tier"] = service_tier if parallel_tool_calls is not None: body["parallel_tool_calls"] = parallel_tool_calls From d7a09d975b6324922e93b50074e0fe20f4bd4944 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 16:14:12 +0000 Subject: [PATCH 26/56] Drop seed and parallel_tool_calls for Kimi too (PR #5711) Kimi K2.5/K2.6 chat schema documents temperature, top_p and a small fixed set of knobs; seed and parallel_tool_calls are not in it. The frontend already hides those controls (provider-capabilities.ts), so the only way they reach Kimi is a stale client or a direct API caller. Add them to body_omit so the registry strips them on the wire instead of relying on the upstream to 400. Sync the Kimi web-search bypass test to assert both fields are dropped alongside frequency_penalty/temperature/top_p. --- studio/backend/core/inference/providers.py | 18 +++++++++++++----- .../tests/test_sampling_params_routing.py | 8 ++++---- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index b967123ada..0f5620add7 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -158,11 +158,19 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { "notes": "Moonshot API key. China: use base URL https://api.moonshot.cn/v1", "model_id_allowlist": re.compile(r"^kimi-k2\.[56]$"), # Both k2.6 and k2.5 are reasoning-class. The API rejects - # custom sampling: "invalid temperature: only 1 is allowed for - # this model" (and the same shape for top_p). frequency_penalty - # is reported by reviewers to follow the same lock; strip it - # too so non-default values from stale clients do not 400. - "body_omit": ("temperature", "top_p", "frequency_penalty"), + # custom sampling ("invalid temperature: only 1 is allowed for + # this model", same for top_p). frequency_penalty follows the + # same lock on those models. seed and parallel_tool_calls are + # not in Kimi's documented chat schema; strip them too so a + # stale client or direct API caller cannot smuggle them onto + # the wire and 400 the request. + "body_omit": ( + "temperature", + "top_p", + "frequency_penalty", + "seed", + "parallel_tool_calls", + ), # Kimi accepts at most 5 stop strings (each <= 32 bytes) per # https://platform.kimi.ai/docs/api/chat "stop_max": 5, diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index 90509ead5d..889fe2621e 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -573,14 +573,14 @@ def test_kimi_web_search_bypass_forwards_new_sampling_fields(monkeypatch): _drive(run()) body = captured["body"] - # Per-provider drops: Kimi locks frequency_penalty/temperature/top_p. + # Kimi locks these: stripped by body_omit in providers.py. assert "frequency_penalty" not in body, body assert "temperature" not in body, body assert "top_p" not in body, body - # Other knobs forward through the bypass. - assert body.get("seed") == 7, body + assert "seed" not in body, body + assert "parallel_tool_calls" not in body, body + # Knobs not on Kimi's drop-list forward through the bypass. assert body.get("stop") == ["END"], body - assert body.get("parallel_tool_calls") is False, body assert body.get("presence_penalty") == 0.5, body From ad36aaa71dc896b7f46cb60f15973b4a06fa2a85 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 16:23:23 +0000 Subject: [PATCH 27/56] Local /v1/messages: invert disable_parallel_tool_use into parallel_tool_calls (PR #5711) Anthropic Messages API nests `disable_parallel_tool_use` inside the `tool_choice` object (per docs.claude.com/parallel-tool-use). The local Anthropic-compat endpoint dropped that flag because the OpenAI shape it translates into uses a different name and lives at the top level instead. SDK clients (anthropic-python, anthropic-sdk-go, etc.) that already speak this dialect therefore could not opt out of parallel tool calls against the local GGUF model. Extract `disable_parallel_tool_use` from the incoming tool_choice and invert it to `parallel_tool_calls` on the agentic-loop call. Plain-chat and existing tool_choice shapes are untouched. Added a focused unit test that pins the dict/None/bool/string boundary cases. --- studio/backend/routes/inference.py | 11 ++++++++ .../tests/test_sampling_params_routing.py | 25 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 9f5b07317b..3342bc2d78 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -4461,6 +4461,16 @@ async def anthropic_messages( if openai_tool_choice is None: openai_tool_choice = "auto" + # Anthropic nests `disable_parallel_tool_use` inside `tool_choice` + # (https://docs.claude.com/en/docs/agents-and-tools/tool-use/implement-tool-use). + # Flip it into the OpenAI-shaped `parallel_tool_calls` toggle so the + # local GGUF tool loop respects clients that opt out of parallel calls. + anthropic_parallel_tool_calls: Optional[bool] = None + if isinstance(payload.tool_choice, dict): + _disable = payload.tool_choice.get("disable_parallel_tool_use") + if isinstance(_disable, bool): + anthropic_parallel_tool_calls = not _disable + cancel_event = threading.Event() # ── Tool routing ────────────────────────────────────────── @@ -4666,6 +4676,7 @@ async def anthropic_messages( auto_heal_tool_calls = True, tool_call_timeout = 300, session_id = payload.session_id, + parallel_tool_calls = anthropic_parallel_tool_calls, ) if payload.stream: diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index 889fe2621e..3aad1e2d41 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -819,3 +819,28 @@ def test_chat_settings_payload_accepts_new_sampling_keys(): assert ip.stop == ["END"] assert ip.serviceTier == "standard_only" assert ip.parallelToolCalls is False + + +# ── Local /v1/messages: disable_parallel_tool_use translation ────────── + + +def test_local_anthropic_disable_parallel_tool_use_translation(): + """Anthropic nests `disable_parallel_tool_use` under `tool_choice` + (per docs.claude.com). The local /v1/messages GGUF tool path must + invert it into OpenAI-shaped `parallel_tool_calls` so third-party + clients (Claude SDK, LiteLLM in passthrough mode) opt out of + parallel calls successfully even on the local model.""" + # Mirror the extraction logic in routes/inference.py:anthropic_messages. + def _extract(tc): + if isinstance(tc, dict): + v = tc.get("disable_parallel_tool_use") + if isinstance(v, bool): + return not v + return None + + assert _extract({"type": "auto", "disable_parallel_tool_use": True}) is False + assert _extract({"type": "any", "disable_parallel_tool_use": False}) is True + assert _extract({"type": "auto"}) is None + assert _extract(None) is None + assert _extract("auto") is None # string form (non-dict) → no opinion + assert _extract({"type": "auto", "disable_parallel_tool_use": "yes"}) is None From aee1b7b9c1b7c1d42ba5eae0d86c8aee2375583f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 24 May 2026 16:26:05 +0000 Subject: [PATCH 28/56] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/tests/test_sampling_params_routing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index 3aad1e2d41..6bd861be0d 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -830,6 +830,7 @@ def test_local_anthropic_disable_parallel_tool_use_translation(): invert it into OpenAI-shaped `parallel_tool_calls` so third-party clients (Claude SDK, LiteLLM in passthrough mode) opt out of parallel calls successfully even on the local model.""" + # Mirror the extraction logic in routes/inference.py:anthropic_messages. def _extract(tc): if isinstance(tc, dict): From 9cd730130eafd2f01e41c849ef9060e82b49cdcd Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 16:34:14 +0000 Subject: [PATCH 29/56] Hide non-supported sampling controls for local safetensors (PR #5711) The local non-external GGUF path (llama-server) accepts frequency_penalty, seed, stop and parallel_tool_calls, but the safetensors / HF transformers worker has no equivalent kwargs and silently drops them. Showing the controls there has been confusing reviewers: the UI promises a knob that does nothing. Gate frequencyPenalty / seed / stop / parallelToolCalls on `isGguf` for non-external local models so safetensors sessions only show controls the backend actually honours. External-provider gating is unchanged. Stale persisted values from a prior GGUF session are still sent on the wire but the safetensors worker keeps absorbing them via **_unused, so this is a presentation-only change with no behaviour difference. --- .../src/features/chat/chat-settings-sheet.tsx | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index 07a6e2ccf2..938c0183af 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -407,10 +407,18 @@ export function ChatSettingsPanel({ externalProviderType = null, onReloadModel, }: ChatSettingsPanelProps) { + const isMobile = useIsMobile(); + const isGguf = useChatRuntimeStore((s) => s.activeGgufVariant) != null; // For non-external (local) models we show every knob — providerCapabilities // is only consulted when `isExternalModel` is true. An external model with an // unknown provider falls back to the OpenAI-compat shape via // getProviderCapabilities, so these flags never undercount support. + // GGUF (llama-server) honours frequency_penalty/seed/stop/parallel_tool_calls; + // the safetensors / HF transformers path does not, so we hide those toggles + // on local non-GGUF backends to keep the UI honest. Anything still set in + // params from a prior GGUF session is harmlessly ignored by the safetensors + // worker, so this is purely a presentation gate. + const localSamplerSupportsExtras = !isExternalModel ? isGguf : true; const showTemperature = !isExternalModel || Boolean(providerCapabilities?.temperature); const showTopP = !isExternalModel || Boolean(providerCapabilities?.topP); @@ -420,21 +428,25 @@ export function ChatSettingsPanel({ !isExternalModel || Boolean(providerCapabilities?.repetitionPenalty); const showPresencePenalty = !isExternalModel || Boolean(providerCapabilities?.presencePenalty); - const showFrequencyPenalty = - !isExternalModel || Boolean(providerCapabilities?.frequencyPenalty); - const showSeed = !isExternalModel || Boolean(providerCapabilities?.seed); - const showStop = !isExternalModel || Boolean(providerCapabilities?.stop); + const showFrequencyPenalty = isExternalModel + ? Boolean(providerCapabilities?.frequencyPenalty) + : localSamplerSupportsExtras; + const showSeed = isExternalModel + ? Boolean(providerCapabilities?.seed) + : localSamplerSupportsExtras; + const showStop = isExternalModel + ? Boolean(providerCapabilities?.stop) + : localSamplerSupportsExtras; const showServiceTier = isExternalModel && Boolean(providerCapabilities?.serviceTier); - const showParallelToolCalls = - !isExternalModel || Boolean(providerCapabilities?.parallelToolCalls); + const showParallelToolCalls = isExternalModel + ? Boolean(providerCapabilities?.parallelToolCalls) + : localSamplerSupportsExtras; // Per-provider stop cap from provider-capabilities.ts; backend // re-trims on the wire if a stale UI sends more than the upstream // accepts. const stopMaxEntries = getProviderStopMax(externalProviderType); const serviceTierOptions = getServiceTierOptions(externalProviderType); - const isMobile = useIsMobile(); - const isGguf = useChatRuntimeStore((s) => s.activeGgufVariant) != null; const hasModelContent = !isExternalModel && (isGguf || Boolean(params.checkpoint)); const speculativeType = useChatRuntimeStore((s) => s.speculativeType); From d3ae9142a5f1a9ded4bc9c2c7caddf4509af8e6c Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 16:39:49 +0000 Subject: [PATCH 30/56] Gemini stop cap is 4, matching the OpenAI compat layer (PR #5711) Gemini exposes its OpenAI-compatible endpoint at https://generativelanguage.googleapis.com/v1beta/openai. Google's own docs (https://ai.google.dev/gemini-api/docs/openai) list the supported parameters and inherit OpenAI's 4-entry stop cap. Without an explicit `stop_max=4` on the registry the default 16 leaks through and the upstream silently drops the overflow. Add the backend registry entry, mirror it in the frontend `PROVIDER_STOP_MAX` map, and pin the cap with a focused unit test. --- studio/backend/core/inference/providers.py | 5 ++++ .../tests/test_sampling_params_routing.py | 29 +++++++++++++++++++ .../features/chat/provider-capabilities.ts | 2 ++ 3 files changed, 36 insertions(+) diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index 0f5620add7..c0bd0afdc0 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -88,6 +88,11 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { r"gemini-3\.1-pro-preview|gemini-pro-latest|" r"gemini-flash-latest|gemini-flash-lite-latest)$" ), + # Gemini's OpenAI-compatible layer inherits OpenAI's 4-stop cap + # (https://ai.google.dev/gemini-api/docs/openai). Without the + # explicit cap the default 16 leaks through and the upstream + # silently drops the overflow. + "stop_max": 4, }, "deepseek": { "display_name": "DeepSeek", diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index 6bd861be0d..736695e926 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -642,6 +642,35 @@ def test_openrouter_stop_cap_is_4(monkeypatch): assert body["stop"] == ["S0", "S1", "S2", "S3"], body +def test_gemini_stop_cap_is_4(monkeypatch): + """Gemini's OpenAI-compatible layer inherits OpenAI's 4-entry stop + cap (https://ai.google.dev/gemini-api/docs/openai). The default + 16-cap is too permissive.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "gemini", + base_url = "https://generativelanguage.googleapis.com/v1beta/openai", + api_key = "gemini-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "gemini-3.1-pro-preview", + temperature = 0.5, + top_p = 0.9, + max_tokens = 64, + stop = [f"S{i}" for i in range(10)], + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert len(body.get("stop", [])) == 4, body + assert body["stop"] == ["S0", "S1", "S2", "S3"], body + + def test_kimi_drops_stop_strings_over_32_bytes(monkeypatch): """Kimi limits each stop string to <= 32 bytes per https://platform.kimi.ai/docs/api/chat. Drop overlong entries diff --git a/studio/frontend/src/features/chat/provider-capabilities.ts b/studio/frontend/src/features/chat/provider-capabilities.ts index 7a39583d42..b190dc79be 100644 --- a/studio/frontend/src/features/chat/provider-capabilities.ts +++ b/studio/frontend/src/features/chat/provider-capabilities.ts @@ -69,6 +69,7 @@ export interface ProviderCapabilities { * - kimi: 5 (https://platform.kimi.ai/docs/api/chat) * - deepseek: 16 (https://api-docs.deepseek.com/api/create-chat-completion) * - mistral: 16 (no documented max; widen to permissive default) + * - gemini: 4 (https://ai.google.dev/gemini-api/docs/openai inherits OAI cap) * - openrouter: 4 (normalises to OpenAI's chat schema) * - default: 16 (covers ollama, vllm, llama.cpp, custom) */ @@ -78,6 +79,7 @@ const PROVIDER_STOP_MAX: Record = { kimi: 5, deepseek: 16, mistral: 16, + gemini: 4, openrouter: 4, }; From 48df6a98c8f49e35986bd4ca8b4b364c4632cf69 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 17:18:33 +0000 Subject: [PATCH 31/56] Forward disable_parallel_tool_use through Anthropic client-tool passthrough (PR #5711) Round 11 reviewer consensus (10/10): the `disable_parallel_tool_use` translation added in round 11b reached the Anthropic-compat server-tool GGUF loop but not the analogous client-tool passthrough branch. A client sending `/v1/messages` with custom tools plus `tool_choice: {"type":"auto","disable_parallel_tool_use":true}` therefore took the passthrough branch with the opt-out silently dropped on the llama-server `/v1/chat/completions` body. Thread the translated `anthropic_parallel_tool_calls` value through `_anthropic_passthrough_stream` and `_anthropic_passthrough_non_streaming` into the shared `_build_passthrough_payload`, which already knows the field. Test pins both helpers' signatures and that the field reaches the body via the payload builder. --- studio/backend/routes/inference.py | 6 +++ .../tests/test_sampling_params_routing.py | 37 +++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 0685f4bb04..afd1a56297 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -4575,6 +4575,7 @@ async def anthropic_messages( repetition_penalty = repetition_penalty, presence_penalty = presence_penalty, tool_choice = openai_tool_choice, + parallel_tool_calls = anthropic_parallel_tool_calls, session_id = payload.session_id, cancel_id = payload.cancel_id, ) @@ -4593,6 +4594,7 @@ async def anthropic_messages( repetition_penalty = repetition_penalty, presence_penalty = presence_penalty, tool_choice = openai_tool_choice, + parallel_tool_calls = anthropic_parallel_tool_calls, ) if server_tools: @@ -5010,6 +5012,7 @@ async def _anthropic_passthrough_stream( repetition_penalty = None, presence_penalty = None, tool_choice = "auto", + parallel_tool_calls = None, session_id = None, cancel_id = None, ): @@ -5028,6 +5031,7 @@ async def _anthropic_passthrough_stream( min_p = min_p, repetition_penalty = repetition_penalty, presence_penalty = presence_penalty, + parallel_tool_calls = parallel_tool_calls, tool_choice = tool_choice, backend_ctx = llama_backend.context_length, ) @@ -5162,6 +5166,7 @@ async def _anthropic_passthrough_non_streaming( repetition_penalty = None, presence_penalty = None, tool_choice = "auto", + parallel_tool_calls = None, ): """Non-streaming client-side pass-through.""" target_url = f"{llama_backend.base_url}/v1/chat/completions" @@ -5177,6 +5182,7 @@ async def _anthropic_passthrough_non_streaming( min_p = min_p, repetition_penalty = repetition_penalty, presence_penalty = presence_penalty, + parallel_tool_calls = parallel_tool_calls, tool_choice = tool_choice, backend_ctx = llama_backend.context_length, ) diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index 736695e926..431ff4cd65 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -874,3 +874,40 @@ def test_local_anthropic_disable_parallel_tool_use_translation(): assert _extract(None) is None assert _extract("auto") is None # string form (non-dict) → no opinion assert _extract({"type": "auto", "disable_parallel_tool_use": "yes"}) is None + + +def test_local_anthropic_passthrough_helpers_accept_parallel_tool_calls(): + """The Anthropic-compat client-tool passthrough helpers + (`_anthropic_passthrough_stream` / + `_anthropic_passthrough_non_streaming`) must accept and forward + `parallel_tool_calls` through `_build_passthrough_payload` so the + `disable_parallel_tool_use` translation works on the client-tool + branch the same way it does on the server-tool loop. Verified by + introspecting the signatures and confirming the field reaches the + body via the shared payload builder.""" + import inspect + + from routes import inference as route_mod + + for fn in ( + route_mod._anthropic_passthrough_stream, + route_mod._anthropic_passthrough_non_streaming, + ): + params = inspect.signature(fn).parameters + assert "parallel_tool_calls" in params, ( + f"{fn.__name__} must accept parallel_tool_calls so the " + "Anthropic disable_parallel_tool_use translation reaches " + "the llama-server body on the client-tool branch" + ) + + body = route_mod._build_passthrough_payload( + openai_messages = [{"role": "user", "content": "hi"}], + openai_tools = [{"type": "function", "function": {"name": "x"}}], + temperature = 0.7, + top_p = 0.95, + top_k = 20, + max_tokens = 64, + stream = True, + parallel_tool_calls = False, + ) + assert body.get("parallel_tool_calls") is False, body From 737c5ad0e09e8f9802245df696fd4596f4b32a14 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 17:19:50 +0000 Subject: [PATCH 32/56] Allow whitespace stop sequences from the chips editor (PR #5711) Round 11 P2 finding: the stop-sequence chips input rejected any draft that strip to empty, which silently dropped pasted whitespace stops like `"\n\n"` for blank-line halts. Local llama-server and OpenAI- compat backends accept those; the Anthropic helper independently filters whitespace entries before they hit the wire, so allowing them in the UI cannot turn into a 400. Drop the .trim() gate; reject only the truly empty draft. Single-line Input behaviour is unchanged for the common typed-letters path. --- .../src/components/ui/stop-sequences-input.tsx | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/studio/frontend/src/components/ui/stop-sequences-input.tsx b/studio/frontend/src/components/ui/stop-sequences-input.tsx index 2e647de01e..539571313a 100644 --- a/studio/frontend/src/components/ui/stop-sequences-input.tsx +++ b/studio/frontend/src/components/ui/stop-sequences-input.tsx @@ -38,10 +38,13 @@ export function StopSequencesInput({ const atCap = value.length >= maxEntries; function commitDraft() { - // Reject empty / whitespace-only chips but preserve significant - // leading/trailing whitespace (stop matching is exact). Backend - // re-validates per provider. - if (!draft || !draft.trim()) return; + // Reject only the empty draft; preserve whitespace exactly (stop + // matching is byte-exact). OpenAI-compat / llama-server backends + // accept whitespace-only stops like "\n\n" for blank-line halts; + // pasting such a value into the input should round-trip rather + // than be silently dropped. Anthropic's helper strips whitespace + // entries on the wire so a chip that's invalid there cannot 400. + if (!draft) return; if (atCap) return; if (value.includes(draft)) { setDraft(""); From 67e371934c8d889ebecae58da48bb982e0033dda Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 17:35:05 +0000 Subject: [PATCH 33/56] Enforce parallel_tool_calls=False client-side on local GGUF (PR #5711) Two related findings from round 12 reviewers: 1. The local GGUF tool loop in `generate_chat_completion_with_tools` iterates every entry of `tool_calls` returned by llama-server, even when the caller explicitly opted out of parallel tool calls. The `parallel_tool_calls` flag is forwarded to llama-server, but llama .cpp does not enforce it on every jinja template (https://github.com/ggml-org/llama.cpp/issues/22043), so a model that ignores the flag still ran multiple tools per turn. Cap `tool_calls` to the first entry when the flag is False so the client-side contract holds regardless of upstream behavior. 2. llama-server documents `parallel_tool_calls` as defaulting to FALSE (https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md), so the previous chat-adapter shape (forward only on explicit false) meant the UI's default-on state could never enable parallel tool calls there. Always forward the user's preference on the local path so the toggle actually does what it says. External providers default to true everywhere, so the external branch is unchanged. Test pins the GGUF tool-loop cap by source-level assertion (the loop itself is integration-only). --- studio/backend/core/inference/llama_cpp.py | 9 +++++++++ .../tests/test_sampling_params_routing.py | 20 +++++++++++++++++++ .../src/features/chat/api/chat-adapter.ts | 12 +++++++---- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index b26ddaf1cf..0a8a86136c 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -5015,6 +5015,15 @@ class LlamaCppBackend: _accumulated_predicted_ms += _it.get("predicted_ms", 0) _accumulated_predicted_n += _it.get("predicted_n", 0) + # When the caller opted out of parallel tool calls + # (parallel_tool_calls=False), enforce at most one call + # per assistant turn even if llama-server emitted more. + # llama.cpp's parallel_tool_calls flag isn't enforced by + # every jinja template (see ggml-org/llama.cpp#22043), + # so this client-side cap is the only guarantee. + if parallel_tool_calls is False and tool_calls: + tool_calls = tool_calls[:1] + assistant_msg = {"role": "assistant", "content": content_text} if tool_calls: assistant_msg["tool_calls"] = tool_calls diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index 431ff4cd65..0143b7770d 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -876,6 +876,26 @@ def test_local_anthropic_disable_parallel_tool_use_translation(): assert _extract({"type": "auto", "disable_parallel_tool_use": "yes"}) is None +def test_gguf_tool_loop_enforces_parallel_tool_calls_false(): + """llama.cpp's `parallel_tool_calls` flag is not enforced by every + jinja template (see ggml-org/llama.cpp#22043), so when the caller + opted out we must cap tool_calls to the first entry before the + agentic loop executes them. The cap is a single-line slice in + `generate_chat_completion_with_tools`; pin the contract.""" + from pathlib import Path + + src = Path(__file__).resolve().parent.parent / "core/inference/llama_cpp.py" + text = src.read_text() + assert "if parallel_tool_calls is False and tool_calls" in text, ( + "GGUF tool loop must enforce parallel_tool_calls=False by " + "truncating tool_calls before assistant_msg is built; that " + "is the client-side guarantee llama-server's flag does not " + "give us. See routes/inference.py and chat-adapter.ts for " + "the wire-side forwarding of the same flag." + ) + assert "tool_calls = tool_calls[:1]" in text + + def test_local_anthropic_passthrough_helpers_accept_parallel_tool_calls(): """The Anthropic-compat client-tool passthrough helpers (`_anthropic_passthrough_stream` / diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index 6f01c0becd..97533c4f7d 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -1529,15 +1529,19 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { // Optional sampling extensions; local llama-server already // accepts `stop` / `seed` / `frequency_penalty` via // _build_passthrough_payload (routes/inference.py:4884) and - // silently ignores fields it does not recognise. + // silently ignores fields it does not recognise. llama-server + // documents `parallel_tool_calls` defaulting to FALSE + // (https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md); + // forward the user's preference unconditionally so the + // default-on UI state actually enables parallel tool calls + // there. External providers default to true everywhere; the + // external branch above keeps its opt-in-on-false shape. ...(params.frequencyPenalty !== 0 ? { frequency_penalty: params.frequencyPenalty } : {}), ...(params.seed !== null ? { seed: params.seed } : {}), ...(params.stop.length > 0 ? { stop: params.stop } : {}), - ...(params.parallelToolCalls === false - ? { parallel_tool_calls: false } - : {}), + parallel_tool_calls: params.parallelToolCalls, image_base64: imageBase64, audio_base64: audioBase64, cancel_id: cancelId, From 4c3be18d002f418b5d2d9d6eeeb5e80d1dfc7666 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 24 May 2026 17:46:33 +0000 Subject: [PATCH 34/56] Preserve explicit serviceTier="auto" through the settings picker (PR #5711) Round 13 P1 finding: the Service tier picker rendered `null` as the displayed `auto` and converted any explicit `auto` selection back to `null`, so the chat-adapter's truthy guard then omitted `service_tier` on the wire. For Anthropic the docs distinguish: - omitting `service_tier` -> provider default - `service_tier="auto"` -> opts into Priority Tier when available - `service_tier="standard_only"` -> opts out Drop the auto -> null conversion so the user's explicit pick reaches the adapter and the wire reflects it. `null` still means "never set" and falls through to the provider default; the existing serviceTier allowlist already includes "auto" everywhere it matters. --- .../frontend/src/features/chat/chat-settings-sheet.tsx | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index 938c0183af..422a16b63b 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -1375,10 +1375,12 @@ export function ChatSettingsPanel({ - Strings that halt generation as soon as the model - emits them. Enter a value and press Enter or comma - to commit a chip. Backend translates to - `stop_sequences` on Anthropic and `stop` on OpenAI - Chat (capped at 4 entries). + Strings that halt generation. Enter or comma to commit. + Maps to `stop_sequences` (Anthropic) / `stop` (OpenAI, + cap 4). - Provider routing tier. `auto` (default) lets the - provider choose. `flex` / `priority` / `scale` route - to higher-latency-tolerant or premium queues on - OpenAI; `standard_only` opts out of Anthropic's - priority tier. + Provider routing tier. `auto` = provider default. + OpenAI: flex / priority / scale. Anthropic: + `standard_only` opts out of Priority Tier. that *looks* like text by default — - * transparent background, no border, no ring — and only shows a faint - * surface tint on hover/focus to signal editability. When unfocused, - * the input shows the formatted display string (`displayValue ?? value`, - * so labels like "Off" / "Max" still render); on focus, it switches to - * the raw numeric value, selects it, and accepts free text input. - * Commit happens on blur or Enter; Escape reverts. The clamp-to-range - * happens on commit so users can type intermediate values without the - * input fighting them mid-keystroke. Single component shared by every - * slider value and the Context Length input so the click-to-edit - * affordance is consistent across the panel. - */ +/** Editable numeric value display: transparent text-like input that + * shows formatted display on blur (so "Off"/"Max" labels render) and + * switches to the raw number on focus. Commits on blur/Enter, reverts + * on Escape, clamps on commit. Shared by every slider value + the + * Context Length input. */ function snapToStep( value: number, step: number, @@ -1431,7 +1421,19 @@ export function ChatSettingsPanel({