diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index 25e1725337..6565d8e643 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -11,7 +11,7 @@ Anthropic uses native Messages API with translation in this client. import json as _json import re import time -from typing import Any, AsyncGenerator, Literal, NamedTuple, Optional +from typing import Any, AsyncGenerator, Literal, NamedTuple, Optional, Union from urllib.parse import urlparse import httpx @@ -348,6 +348,11 @@ class ExternalProviderClient: anthropic_code_exec_container_id: Optional[str] = None, prompt_cache_ttl: Optional[str] = None, compaction_threshold: Optional[int] = None, + frequency_penalty: Optional[float] = None, + seed: Optional[int] = None, + stop: Optional[Union[str, list[str]]] = None, + service_tier: Optional[str] = None, + parallel_tool_calls: Optional[bool] = None, stream: bool = True, ) -> AsyncGenerator[str, None]: """ @@ -360,6 +365,12 @@ class ExternalProviderClient: supplies a value the provider accepts — the frontend's provider-capability map already filters these per provider, so we treat them as opt-in here. + + ``frequency_penalty``, ``seed``, ``stop``, ``service_tier``, + ``parallel_tool_calls`` follow the same rule: the per-provider + stream helpers silently drop fields the upstream API does not + accept (e.g. Responses rejects all of seed / frequency / stop; + Anthropic does not implement seed / frequency / logprobs). """ if not self._is_openai_compatible(): async for line in self._stream_anthropic( @@ -376,6 +387,9 @@ class ExternalProviderClient: anthropic_code_exec_container_id, prompt_cache_ttl, compaction_threshold, + stop = stop, + service_tier = service_tier, + parallel_tool_calls = parallel_tool_calls, ): yield line return @@ -398,6 +412,8 @@ class ExternalProviderClient: enable_prompt_caching, openai_code_exec_container_id, compaction_threshold, + service_tier = service_tier, + parallel_tool_calls = parallel_tool_calls, ): yield line return @@ -438,6 +454,38 @@ class ExternalProviderClient: else: body["max_tokens"] = max_tokens + # Optional sampling extensions (added in #5XXX). Only forwarded + # when the caller passed a value. Each upstream provider that + # 400s on the field appears in `body_omit` (see providers.py) + # so the registry-driven drop loop below removes them before + # the request hits the wire. The Responses path + # (_stream_openai_responses) drops these explicitly because it + # never reaches this body construction. + if frequency_penalty is not None: + body["frequency_penalty"] = frequency_penalty + if seed is not None: + body["seed"] = seed + if stop is not None: + # OpenAI Chat caps the list at 4 entries. Truncate with a + # warning rather than letting the upstream 400; users editing + # chips in the picker shouldn't get a cryptic API error. + if isinstance(stop, list) and len(stop) > 4: + logger.warning( + "stop sequences truncated to 4 entries " + "(received %d, OpenAI's hard cap is 4)", + len(stop), + ) + body["stop"] = stop[:4] + elif isinstance(stop, list) and len(stop) == 0: + # Empty list = unset; don't ship `stop: []`. + pass + else: + body["stop"] = stop + if service_tier is not None: + body["service_tier"] = service_tier + if parallel_tool_calls is not None: + body["parallel_tool_calls"] = parallel_tool_calls + # Strip body fields a provider's registry entry declares unusable — # reasoning-class models that lock these to fixed defaults (e.g. # Kimi k2.5/k2.6 only accept temperature=1, top_p=1) 400 otherwise. @@ -1186,6 +1234,10 @@ class ExternalProviderClient: anthropic_code_exec_container_id: Optional[str] = None, prompt_cache_ttl: Optional[str] = None, compaction_threshold: Optional[int] = None, + *, + stop: Optional[Union[str, list[str]]] = None, + service_tier: Optional[str] = None, + parallel_tool_calls: Optional[bool] = None, ) -> AsyncGenerator[str, None]: """ Call the Anthropic Messages API and translate its SSE to OpenAI format. @@ -1347,6 +1399,29 @@ class ExternalProviderClient: body["temperature"] = temperature if top_k is not None and top_k > 0 and not sampling_removed: body["top_k"] = top_k + + # Optional sampling extensions. Anthropic has no + # frequency_penalty / seed / logprobs equivalents, so those are + # silently dropped by virtue of not being forwarded from + # stream_chat_completion. The three knobs Anthropic does accept + # land here: + # stop → stop_sequences (renamed) + # service_tier → service_tier (auto|standard_only only) + # parallel_tool_calls → disable_parallel_tool_use (inverted) + if stop is not None: + sequences: list[str] + if isinstance(stop, str): + sequences = [stop] if stop else [] + else: + sequences = [s for s in stop if isinstance(s, str) and s] + if sequences: + body["stop_sequences"] = sequences + if service_tier in ("auto", "standard_only"): + body["service_tier"] = service_tier + if parallel_tool_calls is False: + # Default upstream behavior is parallel-allowed; only + # forward when the user explicitly disabled it. + body["disable_parallel_tool_use"] = True # Anthropic only caches a prefix when at least one cache_control # marker is attached to it — the frontend defaults # enable_prompt_caching to True for Anthropic, so treat `None` the @@ -2558,6 +2633,9 @@ class ExternalProviderClient: enable_prompt_caching: Optional[bool] = None, openai_code_exec_container_id: Optional[str] = None, compaction_threshold: Optional[int] = None, + *, + service_tier: Optional[str] = None, + parallel_tool_calls: Optional[bool] = None, ) -> AsyncGenerator[str, None]: """ Call OpenAI's /v1/responses endpoint and translate its SSE stream back @@ -2665,6 +2743,15 @@ class ExternalProviderClient: "input": input_items, "stream": True, } + # Responses accepts service_tier on the same enum set as Chat + # Completions minus `scale`. parallel_tool_calls follows the + # same shape (default true). The frontend capability gate + # (provider-capabilities.ts) already filters the option lists + # per provider, so we just forward what we got. + if service_tier in ("auto", "default", "flex", "priority"): + body["service_tier"] = service_tier + if parallel_tool_calls is not None: + body["parallel_tool_calls"] = bool(parallel_tool_calls) # `summary: "auto"` is what makes /v1/responses emit reasoning # summary events — without it OpenAI returns no thinking text on # most reasoning models, the SSE handler has no … diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index b5626951c4..0c1c3f814b 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -786,6 +786,48 @@ class ChatCompletionRequest(BaseModel): "to auto-create." ), ) + frequency_penalty: Optional[float] = Field( + None, + ge = -2.0, + le = 2.0, + description = ( + "OpenAI Chat Completions frequency penalty (-2.0 to 2.0). " + "Forwarded only on providers that accept it; Anthropic " + "Messages and the OpenAI Responses family silently drop it." + ), + ) + seed: Optional[int] = Field( + None, + description = ( + "Best-effort determinism seed. Forwarded to OpenAI Chat " + "Completions and OpenAI-compatible local backends. The " + "Responses family rejects it server-side and Anthropic does " + "not implement it, so it is silently dropped on those routes." + ), + ) + service_tier: Optional[ + Literal["auto", "default", "flex", "priority", "scale", "standard_only"] + ] = Field( + None, + description = ( + "Provider service tier. Anthropic accepts only `auto` and " + "`standard_only`; OpenAI Chat accepts " + "`auto|default|flex|priority|scale`; OpenAI Responses accepts " + "`auto|default|flex|priority`. Unsupported values per provider " + "are dropped in the per-provider stream helper instead of " + "422'ing here so a stale frontend never breaks a request." + ), + ) + parallel_tool_calls: Optional[bool] = Field( + None, + description = ( + "Whether the provider may dispatch tool calls in parallel. " + "OpenAI: forwarded as `parallel_tool_calls`. Anthropic: " + "inverted into `disable_parallel_tool_use` on the Messages " + "body. Default `None` preserves each provider's upstream " + "default (which is `true` everywhere today)." + ), + ) @model_validator(mode = "after") def _resolve_missing_tool_call_ids(self) -> "ChatCompletionRequest": diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 02270ab405..b6e4c4e370 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1868,6 +1868,11 @@ async def _proxy_to_external_provider( anthropic_code_exec_container_id = payload.anthropic_code_exec_container_id, prompt_cache_ttl = payload.prompt_cache_ttl, compaction_threshold = payload.compaction_threshold, + frequency_penalty = payload.frequency_penalty, + seed = payload.seed, + stop = payload.stop, + service_tier = payload.service_tier, + parallel_tool_calls = payload.parallel_tool_calls, stream = payload.stream, ) try: diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py new file mode 100644 index 0000000000..f00b1b12f4 --- /dev/null +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -0,0 +1,385 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""End-to-end routing tests for the new sampling parameters. + +Pins the per-provider gating contract added by the +expose-sampling-params PR: each of `frequency_penalty`, `seed`, `stop` +/ `stop_sequences`, `service_tier`, `parallel_tool_calls` only appears +on the outbound body when the upstream provider actually accepts it. + +The provider matrix is captured per docs: +- Anthropic Messages: accepts stop_sequences, service_tier + (auto|standard_only), disable_parallel_tool_use (inverted). REJECTS + frequency_penalty, seed, logprobs (silently dropped client-side). +- OpenAI Chat Completions (default OAI-compat branch): accepts every + field; OpenAI cloud uses `max_completion_tokens` rather than + `max_tokens`. +- OpenAI Responses (gpt-5.x / o3): rejects temperature, top_p, + frequency_penalty, seed, stop, logprobs. Accepts service_tier + (auto|default|flex|priority) and parallel_tool_calls. +""" + +import asyncio +import json + +import httpx +import pytest + +from core.inference import external_provider as ep_mod +from core.inference.external_provider import ExternalProviderClient + + +def _drive(coro): + return asyncio.new_event_loop().run_until_complete(coro) + + +def _install_mock(monkeypatch, *, sse_payload: bytes | None = None) -> dict: + captured: dict = {} + + def handler(request: httpx.Request) -> httpx.Response: + try: + captured["body"] = json.loads(request.content.decode("utf-8")) + except json.JSONDecodeError: + captured["body"] = None + captured["url"] = str(request.url) + captured["headers"] = dict(request.headers) + return httpx.Response( + 200, + content = sse_payload + or (b"event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"), + headers = {"content-type": "text/event-stream"}, + ) + + monkeypatch.setattr( + ep_mod, + "_http_client", + httpx.AsyncClient(transport = httpx.MockTransport(handler)), + ) + return captured + + +# ── Anthropic ────────────────────────────────────────────────────────── + + +def _drive_anthropic(captured, **kwargs) -> dict: + async def run(): + client = ExternalProviderClient( + provider_type = "anthropic", + base_url = "https://api.anthropic.com/v1", + api_key = "sk-ant-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "claude-opus-4-7", + temperature = 0.7, + top_p = 0.95, + max_tokens = 64, + **kwargs, + ): + pass + await client.close() + + _drive(run()) + return captured["body"] + + +def test_anthropic_stop_sequences_forwarded_as_renamed_field(monkeypatch): + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, stop = ["END", "DONE"]) + assert body.get("stop_sequences") == ["END", "DONE"], body + # Anthropic does not have a `stop` field; the unrenamed key must not appear. + assert "stop" not in body, body + + +def test_anthropic_single_string_stop_is_wrapped(monkeypatch): + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, stop = "STOPHERE") + assert body.get("stop_sequences") == ["STOPHERE"], body + + +def test_anthropic_empty_stop_omitted(monkeypatch): + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, stop = []) + assert "stop_sequences" not in body, body + assert "stop" not in body, body + + +def test_anthropic_service_tier_forwarded_when_valid(monkeypatch): + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, service_tier = "standard_only") + assert body.get("service_tier") == "standard_only", body + + +@pytest.mark.parametrize("bogus", ["flex", "priority", "scale", "default", "", "auto-foo"]) +def test_anthropic_service_tier_unsupported_values_dropped(monkeypatch, bogus): + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, service_tier = bogus) + assert "service_tier" not in body, body + + +def test_anthropic_disable_parallel_tool_use_only_when_false(monkeypatch): + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, parallel_tool_calls = False) + assert body.get("disable_parallel_tool_use") is True, body + # Anthropic has no `parallel_tool_calls` field. + assert "parallel_tool_calls" not in body, body + + +def test_anthropic_parallel_tool_calls_default_not_sent(monkeypatch): + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, parallel_tool_calls = True) + # True is the upstream default; do not surface + # `disable_parallel_tool_use: false` which would over-specify the request. + assert "disable_parallel_tool_use" not in body, body + assert "parallel_tool_calls" not in body, body + + +def test_anthropic_rejects_openai_only_knobs(monkeypatch): + """frequency_penalty / seed are dropped at the dispatch layer. + + Anthropic has no equivalent; the keyword args are not even forwarded + from stream_chat_completion to _stream_anthropic. This test pins + that no such field reaches the Messages body. + """ + captured = _install_mock(monkeypatch) + body = _drive_anthropic( + captured, + frequency_penalty = 1.5, + seed = 42, + ) + assert "frequency_penalty" not in body, body + assert "seed" not in body, body + + +# ── OpenAI Chat Completions (default OAI-compat) ───────────────────────── + + +def _drive_openai_compat(captured, **kwargs) -> dict: + """Send through the default OAI-compat branch (NOT /v1/responses). + + Use a non-OpenAI provider_type so the dispatcher takes the default + branch at the bottom of stream_chat_completion rather than the + Responses translator path that routes provider_type=="openai". + """ + + async def run(): + client = ExternalProviderClient( + provider_type = "mistral", + base_url = "https://api.mistral.ai/v1", + api_key = "test-key", + ) + # mistral's OpenAI-compat /v1/chat/completions returns OpenAI + # SSE; a single DONE frame is enough to drain the stream. + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "mistral-small-latest", + temperature = 0.5, + top_p = 0.9, + max_tokens = 64, + **kwargs, + ): + pass + await client.close() + + _drive(run()) + return captured["body"] + + +def _oai_done_payload() -> bytes: + return b"data: [DONE]\n\n" + + +def test_openai_compat_forwards_frequency_penalty(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured, frequency_penalty = 1.25) + assert body.get("frequency_penalty") == 1.25, body + + +def test_openai_compat_forwards_seed(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured, seed = 12345) + assert body.get("seed") == 12345, body + + +def test_openai_compat_forwards_stop_array(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured, stop = ["END", "DONE"]) + assert body.get("stop") == ["END", "DONE"], body + # The default OAI-compat branch does not rename to stop_sequences. + assert "stop_sequences" not in body, body + + +def test_openai_compat_truncates_stop_to_four(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat( + captured, stop = ["a", "b", "c", "d", "e", "f"] + ) + assert body.get("stop") == ["a", "b", "c", "d"], body + + +def test_openai_compat_empty_stop_omitted(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured, stop = []) + assert "stop" not in body, body + + +def test_openai_compat_forwards_service_tier(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured, service_tier = "flex") + assert body.get("service_tier") == "flex", body + + +def test_openai_compat_forwards_parallel_tool_calls(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured, parallel_tool_calls = False) + assert body.get("parallel_tool_calls") is False, body + + +def test_openai_compat_omits_unset_optionals(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured) + # Optional knobs default to None / unset -> never appear. + assert "frequency_penalty" not in body, body + assert "seed" not in body, body + assert "stop" not in body, body + assert "service_tier" not in body, body + assert "parallel_tool_calls" not in body, body + + +# ── OpenAI Responses (gpt-5.x via /v1/responses) ───────────────────────── + + +def _responses_done_payload() -> bytes: + return ( + b"event: response.completed\n" + b"data: {\"type\":\"response.completed\",\"response\":{\"usage\":{}}}\n\n" + ) + + +def _drive_openai_responses(captured, **kwargs) -> dict: + async def run(): + client = ExternalProviderClient( + provider_type = "openai", + base_url = "https://api.openai.com/v1", + api_key = "sk-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "gpt-5.5", + temperature = 1.0, + top_p = 1.0, + max_tokens = 64, + **kwargs, + ): + pass + await client.close() + + _drive(run()) + return captured["body"] + + +def test_openai_responses_drops_temperature_top_p(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) + body = _drive_openai_responses(captured) + assert "temperature" not in body, body + assert "top_p" not in body, body + + +def test_openai_responses_drops_frequency_penalty_seed_stop(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) + body = _drive_openai_responses( + captured, + frequency_penalty = 1.5, + seed = 99, + stop = ["END"], + ) + # Responses 400s on any of these; the dispatch must drop them + # before they hit the wire. + assert "frequency_penalty" not in body, body + assert "seed" not in body, body + assert "stop" not in body, body + assert "stop_sequences" not in body, body + + +def test_openai_responses_forwards_service_tier(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) + body = _drive_openai_responses(captured, service_tier = "priority") + assert body.get("service_tier") == "priority", body + + +def test_openai_responses_rejects_chat_only_service_tier(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) + body = _drive_openai_responses(captured, service_tier = "scale") + # Responses only accepts auto|default|flex|priority -- `scale` is + # silently dropped so a stale frontend cannot 400 the request. + assert "service_tier" not in body, body + + +def test_openai_responses_forwards_parallel_tool_calls(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) + body = _drive_openai_responses(captured, parallel_tool_calls = False) + assert body.get("parallel_tool_calls") is False, body + + +def test_openai_responses_omits_unset_optionals(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) + body = _drive_openai_responses(captured) + assert "service_tier" not in body, body + assert "parallel_tool_calls" not in body, body + + +# ── Schema-level smoke tests ───────────────────────────────────────────── + + +def test_chat_completion_request_accepts_new_sampling_fields(): + from models.inference import ChatCompletionRequest + + payload = ChatCompletionRequest.model_validate( + { + "messages": [{"role": "user", "content": "hi"}], + "frequency_penalty": -1.0, + "seed": 0, + "stop": ["END"], + "service_tier": "auto", + "parallel_tool_calls": True, + } + ) + assert payload.frequency_penalty == -1.0 + assert payload.seed == 0 + assert payload.stop == ["END"] + assert payload.service_tier == "auto" + assert payload.parallel_tool_calls is True + + +def test_chat_completion_request_rejects_bad_service_tier(): + import pydantic + from models.inference import ChatCompletionRequest + + with pytest.raises(pydantic.ValidationError): + ChatCompletionRequest.model_validate( + { + "messages": [{"role": "user", "content": "hi"}], + "service_tier": "bogus", + } + ) + + +def test_chat_completion_request_clamps_frequency_penalty_range(): + import pydantic + from models.inference import ChatCompletionRequest + + with pytest.raises(pydantic.ValidationError): + ChatCompletionRequest.model_validate( + { + "messages": [{"role": "user", "content": "hi"}], + "frequency_penalty": 3.0, + } + ) + with pytest.raises(pydantic.ValidationError): + ChatCompletionRequest.model_validate( + { + "messages": [{"role": "user", "content": "hi"}], + "frequency_penalty": -3.0, + } + ) diff --git a/studio/frontend/src/components/ui/stop-sequences-input.tsx b/studio/frontend/src/components/ui/stop-sequences-input.tsx new file mode 100644 index 0000000000..223c91c16d --- /dev/null +++ b/studio/frontend/src/components/ui/stop-sequences-input.tsx @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { cn } from "@/lib/utils"; +import { XIcon } from "lucide-react"; +import { type KeyboardEvent, useState } from "react"; + +/** + * Chips editor for the `stop` / `stop_sequences` array. + * + * Commit a chip with Enter or comma; press Backspace on an empty input + * to delete the most recent chip. Caps at `maxEntries` -- OpenAI Chat + * Completions documents a hard cap of 4 stop sequences; Anthropic + * Messages accepts arbitrarily many. Pass `Infinity` to disable the cap. + */ +export interface StopSequencesInputProps { + value: string[]; + onChange: (next: string[]) => void; + maxEntries?: number; + disabled?: boolean; + placeholder?: string; + className?: string; + "aria-label"?: string; +} + +export function StopSequencesInput({ + value, + onChange, + maxEntries = 4, + disabled, + placeholder = "Add stop sequence", + className, + "aria-label": ariaLabel, +}: StopSequencesInputProps) { + const [draft, setDraft] = useState(""); + const atCap = value.length >= maxEntries; + + function commitDraft() { + const trimmed = draft.trim(); + if (!trimmed) return; + if (atCap) return; + if (value.includes(trimmed)) { + setDraft(""); + return; + } + onChange([...value, trimmed]); + setDraft(""); + } + + function removeChip(index: number) { + if (disabled) return; + onChange(value.filter((_, i) => i !== index)); + } + + function handleKeyDown(event: KeyboardEvent) { + if (disabled) return; + if (event.key === "Enter" || event.key === ",") { + event.preventDefault(); + commitDraft(); + return; + } + if (event.key === "Backspace" && !draft && value.length > 0) { + event.preventDefault(); + onChange(value.slice(0, -1)); + } + } + + return ( +
+ {value.map((entry, index) => ( + + {entry} + {!disabled ? ( + + ) : null} + + ))} + setDraft(event.target.value)} + onKeyDown={handleKeyDown} + onBlur={commitDraft} + placeholder={atCap ? `Max ${maxEntries} stops` : placeholder} + disabled={disabled || atCap} + className={cn( + "h-6 min-w-[8ch] flex-1 border-0 bg-transparent p-0 text-sm shadow-none", + "focus-visible:ring-0 focus-visible:border-0", + )} + /> +
+ ); +} diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index 0c557f1b01..6f01c0becd 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -1403,6 +1403,29 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { ...(externalCapabilities?.presencePenalty ? { presence_penalty: params.presencePenalty } : {}), + // Optional sampling extensions. Each gate is per-provider + // (see provider-capabilities.ts); the backend additionally + // drops fields the upstream API does not accept, so a + // stale frontend cannot 400 the request. + ...(externalCapabilities?.frequencyPenalty + ? { frequency_penalty: params.frequencyPenalty } + : {}), + ...(externalCapabilities?.seed && params.seed !== null + ? { seed: params.seed } + : {}), + ...(externalCapabilities?.stop && params.stop.length > 0 + ? { stop: params.stop } + : {}), + ...(externalCapabilities?.serviceTier && params.serviceTier + ? { service_tier: params.serviceTier } + : {}), + // Forward parallel_tool_calls when the user explicitly + // turned it off (the upstream default is `true` on every + // provider we ship, so default true is a no-op). + ...(externalCapabilities?.parallelToolCalls && + params.parallelToolCalls === false + ? { parallel_tool_calls: false } + : {}), // Built-in tools: Search pill maps to provider-side // web_search (currently OpenAI / Anthropic / OpenRouter / // Kimi); Code pill maps to Anthropic's server-side @@ -1503,6 +1526,18 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { min_p: params.minP, repetition_penalty: params.repetitionPenalty, presence_penalty: params.presencePenalty, + // Optional sampling extensions; local llama-server already + // accepts `stop` / `seed` / `frequency_penalty` via + // _build_passthrough_payload (routes/inference.py:4884) and + // silently ignores fields it does not recognise. + ...(params.frequencyPenalty !== 0 + ? { frequency_penalty: params.frequencyPenalty } + : {}), + ...(params.seed !== null ? { seed: params.seed } : {}), + ...(params.stop.length > 0 ? { stop: params.stop } : {}), + ...(params.parallelToolCalls === false + ? { parallel_tool_calls: false } + : {}), image_base64: imageBase64, audio_base64: audioBase64, cancel_id: cancelId, diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index 9cd4db2705..5c0c531142 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -86,10 +86,13 @@ import { EXTERNAL_MAX_OUTPUT_TOKENS, type ProviderCapabilities, getExternalMinOutputTokens, + getServiceTierOptions, providerSupportsBuiltinCodeExecution, } from "./provider-capabilities"; +import { StopSequencesInput } from "@/components/ui/stop-sequences-input"; import { useChatRuntimeStore } from "./stores/chat-runtime-store"; -import type { InferenceParams } from "./types/runtime"; +import type { InferenceParams, ServiceTier } from "./types/runtime"; +import { Input } from "@/components/ui/input"; export { defaultInferenceParams, type Preset } from "./presets/preset-policy"; export type { InferenceParams } from "./types/runtime"; @@ -416,6 +419,19 @@ export function ChatSettingsPanel({ !isExternalModel || Boolean(providerCapabilities?.repetitionPenalty); const showPresencePenalty = !isExternalModel || Boolean(providerCapabilities?.presencePenalty); + const showFrequencyPenalty = + !isExternalModel || Boolean(providerCapabilities?.frequencyPenalty); + const showSeed = !isExternalModel || Boolean(providerCapabilities?.seed); + const showStop = !isExternalModel || Boolean(providerCapabilities?.stop); + const showServiceTier = + isExternalModel && Boolean(providerCapabilities?.serviceTier); + const showParallelToolCalls = + !isExternalModel || Boolean(providerCapabilities?.parallelToolCalls); + // OpenAI Chat docs cap `stop` at 4 entries; Anthropic accepts more. + // Pick the right ceiling per active connection so the chips editor's + // placeholder doesn't lie. + const stopMaxEntries = externalProviderType === "anthropic" ? 16 : 4; + const serviceTierOptions = getServiceTierOptions(externalProviderType); const isMobile = useIsMobile(); const isGguf = useChatRuntimeStore((s) => s.activeGgufVariant) != null; const hasModelContent = @@ -1259,6 +1275,145 @@ export function ChatSettingsPanel({ info="Penalizes any token that has already appeared at least once, encouraging the model to introduce new topics. 0 = off." /> ) : null} + {showFrequencyPenalty ? ( + + ) : null} + {showSeed ? ( +
+
+ + Seed + + + Best-effort determinism seed. Same seed + prompt = + same output (approximately). OpenAI Chat Completions + and OpenAI-compat local backends honor it; OpenAI + Responses and Anthropic silently drop it. + +
+ { + const raw = event.target.value; + if (raw === "") { + set("seed")(null); + return; + } + const parsed = Number.parseInt(raw, 10); + if (Number.isFinite(parsed)) { + set("seed")(parsed); + } + }} + className="h-8 w-[124px] shrink-0 text-right font-mono text-xs" + aria-label="Seed" + /> +
+ ) : null} + {showStop ? ( +
+
+ + Stop sequences + + + Strings that halt generation as soon as the model + emits them. Enter a value and press Enter or comma + to commit a chip. Backend translates to + `stop_sequences` on Anthropic and `stop` on OpenAI + Chat (capped at 4 entries). + +
+ +
+ ) : null} + {showServiceTier ? ( +
+
+ + Service tier + + + Provider routing tier. `auto` (default) lets the + provider choose. `flex` / `priority` / `scale` route + to higher-latency-tolerant or premium queues on + OpenAI; `standard_only` opts out of Anthropic's + priority tier. + +
+ +
+ ) : null} + {showParallelToolCalls ? ( +
+
+ + Parallel tool calls + + + When on, the model may dispatch multiple tool calls + in a single turn (default). Turn off to force one + tool call at a time. Anthropic implements this as + `disable_parallel_tool_use`; OpenAI as + `parallel_tool_calls`. + +
+ +
+ ) : null} {!isExternalModel && !isGguf && ( = { @@ -302,6 +368,8 @@ const PROVIDER_CAPABILITIES: Record = { // models served via /v1/responses, which rejects temperature, top_p, and // presence/frequency penalty. See backend // external_provider._stream_openai_responses for the proxy. + // service_tier and parallel_tool_calls are accepted on /v1/responses; + // seed / stop / frequency_penalty are 400'd alongside temperature/top_p. openai: { temperature: false, topP: false, @@ -309,14 +377,22 @@ const PROVIDER_CAPABILITIES: Record = { minP: false, repetitionPenalty: false, presencePenalty: false, + frequencyPenalty: false, + seed: false, + stop: false, + serviceTier: true, + parallelToolCalls: true, }, // Anthropic's Messages API accepts top_k on 3.x and 4.5/4.6, but Claude // 4.7 (Opus/Sonnet/Haiku) deprecated it and returns 400 if it is set. // We surface top_k in the panel for all Anthropic providers and let the // backend strip it per-model — see _stream_anthropic in // studio/backend/core/inference/external_provider.py. - // Presence/frequency penalty is not part of the Messages API on any - // Claude generation. + // Presence/frequency penalty / seed / logprobs are not part of the + // Messages API on any Claude generation. stop_sequences (Anthropic name + // for `stop`), service_tier (auto|standard_only), and + // disable_parallel_tool_use (inverse of parallel_tool_calls) ARE + // supported. anthropic: { temperature: true, topP: true, @@ -324,6 +400,11 @@ const PROVIDER_CAPABILITIES: Record = { minP: false, repetitionPenalty: false, presencePenalty: false, + frequencyPenalty: false, + seed: false, + stop: true, + serviceTier: true, + parallelToolCalls: true, }, mistral: OPENAI_COMPAT_BASE, gemini: OPENAI_COMPAT_BASE, @@ -340,6 +421,11 @@ const PROVIDER_CAPABILITIES: Record = { minP: false, repetitionPenalty: false, presencePenalty: true, + frequencyPenalty: true, + seed: true, + stop: true, + serviceTier: false, + parallelToolCalls: true, }, // DeepSeek deprecated presence/frequency penalty in their current docs. deepseek: { @@ -349,6 +435,11 @@ const PROVIDER_CAPABILITIES: Record = { minP: false, repetitionPenalty: false, presencePenalty: false, + frequencyPenalty: false, + seed: true, + stop: true, + serviceTier: false, + parallelToolCalls: true, }, qwen: OPENAI_COMPAT_BASE, huggingface: OPENAI_COMPAT_BASE, diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts index a00b53a44c..dd26006228 100644 --- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts +++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts @@ -373,6 +373,11 @@ const PERSISTED_INFERENCE_PARAM_KEYS = [ "minP", "repetitionPenalty", "presencePenalty", + "frequencyPenalty", + "seed", + "stop", + "serviceTier", + "parallelToolCalls", "maxSeqLength", "maxTokens", "systemPrompt", diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index 1e6bcf8b87..764e571bfa 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -262,6 +262,41 @@ export interface OpenAIChatCompletionsRequest { * the Anthropic provider with `code_execution` in `enabled_tools`. */ anthropic_code_exec_container_id?: string | null; + /** + * OpenAI Chat Completions only; rejected by the Responses family and + * silently dropped by Anthropic. Range -2.0 .. 2.0. + */ + frequency_penalty?: number; + /** + * Best-effort determinism seed. OpenAI Chat / OpenAI-compat backends + * forward it; Responses + Anthropic drop it server-side. + */ + seed?: number; + /** + * Custom stop sequences. Backend translates to `stop_sequences` for + * Anthropic; OpenAI Chat caps at 4 entries (server-side truncates + * with a warning). Empty arrays are omitted. + */ + stop?: string[]; + /** + * Provider service tier. Anthropic accepts `auto|standard_only`; + * OpenAI Chat accepts `auto|default|flex|priority|scale`; OpenAI + * Responses accepts `auto|default|flex|priority`. + */ + service_tier?: + | "auto" + | "default" + | "flex" + | "priority" + | "scale" + | "standard_only"; + /** + * Whether the provider may dispatch tool calls in parallel. + * OpenAI: forwarded as `parallel_tool_calls`. Anthropic: inverted + * into `disable_parallel_tool_use` server-side. Default `undefined` + * keeps each provider's upstream default. + */ + parallel_tool_calls?: boolean; } export interface OpenAIChatDelta { diff --git a/studio/frontend/src/features/chat/types/runtime.ts b/studio/frontend/src/features/chat/types/runtime.ts index 2967584653..49c7c291e7 100644 --- a/studio/frontend/src/features/chat/types/runtime.ts +++ b/studio/frontend/src/features/chat/types/runtime.ts @@ -1,6 +1,14 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 +export type ServiceTier = + | "auto" + | "default" + | "flex" + | "priority" + | "scale" + | "standard_only"; + export interface InferenceParams { temperature: number; topP: number; @@ -8,6 +16,34 @@ export interface InferenceParams { minP: number; repetitionPenalty: number; presencePenalty: number; + /** OpenAI Chat Completions only; rejected by Responses + Anthropic. */ + frequencyPenalty: number; + /** + * Best-effort determinism seed. OpenAI Chat Completions only; the + * Responses family and Anthropic reject it (silently dropped server-side). + * `null` = unset (no `seed` field on the wire). + */ + seed: number | null; + /** + * Custom stop sequences. Maps to `stop` on OpenAI Chat Completions and + * `stop_sequences` on Anthropic Messages. OpenAI caps the array at 4 + * entries; backend truncates with a warning. Empty array = unset. + */ + stop: string[]; + /** + * Provider service tier. Each provider accepts a different enum set; + * `getServiceTierOptions(providerType)` resolves the legal values. `null` + * means "let the provider pick its default" and is the safe choice on + * provider switch. + */ + serviceTier: ServiceTier | null; + /** + * Whether the provider may dispatch tool calls in parallel. Maps to + * `parallel_tool_calls` on both OpenAI APIs and is inverted into + * `disable_parallel_tool_use` for Anthropic. Default true matches the + * upstream defaults across all three. + */ + parallelToolCalls: boolean; maxSeqLength: number; maxTokens: number; systemPrompt: string; @@ -23,6 +59,11 @@ export const DEFAULT_INFERENCE_PARAMS: InferenceParams = { minP: 0.01, repetitionPenalty: 1.0, presencePenalty: 0.0, + frequencyPenalty: 0.0, + seed: null, + stop: [], + serviceTier: null, + parallelToolCalls: true, maxSeqLength: 4096, maxTokens: 8192, systemPrompt: "",