diff --git a/studio/backend/core/inference/anthropic_compat.py b/studio/backend/core/inference/anthropic_compat.py index bc792c3b99..cb2736a525 100644 --- a/studio/backend/core/inference/anthropic_compat.py +++ b/studio/backend/core/inference/anthropic_compat.py @@ -404,12 +404,18 @@ class AnthropicPassthroughEmitter: streaming response back to Anthropic format without executing anything. """ - def __init__(self) -> None: + def __init__(self, *, parallel_tool_calls: Optional[bool] = None) -> None: self.block_index: int = -1 self._current_block_type: Optional[str] = None # "text" | "tool_use" | None self._tool_call_states: dict = {} # delta index -> {block_index, id, name} self._usage: dict = {} self._stop_reason: str = "end_turn" + # parallel_tool_calls=False (Anthropic + # tool_choice.disable_parallel_tool_use=true): emit only the first + # tool-call index; llama.cpp's flag isn't enforced by every jinja + # template (ggml-org/llama.cpp#22043). + self._serial_tool_calls: bool = parallel_tool_calls is False + self._first_tool_call_idx: Optional[int] = None def start(self, message_id: str, model: str) -> list[str]: return [ @@ -470,6 +476,13 @@ class AnthropicPassthroughEmitter: tool_calls = delta.get("tool_calls") or [] for tc in tool_calls: tc_idx = tc.get("index", 0) + # Serial-tool-call gate: latch on the first index we see + # and silently drop deltas for any other. + if self._serial_tool_calls: + if self._first_tool_call_idx is None: + self._first_tool_call_idx = tc_idx + if tc_idx != self._first_tool_call_idx: + continue fn = tc.get("function") or {} if tc_idx not in self._tool_call_states: # New tool call — close prior block, open tool_use block diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index 8a1edd608b..c71ef9ee1c 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -29,30 +29,62 @@ import structlog logger = structlog.get_logger(__name__) -# Claude 4.7 (Opus/Sonnet/Haiku) removed temperature, top_p, and top_k — -# the API returns 400 " is deprecated for this model" if any of -# them is set to a non-default value. The "Sampling parameters removed" -# section of the 4.7 release notes is the authoritative reference: -# https://platform.claude.com/docs/en/about-claude/models/whats-new-claude-4-7 -# 3.x and 4.5/4.6 still accept all three; match the 4-7 line strictly so -# the knobs keep working on earlier families. The trailing -4-7[-.]/EOL -# anchor keeps future versions (e.g. claude-opus-5) unaffected. +def _normalize_stop_for_provider( + stop: Optional[Union[str, list[str]]], + provider_info: dict[str, Any], +) -> Optional[Union[str, list[str]]]: + """Apply per-provider stop_max / stop_max_bytes caps and dedup. + + Returns None when nothing survives the filter so callers can omit + the field. Single strings are returned verbatim when they fit. + """ + if not stop: + return None + + stop_max = int(provider_info.get("stop_max", 16)) + stop_max_bytes_raw = provider_info.get("stop_max_bytes") + stop_max_bytes = int(stop_max_bytes_raw) if stop_max_bytes_raw is not None else None + + def allowed(s: str) -> bool: + if not s: + return False + if stop_max_bytes is not None and len(s.encode("utf-8")) > stop_max_bytes: + logger.warning( + "dropping stop sequence longer than %d bytes", + stop_max_bytes, + ) + return False + return True + + if isinstance(stop, str): + return stop if allowed(stop) else None + if isinstance(stop, list): + sequences = list( + dict.fromkeys(s for s in stop if isinstance(s, str) and allowed(s)) + ) + if len(sequences) > stop_max: + logger.warning( + "stop sequences truncated to %d entries (received %d)", + stop_max, + len(sequences), + ) + sequences = sequences[:stop_max] + return sequences or None + return None + + +# Opus 4.7 removed temperature/top_p/top_k (400s on any non-default). +# Only Opus shipped in 4.7; 3.x and 4.5/4.6 still accept all three. +# Trailing -4-7[-.]/EOL anchor keeps future families (claude-opus-5 +# etc) unaffected. +# https://platform.claude.com/docs/en/about-claude/models/whats-new-claude-4-7 def _is_openai_family_cloud(base_url: Optional[str]) -> bool: """True iff ``base_url`` points at OpenAI cloud or Azure OpenAI Foundry. - Anchored to the URL host so an attacker can't bypass the gate with a - path or subdomain like ``https://evil.com/api.openai.com/v1`` or - ``https://api.openai.com.attacker.com/v1`` (CodeQL py/incomplete-url- - substring-sanitization). Used to scope cloud-only Responses-API - extensions (prompt_cache_retention, context_management compaction, - container shell tool) that 400 on non-cloud OpenAI-compatible - servers (ollama / llama.cpp / vLLM). - - Azure Foundry resources are scoped to - ``.openai.azure.com``; match any subdomain via an - `endswith` on the lowercased hostname, with the leading dot so - `openai.azure.com` itself doesn't slip through (there is no - apex-hosted Azure Foundry endpoint). + Host-anchored against subdomain-injection (api.openai.com.attacker.com). + Gates Responses-API extensions (prompt_cache_retention, context_management, + container shell) that 400 on non-cloud OAI-compat servers. Azure Foundry + matches via .openai.azure.com suffix; leading dot blocks the apex. """ if not base_url: return False @@ -65,9 +97,7 @@ def _is_openai_family_cloud(base_url: Optional[str]) -> bool: return host == "api.openai.com" or host.endswith(".openai.azure.com") -_ANTHROPIC_4_7_SAMPLING_REMOVED = re.compile( - r"^claude-(?:opus|sonnet|haiku)-4-7(?:[-.]|$)" -) +_ANTHROPIC_4_7_SAMPLING_REMOVED = re.compile(r"^claude-opus-4-7(?:[-.]|$)") _OPENAI_REASONING_SUMMARY_UNSUPPORTED = re.compile(r"^o3(?:[-.]|$)") _OPENAI_REASONING_STATUSES = {"in_progress", "completed", "incomplete"} @@ -866,9 +896,42 @@ class ExternalProviderClient: anthropic_code_exec_container_id: Optional[str] = None, prompt_cache_ttl: Optional[str] = None, compaction_threshold: Optional[int] = None, + frequency_penalty: Optional[float] = None, + seed: Optional[int] = None, + stop: Optional[Union[str, list[str]]] = None, + service_tier: Optional[str] = None, + parallel_tool_calls: Optional[bool] = None, tools: Optional[list[dict[str, Any]]] = None, tool_choice: Optional[Any] = None, fast_mode: Optional[bool] = None, + typical_p: Optional[float] = None, + top_n_sigma: Optional[float] = None, + repeat_last_n: Optional[int] = None, + dynatemp_range: Optional[float] = None, + dynatemp_exponent: Optional[float] = None, + mirostat: Optional[int] = None, + mirostat_tau: Optional[float] = None, + mirostat_eta: Optional[float] = None, + top_a: Optional[float] = None, + dry_multiplier: Optional[float] = None, + dry_base: Optional[float] = None, + dry_allowed_length: Optional[int] = None, + dry_penalty_last_n: Optional[int] = None, + xtc_probability: Optional[float] = None, + xtc_threshold: Optional[float] = None, + min_keep: Optional[int] = None, + ignore_eos: Optional[bool] = None, + min_tokens: Optional[int] = None, + skip_special_tokens: Optional[bool] = None, + spaces_between_special_tokens: Optional[bool] = None, + include_stop_str_in_output: Optional[bool] = None, + truncate_prompt_tokens: Optional[int] = None, + n_keep: Optional[int] = None, + n_probs: Optional[int] = None, + cache_prompt: Optional[bool] = None, + return_tokens: Optional[bool] = None, + timings_per_token: Optional[bool] = None, + post_sampling_probs: Optional[bool] = None, stream: bool = True, ) -> AsyncGenerator[str, None]: """ @@ -877,13 +940,14 @@ class ExternalProviderClient: For OpenAI-compatible providers, lines are forwarded verbatim. For Anthropic, the native Messages API SSE is translated to OpenAI format. - ``top_k`` and ``presence_penalty`` are forwarded only when the caller - supplies a value the provider accepts — the frontend's - provider-capability map already filters these per provider, so we - treat them as opt-in here. + Optional sampling extras (``top_k``, ``presence_penalty``, + ``frequency_penalty``, ``seed``, ``stop``, ``service_tier``, + ``parallel_tool_calls``) are opt-in. Per-provider helpers silently + drop fields the upstream rejects (Responses: seed/freq/stop; + Anthropic: seed/freq/logprobs). - ``fast_mode`` only applies to Anthropic Opus 4.6 / 4.7 (silently - dropped elsewhere); adds the beta header and ``speed: "fast"``. + ``fast_mode``: Anthropic Opus 4.6/4.7 only; adds the beta header + and ``speed: "fast"``. """ # tool_choice="none" hard-disables hosted/builtin tools across # every provider so enabled_tools cannot accidentally bill or leak. @@ -911,6 +975,7 @@ class ExternalProviderClient: reasoning_effort, tools, tool_choice, + stop = stop, ): yield line return @@ -929,6 +994,9 @@ class ExternalProviderClient: prompt_cache_ttl, compaction_threshold, tool_choice, + stop = stop, + service_tier = service_tier, + parallel_tool_calls = parallel_tool_calls, fast_mode = fast_mode, ): yield line @@ -954,6 +1022,8 @@ class ExternalProviderClient: compaction_threshold, tools, tool_choice, + service_tier = service_tier, + parallel_tool_calls = parallel_tool_calls, ): yield line return @@ -978,6 +1048,11 @@ class ExternalProviderClient: messages, model, max_tokens, + frequency_penalty = frequency_penalty, + seed = seed, + stop = stop, + parallel_tool_calls = parallel_tool_calls, + presence_penalty = presence_penalty, ): yield line return @@ -997,14 +1072,99 @@ class ExternalProviderClient: else: body["max_tokens"] = max_tokens - # Drop fields the registry flags as unusable so reasoning-class - # models with fixed defaults (Kimi k2.6 etc) don't 400 on pydantic - # default values that the route layer still fills in. + # Optional sampling extras; `seed_field` renames seed (Mistral), + # `body_omit` strips upstream-rejected fields. from core.inference.providers import get_provider_info provider_info = get_provider_info(self.provider_type) or {} + if frequency_penalty is not None: + body["frequency_penalty"] = frequency_penalty + if seed is not None: + # Mistral renames `seed` to `random_seed` on /v1/chat/completions. + seed_field = provider_info.get("seed_field", "seed") + body[seed_field] = seed + normalized_stop = _normalize_stop_for_provider(stop, provider_info) + if normalized_stop: + body["stop"] = normalized_stop + # service_tier is OAI-Chat-only here (accepts_service_tier registry + # opt-in); Anthropic/Responses branches set it themselves. + if service_tier is not None and provider_info.get( + "accepts_service_tier", False + ): + body["service_tier"] = service_tier + if parallel_tool_calls is not None: + body["parallel_tool_calls"] = parallel_tool_calls + + # Extended OAI-compat samplers (OpenRouter `top_a`, vLLM output + # knobs, llama.cpp samplers on custom proxies). Each is gated `is + # not None` so explicit 0/False reach the wire; `body_omit` below + # strips fields the upstream rejects. + if typical_p is not None: + body["typical_p"] = typical_p + if top_n_sigma is not None: + body["top_n_sigma"] = top_n_sigma + if repeat_last_n is not None: + body["repeat_last_n"] = repeat_last_n + if dynatemp_range is not None: + body["dynatemp_range"] = dynatemp_range + if dynatemp_exponent is not None: + body["dynatemp_exponent"] = dynatemp_exponent + if mirostat is not None: + body["mirostat"] = mirostat + if mirostat_tau is not None: + body["mirostat_tau"] = mirostat_tau + if mirostat_eta is not None: + body["mirostat_eta"] = mirostat_eta + if top_a is not None: + body["top_a"] = top_a + if dry_multiplier is not None: + body["dry_multiplier"] = dry_multiplier + if dry_base is not None: + body["dry_base"] = dry_base + if dry_allowed_length is not None: + body["dry_allowed_length"] = dry_allowed_length + if dry_penalty_last_n is not None: + body["dry_penalty_last_n"] = dry_penalty_last_n + if xtc_probability is not None: + body["xtc_probability"] = xtc_probability + if xtc_threshold is not None: + body["xtc_threshold"] = xtc_threshold + if min_keep is not None: + body["min_keep"] = min_keep + if ignore_eos is not None: + body["ignore_eos"] = ignore_eos + if min_tokens is not None: + body["min_tokens"] = min_tokens + if skip_special_tokens is not None: + body["skip_special_tokens"] = skip_special_tokens + if spaces_between_special_tokens is not None: + body["spaces_between_special_tokens"] = spaces_between_special_tokens + if include_stop_str_in_output is not None: + body["include_stop_str_in_output"] = include_stop_str_in_output + if truncate_prompt_tokens is not None: + body["truncate_prompt_tokens"] = truncate_prompt_tokens + if n_keep is not None: + body["n_keep"] = n_keep + if n_probs is not None: + body["n_probs"] = n_probs + if cache_prompt is not None: + body["cache_prompt"] = cache_prompt + if return_tokens is not None: + body["return_tokens"] = return_tokens + if timings_per_token is not None: + body["timings_per_token"] = timings_per_token + if post_sampling_probs is not None: + body["post_sampling_probs"] = post_sampling_probs + + # Drop body fields the provider's registry entry locks down + # (e.g. Kimi k2.5/k2.6 only accept temperature=1, top_p=1). + # Also pop the renamed seed field so `body_omit=("seed",)` on a + # provider with `seed_field` rename still strips correctly. + _seed_field = provider_info.get("seed_field", "seed") for field in provider_info.get("body_omit", ()): body.pop(field, None) + if field == "seed" and _seed_field != "seed": + body.pop(_seed_field, None) # Kimi thinking is a top-level body field. kimi-k2-thinking is # always on (ignore the toggle); kimi-k2.6 defaults on, can be @@ -1345,6 +1505,12 @@ class ExternalProviderClient: messages: list[dict[str, Any]], model: str, max_tokens: Optional[int], + *, + frequency_penalty: Optional[float] = None, + seed: Optional[int] = None, + stop: Optional[Union[str, list[str]]] = None, + parallel_tool_calls: Optional[bool] = None, + presence_penalty: Optional[float] = None, ) -> AsyncGenerator[str, None]: """ Kimi $web_search round-trip. @@ -1384,11 +1550,25 @@ class ExternalProviderClient: if max_tokens is not None: body["max_tokens"] = max_tokens - # Strip body fields the Kimi registry declares unusable - # (temperature/top_p — see body_omit in providers.py). + # Kimi-with-search returns early before the default OAI-compat + # body build; re-apply provider-aware sampling/stop here. from core.inference.providers import get_provider_info provider_info = get_provider_info(self.provider_type) or {} + if presence_penalty is not None: + body["presence_penalty"] = presence_penalty + if frequency_penalty is not None: + body["frequency_penalty"] = frequency_penalty + if seed is not None: + seed_field = provider_info.get("seed_field", "seed") + body[seed_field] = seed + normalized_stop = _normalize_stop_for_provider(stop, provider_info) + if normalized_stop: + body["stop"] = normalized_stop + if parallel_tool_calls is not None: + body["parallel_tool_calls"] = parallel_tool_calls + + # Drop body fields the provider's registry entry locks down. for field in provider_info.get("body_omit", ()): body.pop(field, None) @@ -1736,6 +1916,9 @@ class ExternalProviderClient: compaction_threshold: Optional[int] = None, tool_choice: Optional[Any] = None, *, + stop: Optional[Union[str, list[str]]] = None, + service_tier: Optional[str] = None, + parallel_tool_calls: Optional[bool] = None, fast_mode: Optional[bool] = None, ) -> AsyncGenerator[str, None]: """ @@ -2037,6 +2220,32 @@ class ExternalProviderClient: body["temperature"] = temperature if top_k is not None and top_k > 0 and not sampling_removed: body["top_k"] = top_k + + # Anthropic body-knob mapping: stop -> stop_sequences (ws-stripped, + # dedup), service_tier (auto|standard_only). parallel_tool_calls is + # handled after tools wiring (Anthropic nests under tool_choice). + if stop: + sequences: list[str] + if isinstance(stop, str): + sequences = [stop] if stop.strip() else [] + else: + # Anthropic rejects whitespace-only stop_sequences ("must + # contain non-whitespace"); 16-cap is a client-side guard + # (undocumented max; every SDK uses 16, Bedrock at 8191). + sequences = list( + dict.fromkeys(s for s in stop if isinstance(s, str) and s.strip()) + ) + if len(sequences) > 16: + logger.warning( + "stop_sequences truncated to 16 entries " + "(received %d, client-side guard ceiling)", + len(sequences), + ) + sequences = sequences[:16] + if sequences: + body["stop_sequences"] = sequences + if service_tier in ("auto", "standard_only"): + body["service_tier"] = service_tier # Anthropic only caches a prefix when at least one cache_control # marker is attached to it — the frontend defaults # enable_prompt_caching to True for Anthropic, so treat `None` the @@ -2232,6 +2441,16 @@ class ExternalProviderClient: if anthropic_code_exec_container_id: body["container"] = anthropic_code_exec_container_id + # parallel_tool_calls=False -> tool_choice.disable_parallel_tool_use=True + # (top-level 400s; no-op without tools). + # https://platform.claude.com/docs/en/agents-and-tools/tool-use/implement-tool-use + if parallel_tool_calls is False and body.get("tools"): + tc = body.get("tool_choice") + if not isinstance(tc, dict): + tc = {"type": "auto"} + tc["disable_parallel_tool_use"] = True + body["tool_choice"] = tc + # Server-side compaction (beta `compact-2026-01-12`). Clamps # below-min thresholds to 50K so the request doesn't 400. # https://platform.claude.com/docs/en/build-with-claude/compaction @@ -3235,6 +3454,8 @@ class ExternalProviderClient: reasoning_effort: Optional[str] = None, tools: Optional[list[dict[str, Any]]] = None, tool_choice: Optional[Any] = None, + *, + stop: Optional[Union[str, list[str]]] = None, ) -> AsyncGenerator[str, None]: """ Call Google's native Gemini API and translate its streaming @@ -3928,6 +4149,14 @@ class ExternalProviderClient: "thinkingBudget": thinking_budget, } + # Gemini's generationConfig.stopSequences (max 5 per native docs). + # https://ai.google.dev/api/generate-content#generationconfig + if stop is not None: + seqs = [stop] if isinstance(stop, str) else list(stop) + seqs = [s for s in seqs if isinstance(s, str) and s][:5] + if seqs: + gen_config["stopSequences"] = seqs + if gen_config: body["generationConfig"] = gen_config @@ -4951,6 +5180,9 @@ class ExternalProviderClient: compaction_threshold: Optional[int] = None, tools: Optional[list[dict[str, Any]]] = None, tool_choice: Optional[Any] = None, + *, + service_tier: Optional[str] = None, + parallel_tool_calls: Optional[bool] = None, ) -> AsyncGenerator[str, None]: """ Call OpenAI's /v1/responses endpoint and translate its SSE stream back @@ -5272,6 +5504,12 @@ class ExternalProviderClient: "input": input_items, "stream": True, } + # Responses: auto|default|flex|priority (SDK type lists "scale" + # but server 400s; Scale Tier stays on Chat Completions). + if service_tier in ("auto", "default", "flex", "priority"): + body["service_tier"] = service_tier + if parallel_tool_calls is not None: + body["parallel_tool_calls"] = bool(parallel_tool_calls) if previous_response_id: body["previous_response_id"] = previous_response_id # `summary: "auto"` is what makes /v1/responses emit reasoning diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 0325620b2d..0b3e426dc1 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -4247,6 +4247,36 @@ class LlamaCppBackend: enable_thinking: Optional[bool] = None, reasoning_effort: Optional[str] = None, preserve_thinking: Optional[bool] = None, + frequency_penalty: Optional[float] = None, + seed: Optional[int] = None, + parallel_tool_calls: Optional[bool] = None, + typical_p: Optional[float] = None, + top_n_sigma: Optional[float] = None, + repeat_last_n: Optional[int] = None, + dynatemp_range: Optional[float] = None, + dynatemp_exponent: Optional[float] = None, + mirostat: Optional[int] = None, + mirostat_tau: Optional[float] = None, + mirostat_eta: Optional[float] = None, + dry_multiplier: Optional[float] = None, + dry_base: Optional[float] = None, + dry_allowed_length: Optional[int] = None, + dry_penalty_last_n: Optional[int] = None, + xtc_probability: Optional[float] = None, + xtc_threshold: Optional[float] = None, + min_keep: Optional[int] = None, + ignore_eos: Optional[bool] = None, + min_tokens: Optional[int] = None, + skip_special_tokens: Optional[bool] = None, + spaces_between_special_tokens: Optional[bool] = None, + include_stop_str_in_output: Optional[bool] = None, + truncate_prompt_tokens: Optional[int] = None, + n_keep: Optional[int] = None, + n_probs: Optional[int] = None, + cache_prompt: Optional[bool] = None, + return_tokens: Optional[bool] = None, + timings_per_token: Optional[bool] = None, + post_sampling_probs: Optional[bool] = None, ) -> Generator[str | dict, None, None]: """ Send a chat completion request to llama-server and stream tokens back. @@ -4286,8 +4316,77 @@ class LlamaCppBackend: else (self._effective_context_length or _DEFAULT_MAX_TOKENS_FLOOR) ) payload["t_max_predict_ms"] = _DEFAULT_T_MAX_PREDICT_MS + # Strip empty/non-string stop entries (mirrors + # `_normalize_stop_for_provider`); stale `stop=["", "END"]` 400s. if stop: - payload["stop"] = stop + if isinstance(stop, str): + payload["stop"] = stop + elif isinstance(stop, list): + _cleaned = [s for s in stop if isinstance(s, str) and s] + if _cleaned: + payload["stop"] = _cleaned + # `is not None` gate so explicit 0/False reach the wire; + # llama-server ignores unknown fields. + if frequency_penalty is not None: + payload["frequency_penalty"] = frequency_penalty + if seed is not None: + payload["seed"] = seed + if parallel_tool_calls is not None: + payload["parallel_tool_calls"] = parallel_tool_calls + if typical_p is not None: + payload["typical_p"] = typical_p + if top_n_sigma is not None: + payload["top_n_sigma"] = top_n_sigma + if repeat_last_n is not None: + payload["repeat_last_n"] = repeat_last_n + if dynatemp_range is not None: + payload["dynatemp_range"] = dynatemp_range + if dynatemp_exponent is not None: + payload["dynatemp_exponent"] = dynatemp_exponent + if mirostat is not None: + payload["mirostat"] = mirostat + if mirostat_tau is not None: + payload["mirostat_tau"] = mirostat_tau + if mirostat_eta is not None: + payload["mirostat_eta"] = mirostat_eta + if dry_multiplier is not None: + payload["dry_multiplier"] = dry_multiplier + if dry_base is not None: + payload["dry_base"] = dry_base + if dry_allowed_length is not None: + payload["dry_allowed_length"] = dry_allowed_length + if dry_penalty_last_n is not None: + payload["dry_penalty_last_n"] = dry_penalty_last_n + if xtc_probability is not None: + payload["xtc_probability"] = xtc_probability + if xtc_threshold is not None: + payload["xtc_threshold"] = xtc_threshold + if min_keep is not None: + payload["min_keep"] = min_keep + if ignore_eos is not None: + payload["ignore_eos"] = ignore_eos + if min_tokens is not None: + payload["min_tokens"] = min_tokens + if skip_special_tokens is not None: + payload["skip_special_tokens"] = skip_special_tokens + if spaces_between_special_tokens is not None: + payload["spaces_between_special_tokens"] = spaces_between_special_tokens + if include_stop_str_in_output is not None: + payload["include_stop_str_in_output"] = include_stop_str_in_output + if truncate_prompt_tokens is not None: + payload["truncate_prompt_tokens"] = truncate_prompt_tokens + if n_keep is not None: + payload["n_keep"] = n_keep + if n_probs is not None: + payload["n_probs"] = n_probs + if cache_prompt is not None: + payload["cache_prompt"] = cache_prompt + if return_tokens is not None: + payload["return_tokens"] = return_tokens + if timings_per_token is not None: + payload["timings_per_token"] = timings_per_token + if post_sampling_probs is not None: + payload["post_sampling_probs"] = post_sampling_probs payload["stream_options"] = {"include_usage": True} url = f"{self.base_url}/v1/chat/completions" @@ -4430,6 +4529,36 @@ class LlamaCppBackend: auto_heal_tool_calls: bool = True, tool_call_timeout: int = 300, session_id: Optional[str] = None, + frequency_penalty: Optional[float] = None, + seed: Optional[int] = None, + parallel_tool_calls: Optional[bool] = None, + typical_p: Optional[float] = None, + top_n_sigma: Optional[float] = None, + repeat_last_n: Optional[int] = None, + dynatemp_range: Optional[float] = None, + dynatemp_exponent: Optional[float] = None, + mirostat: Optional[int] = None, + mirostat_tau: Optional[float] = None, + mirostat_eta: Optional[float] = None, + dry_multiplier: Optional[float] = None, + dry_base: Optional[float] = None, + dry_allowed_length: Optional[int] = None, + dry_penalty_last_n: Optional[int] = None, + xtc_probability: Optional[float] = None, + xtc_threshold: Optional[float] = None, + min_keep: Optional[int] = None, + ignore_eos: Optional[bool] = None, + min_tokens: Optional[int] = None, + skip_special_tokens: Optional[bool] = None, + spaces_between_special_tokens: Optional[bool] = None, + include_stop_str_in_output: Optional[bool] = None, + truncate_prompt_tokens: Optional[int] = None, + n_keep: Optional[int] = None, + n_probs: Optional[int] = None, + cache_prompt: Optional[bool] = None, + return_tokens: Optional[bool] = None, + timings_per_token: Optional[bool] = None, + post_sampling_probs: Optional[bool] = None, ) -> Generator[dict, None, None]: """ Agentic loop: let the model call tools, execute them, and continue. @@ -4512,8 +4641,75 @@ class LlamaCppBackend: else (self._effective_context_length or _DEFAULT_MAX_TOKENS_FLOOR) ) payload["t_max_predict_ms"] = _DEFAULT_T_MAX_PREDICT_MS + # Same empty-string filter as the standard payload builder. if stop: - payload["stop"] = stop + if isinstance(stop, str): + payload["stop"] = stop + elif isinstance(stop, list): + _cleaned = [s for s in stop if isinstance(s, str) and s] + if _cleaned: + payload["stop"] = _cleaned + # Optional sampling extensions; gated on `is not None`. + if frequency_penalty is not None: + payload["frequency_penalty"] = frequency_penalty + if seed is not None: + payload["seed"] = seed + if parallel_tool_calls is not None: + payload["parallel_tool_calls"] = parallel_tool_calls + if typical_p is not None: + payload["typical_p"] = typical_p + if top_n_sigma is not None: + payload["top_n_sigma"] = top_n_sigma + if repeat_last_n is not None: + payload["repeat_last_n"] = repeat_last_n + if dynatemp_range is not None: + payload["dynatemp_range"] = dynatemp_range + if dynatemp_exponent is not None: + payload["dynatemp_exponent"] = dynatemp_exponent + if mirostat is not None: + payload["mirostat"] = mirostat + if mirostat_tau is not None: + payload["mirostat_tau"] = mirostat_tau + if mirostat_eta is not None: + payload["mirostat_eta"] = mirostat_eta + if dry_multiplier is not None: + payload["dry_multiplier"] = dry_multiplier + if dry_base is not None: + payload["dry_base"] = dry_base + if dry_allowed_length is not None: + payload["dry_allowed_length"] = dry_allowed_length + if dry_penalty_last_n is not None: + payload["dry_penalty_last_n"] = dry_penalty_last_n + if xtc_probability is not None: + payload["xtc_probability"] = xtc_probability + if xtc_threshold is not None: + payload["xtc_threshold"] = xtc_threshold + if min_keep is not None: + payload["min_keep"] = min_keep + if ignore_eos is not None: + payload["ignore_eos"] = ignore_eos + if min_tokens is not None: + payload["min_tokens"] = min_tokens + if skip_special_tokens is not None: + payload["skip_special_tokens"] = skip_special_tokens + if spaces_between_special_tokens is not None: + payload["spaces_between_special_tokens"] = spaces_between_special_tokens + if include_stop_str_in_output is not None: + payload["include_stop_str_in_output"] = include_stop_str_in_output + if truncate_prompt_tokens is not None: + payload["truncate_prompt_tokens"] = truncate_prompt_tokens + if n_keep is not None: + payload["n_keep"] = n_keep + if n_probs is not None: + payload["n_probs"] = n_probs + if cache_prompt is not None: + payload["cache_prompt"] = cache_prompt + if return_tokens is not None: + payload["return_tokens"] = return_tokens + if timings_per_token is not None: + payload["timings_per_token"] = timings_per_token + if post_sampling_probs is not None: + payload["post_sampling_probs"] = post_sampling_probs try: _auth_headers = ( @@ -4996,6 +5192,12 @@ class LlamaCppBackend: _accumulated_predicted_ms += _it.get("predicted_ms", 0) _accumulated_predicted_n += _it.get("predicted_n", 0) + # parallel_tool_calls=False: client-side cap to 1 + # (llama.cpp flag isn't enforced by every jinja template, + # ggml-org/llama.cpp#22043). + if parallel_tool_calls is False and tool_calls: + tool_calls = tool_calls[:1] + assistant_msg = {"role": "assistant", "content": content_text} if tool_calls: assistant_msg["tool_calls"] = tool_calls @@ -5198,8 +5400,78 @@ class LlamaCppBackend: else (self._effective_context_length or _DEFAULT_MAX_TOKENS_FLOOR) ) stream_payload["t_max_predict_ms"] = _DEFAULT_T_MAX_PREDICT_MS + # Same empty-string filter as the standard / tool payload builder. if stop: - stream_payload["stop"] = stop + if isinstance(stop, str): + stream_payload["stop"] = stop + elif isinstance(stop, list): + _cleaned = [s for s in stop if isinstance(s, str) and s] + if _cleaned: + stream_payload["stop"] = _cleaned + # Match per-iteration tool-loop sampling for the cap-exhausted + # final-answer pass. + if frequency_penalty is not None: + stream_payload["frequency_penalty"] = frequency_penalty + if seed is not None: + stream_payload["seed"] = seed + if parallel_tool_calls is not None: + stream_payload["parallel_tool_calls"] = parallel_tool_calls + if typical_p is not None: + stream_payload["typical_p"] = typical_p + if top_n_sigma is not None: + stream_payload["top_n_sigma"] = top_n_sigma + if repeat_last_n is not None: + stream_payload["repeat_last_n"] = repeat_last_n + if dynatemp_range is not None: + stream_payload["dynatemp_range"] = dynatemp_range + if dynatemp_exponent is not None: + stream_payload["dynatemp_exponent"] = dynatemp_exponent + if mirostat is not None: + stream_payload["mirostat"] = mirostat + if mirostat_tau is not None: + stream_payload["mirostat_tau"] = mirostat_tau + if mirostat_eta is not None: + stream_payload["mirostat_eta"] = mirostat_eta + if dry_multiplier is not None: + stream_payload["dry_multiplier"] = dry_multiplier + if dry_base is not None: + stream_payload["dry_base"] = dry_base + if dry_allowed_length is not None: + stream_payload["dry_allowed_length"] = dry_allowed_length + if dry_penalty_last_n is not None: + stream_payload["dry_penalty_last_n"] = dry_penalty_last_n + if xtc_probability is not None: + stream_payload["xtc_probability"] = xtc_probability + if xtc_threshold is not None: + stream_payload["xtc_threshold"] = xtc_threshold + if min_keep is not None: + stream_payload["min_keep"] = min_keep + if ignore_eos is not None: + stream_payload["ignore_eos"] = ignore_eos + if min_tokens is not None: + stream_payload["min_tokens"] = min_tokens + if skip_special_tokens is not None: + stream_payload["skip_special_tokens"] = skip_special_tokens + if spaces_between_special_tokens is not None: + stream_payload["spaces_between_special_tokens"] = ( + spaces_between_special_tokens + ) + if include_stop_str_in_output is not None: + stream_payload["include_stop_str_in_output"] = include_stop_str_in_output + if truncate_prompt_tokens is not None: + stream_payload["truncate_prompt_tokens"] = truncate_prompt_tokens + if n_keep is not None: + stream_payload["n_keep"] = n_keep + if n_probs is not None: + stream_payload["n_probs"] = n_probs + if cache_prompt is not None: + stream_payload["cache_prompt"] = cache_prompt + if return_tokens is not None: + stream_payload["return_tokens"] = return_tokens + if timings_per_token is not None: + stream_payload["timings_per_token"] = timings_per_token + if post_sampling_probs is not None: + stream_payload["post_sampling_probs"] = post_sampling_probs stream_payload["stream_options"] = {"include_usage": True} cumulative = "" diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py index 7e7d7026f6..f392fb4ce9 100644 --- a/studio/backend/core/inference/orchestrator.py +++ b/studio/backend/core/inference/orchestrator.py @@ -839,6 +839,7 @@ class InferenceOrchestrator: tool_call_timeout: int = 300, session_id: Optional[str] = None, use_adapter: Optional[Union[bool, str]] = None, + parallel_tool_calls: Optional[bool] = None, **_unused, ): """Run the safetensors agentic tool loop in this (parent) @@ -895,6 +896,7 @@ class InferenceOrchestrator: max_tool_iterations = max_tool_iterations, tool_call_timeout = tool_call_timeout, session_id = session_id, + parallel_tool_calls = parallel_tool_calls, ) def generate_with_adapter_control( diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index 785f1dec3b..46ad4cee4d 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -137,11 +137,20 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { r"gemini-pro-latest|gemini-flash-latest|gemini-flash-lite-latest" r")$" ), + # Gemini OAI-compat inherits OpenAI's 4-stop cap; default 16 + # silently truncates upstream. + # https://ai.google.dev/gemini-api/docs/openai + "stop_max": 4, }, "deepseek": { "display_name": "DeepSeek", "base_url": "https://api.deepseek.com/v1", + # deepseek-chat / deepseek-reasoner retire 2026-07-24; list + # v4-pro / v4-flash alongside for cutover. + # https://api-docs.deepseek.com/updates "default_models": [ + "deepseek-v4-pro", + "deepseek-v4-flash", "deepseek-chat", "deepseek-reasoner", ], @@ -150,7 +159,7 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { "supports_tool_calling": True, "auth_header": "Authorization", "auth_prefix": "Bearer ", - "notes": "OpenAI-compatible API. deepseek-chat = V3, deepseek-reasoner = R1 thinking mode.", + "notes": "OpenAI-compatible API. deepseek-v4-pro / deepseek-v4-flash are the new canonical ids; deepseek-chat / deepseek-reasoner remain as legacy aliases until 2026-07-24.", }, "mistral": { "display_name": "Mistral AI", @@ -180,6 +189,12 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { r"mistral-(?:large|medium|small|tiny)-latest|" r"mistral-vibe-cli-latest)$" ), + # Mistral renames OpenAI's `seed` to `random_seed` on + # /v1/chat/completions. https://docs.mistral.ai/api/endpoint/chat + "seed_field": "random_seed", + # Mistral's docs publish no max but third-party shims cap at 4; + # match OpenAI Chat's cap to avoid silent upstream truncation. + "stop_max": 4, }, "kimi": { "display_name": "Kimi", @@ -203,11 +218,21 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { "auth_prefix": "Bearer ", "notes": "Moonshot API key. China: use base URL https://api.moonshot.cn/v1", "model_id_allowlist": re.compile(r"^kimi-k2\.[56]$"), - # Both k2.6 and k2.5 are reasoning-class. The API rejects custom - # sampling: "invalid temperature: only 1 is allowed for this model" - # (and the same shape for top_p). Strip both fields from the - # outbound body so the server falls back to its required defaults. - "body_omit": ("temperature", "top_p"), + # k2.5/k2.6 are reasoning-class: API locks temperature=1, top_p=1, + # frequency_penalty; presence_penalty / seed / parallel_tool_calls + # are undocumented in the Kimi chat schema. + "body_omit": ( + "temperature", + "top_p", + "frequency_penalty", + "presence_penalty", + "seed", + "parallel_tool_calls", + ), + # Kimi accepts at most 5 stop strings (each <= 32 bytes) per + # https://platform.kimi.ai/docs/api/chat + "stop_max": 5, + "stop_max_bytes": 32, }, "qwen": { "display_name": "Qwen", @@ -354,6 +379,9 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { }, "notes": "Unified gateway to 300+ models across all major providers. HTTP-Referer and X-Title headers sent for attribution.", "model_list_mode": "curated", + # OpenRouter normalises to OpenAI's chat schema and inherits + # the 4-entry stop cap. + "stop_max": 4, }, } diff --git a/studio/backend/core/inference/safetensors_agentic.py b/studio/backend/core/inference/safetensors_agentic.py index 73bb3d090a..b262fafc84 100644 --- a/studio/backend/core/inference/safetensors_agentic.py +++ b/studio/backend/core/inference/safetensors_agentic.py @@ -105,6 +105,7 @@ def run_safetensors_tool_loop( max_tool_iterations: int = 25, tool_call_timeout: int = 300, session_id: Optional[str] = None, + parallel_tool_calls: Optional[bool] = None, ) -> Generator[dict, None, None]: """Drive an agentic tool loop on top of a cumulative-text generator. @@ -293,6 +294,12 @@ def run_safetensors_tool_loop( yield {"type": "status", "text": ""} return + # Mirror the GGUF agentic-loop cap: when the caller opted out + # of parallel tool calls, execute at most one per assistant + # turn even if the model parsed more. + if parallel_tool_calls is False and tool_calls: + tool_calls = tool_calls[:1] + assistant_msg: dict = {"role": "assistant", "content": content_text} if tool_calls: assistant_msg["tool_calls"] = tool_calls diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index bb1bd394d1..e16a074a85 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -863,13 +863,187 @@ class ChatCompletionRequest(BaseModel): "to auto-create." ), ) + frequency_penalty: Optional[float] = Field( + None, + ge = -2.0, + le = 2.0, + description = ( + "OpenAI Chat Completions frequency penalty (-2.0 to 2.0). " + "Forwarded only on providers that accept it; Anthropic " + "Messages and the OpenAI Responses family silently drop it." + ), + ) + seed: Optional[int] = Field( + None, + description = ( + "Best-effort determinism seed. Forwarded to OpenAI Chat and " + "OAI-compat local backends; dropped on Anthropic and OpenAI Responses." + ), + ) + service_tier: Optional[ + Literal["auto", "default", "flex", "priority", "scale", "standard_only"] + ] = Field( + None, + description = ( + "Provider service tier. Anthropic: auto|standard_only. " + "OpenAI Chat: auto|default|flex|priority|scale. " + "OpenAI Responses: auto|default|flex|priority. " + "Unsupported values are dropped per provider rather than 422'd here." + ), + ) + parallel_tool_calls: Optional[bool] = Field( + None, + description = ( + "Allow parallel tool calls. Forwarded as `parallel_tool_calls` " + "on OpenAI; inverted to `disable_parallel_tool_use` on Anthropic. " + "None preserves upstream default (currently true everywhere)." + ), + ) + typical_p: Optional[float] = Field( + None, + ge = 0.0, + le = 1.0, + description = "llama.cpp `typ_p`. 1.0 disables. Local only.", + ) + top_n_sigma: Optional[float] = Field( + None, + description = "llama.cpp `top_n_sigma`. -1 disables. Local only.", + ) + repeat_last_n: Optional[int] = Field( + None, + description = "llama.cpp `repeat_last_n`. 0 disables, -1 = ctx-size. Local only.", + ) + dynatemp_range: Optional[float] = Field( + None, + ge = 0.0, + description = "llama.cpp `dynatemp_range`. 0 disables. Local only.", + ) + dynatemp_exponent: Optional[float] = Field( + None, + ge = 0.0, + description = "llama.cpp `dynatemp_exponent`. Pairs with dynatemp_range. Local only.", + ) + mirostat: Optional[int] = Field( + None, + ge = 0, + le = 2, + description = "llama.cpp `mirostat` (0=off, 1=Mirostat, 2=Mirostat 2.0). Local only.", + ) + mirostat_tau: Optional[float] = Field( + None, + ge = 0.0, + description = "llama.cpp `mirostat_tau`. Local only.", + ) + mirostat_eta: Optional[float] = Field( + None, + ge = 0.0, + description = "llama.cpp `mirostat_eta`. Local only.", + ) + top_a: Optional[float] = Field( + None, + ge = 0.0, + le = 1.0, + description = ( + "OpenRouter `top_a`. OpenRouter-only. " + "https://openrouter.ai/docs/api/reference/parameters" + ), + ) + dry_multiplier: Optional[float] = Field( + None, + ge = 0.0, + description = ( + "llama.cpp DRY multiplier. 0 disables the dry_base / " + "dry_allowed_length / dry_penalty_last_n chain. Local only." + ), + ) + dry_base: Optional[float] = Field( + None, + ge = 1.0, + description = "llama.cpp DRY base. Default 1.75. Local only.", + ) + dry_allowed_length: Optional[int] = Field( + None, + ge = 0, + description = "llama.cpp DRY allowed-length. Default 2. Local only.", + ) + dry_penalty_last_n: Optional[int] = Field( + None, + description = "llama.cpp DRY scan window. 0 disables, -1 = ctx-size. Local only.", + ) + xtc_probability: Optional[float] = Field( + None, + ge = 0.0, + le = 1.0, + description = "llama.cpp XTC probability. 0 disables; pairs with xtc_threshold. Local only.", + ) + xtc_threshold: Optional[float] = Field( + None, + ge = 0.0, + le = 1.0, + description = "llama.cpp XTC threshold. Default 0.1. Local only.", + ) + min_keep: Optional[int] = Field( + None, + ge = 0, + description = "llama.cpp `min_keep` (force min N past every filter). Local only.", + ) + ignore_eos: Optional[bool] = Field( + None, + description = "Continue past EOS. llama.cpp + vLLM only.", + ) + min_tokens: Optional[int] = Field( + None, + ge = 0, + description = "Min output tokens before stop / EOS. llama.cpp + vLLM only.", + ) + skip_special_tokens: Optional[bool] = Field( + None, + description = "vLLM `skip_special_tokens` (default true). vLLM only.", + ) + spaces_between_special_tokens: Optional[bool] = Field( + None, + description = "vLLM `spaces_between_special_tokens` (default true). vLLM only.", + ) + include_stop_str_in_output: Optional[bool] = Field( + None, + description = "vLLM `include_stop_str_in_output`. Useful for agentic tools. vLLM only.", + ) + truncate_prompt_tokens: Optional[int] = Field( + None, + ge = 1, + description = "vLLM `truncate_prompt_tokens` (left-truncate prompt). vLLM only.", + ) + n_keep: Optional[int] = Field( + None, + description = "llama.cpp `n_keep`. 0 disables, -1 = keep all. Local only.", + ) + n_probs: Optional[int] = Field( + None, + ge = 0, + description = "llama.cpp `n_probs` (top-N token probs). 0 disables. Local only.", + ) + cache_prompt: Optional[bool] = Field( + None, + description = "llama.cpp `cache_prompt` (default true upstream). Local only.", + ) + return_tokens: Optional[bool] = Field( + None, + description = "llama.cpp `return_tokens` (debug). Local only.", + ) + timings_per_token: Optional[bool] = Field( + None, + description = "llama.cpp `timings_per_token` (perf debug). Local only.", + ) + post_sampling_probs: Optional[bool] = Field( + None, + description = "llama.cpp `post_sampling_probs` (sampler debug). Local only.", + ) fast_mode: Optional[bool] = Field( None, description = ( - "[x-unsloth] Anthropic fast-mode toggle. On Claude Opus 4.6 / " - "4.7 adds the `fast-mode-2026-02-01` beta header and sends " - "`speed: 'fast'` for higher OTPS at premium pricing. Silently " - "ignored on every other model + provider. See " + "[x-unsloth] Anthropic fast-mode on Opus 4.6 / 4.7. Adds the " + "fast-mode-2026-02-01 beta header + speed:'fast' for higher " + "OTPS at premium pricing. Silently dropped elsewhere. " "https://platform.claude.com/docs/en/build-with-claude/fast-mode" ), ) diff --git a/studio/backend/routes/chat_history.py b/studio/backend/routes/chat_history.py index ed808040d2..4e4faeb1ba 100644 --- a/studio/backend/routes/chat_history.py +++ b/studio/backend/routes/chat_history.py @@ -99,6 +99,9 @@ class ChatExportResponse(BaseModel): class ChatInferenceSettings(BaseModel): + # extra="forbid" requires every persisted key to be listed. Keep + # aligned with PERSISTED_INFERENCE_PARAM_KEYS in + # studio/frontend/src/features/chat/stores/chat-runtime-store.ts. model_config = ConfigDict(extra = "forbid") temperature: Optional[float] = None @@ -107,10 +110,47 @@ class ChatInferenceSettings(BaseModel): minP: Optional[float] = None repetitionPenalty: Optional[float] = None presencePenalty: Optional[float] = None + frequencyPenalty: Optional[float] = Field(default = None, ge = -2.0, le = 2.0) + seed: Optional[int] = None + stop: Optional[list[str]] = None + serviceTier: Optional[ + Literal["auto", "default", "flex", "priority", "scale", "standard_only"] + ] = None + parallelToolCalls: Optional[bool] = None maxSeqLength: Optional[float] = None maxTokens: Optional[float] = None systemPrompt: Optional[str] = None trustRemoteCode: Optional[bool] = None + fastMode: Optional[bool] = None + # Extended llama.cpp / vLLM / OpenRouter samplers exposed by PR #5711. + typicalP: Optional[float] = None + topNSigma: Optional[float] = None + repeatLastN: Optional[int] = None + dynatempRange: Optional[float] = None + dynatempExponent: Optional[float] = None + mirostat: Optional[int] = None + mirostatTau: Optional[float] = None + mirostatEta: Optional[float] = None + topA: Optional[float] = None + dryMultiplier: Optional[float] = None + dryBase: Optional[float] = None + dryAllowedLength: Optional[int] = None + dryPenaltyLastN: Optional[int] = None + xtcProbability: Optional[float] = None + xtcThreshold: Optional[float] = None + minKeep: Optional[int] = None + ignoreEos: Optional[bool] = None + minTokens: Optional[int] = None + skipSpecialTokens: Optional[bool] = None + spacesBetweenSpecialTokens: Optional[bool] = None + includeStopStrInOutput: Optional[bool] = None + truncatePromptTokens: Optional[int] = None + nKeep: Optional[int] = None + nProbs: Optional[int] = None + cachePrompt: Optional[bool] = None + returnTokens: Optional[bool] = None + timingsPerToken: Optional[bool] = None + postSamplingProbs: Optional[bool] = None class ChatPreset(BaseModel): diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 259337616c..481c9239d7 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -239,6 +239,19 @@ router = APIRouter() studio_router = APIRouter() +def _clean_local_stop_list(stop) -> Optional[list[str]]: + """Strip empty/non-string stop entries; returns None when empty so + callers can omit. Mirrors `_normalize_stop_for_provider` so + `stop=["", "END"]` cannot 400 llama-server. + """ + if isinstance(stop, str): + return [stop] if stop else None + if isinstance(stop, list): + cleaned = [s for s in stop if isinstance(s, str) and s] + return cleaned or None + return None + + def _detect_safetensors_features(backend, chat_template: Optional[str]) -> dict: """Classify reasoning/tool capabilities via the GGUF classifier so flags match across backends. gpt-oss is overridden because Harmony @@ -2147,9 +2160,42 @@ async def _proxy_to_external_provider( anthropic_code_exec_container_id = payload.anthropic_code_exec_container_id, prompt_cache_ttl = payload.prompt_cache_ttl, compaction_threshold = payload.compaction_threshold, + frequency_penalty = payload.frequency_penalty, + seed = payload.seed, + stop = payload.stop, + service_tier = payload.service_tier, + parallel_tool_calls = payload.parallel_tool_calls, tools = payload.tools, tool_choice = payload.tool_choice, fast_mode = payload.fast_mode, + typical_p = payload.typical_p, + top_n_sigma = payload.top_n_sigma, + repeat_last_n = payload.repeat_last_n, + dynatemp_range = payload.dynatemp_range, + dynatemp_exponent = payload.dynatemp_exponent, + mirostat = payload.mirostat, + mirostat_tau = payload.mirostat_tau, + mirostat_eta = payload.mirostat_eta, + top_a = payload.top_a, + dry_multiplier = payload.dry_multiplier, + dry_base = payload.dry_base, + dry_allowed_length = payload.dry_allowed_length, + dry_penalty_last_n = payload.dry_penalty_last_n, + xtc_probability = payload.xtc_probability, + xtc_threshold = payload.xtc_threshold, + min_keep = payload.min_keep, + ignore_eos = payload.ignore_eos, + min_tokens = payload.min_tokens, + skip_special_tokens = payload.skip_special_tokens, + spaces_between_special_tokens = payload.spaces_between_special_tokens, + include_stop_str_in_output = payload.include_stop_str_in_output, + truncate_prompt_tokens = payload.truncate_prompt_tokens, + n_keep = payload.n_keep, + n_probs = payload.n_probs, + cache_prompt = payload.cache_prompt, + return_tokens = payload.return_tokens, + timings_per_token = payload.timings_per_token, + post_sampling_probs = payload.post_sampling_probs, stream = payload.stream, ) try: @@ -2776,6 +2822,7 @@ async def openai_chat_completions( max_tokens = payload.max_tokens, repetition_penalty = payload.repetition_penalty, presence_penalty = payload.presence_penalty, + stop = _clean_local_stop_list(payload.stop), cancel_event = cancel_event, enable_thinking = payload.enable_thinking, reasoning_effort = payload.reasoning_effort, @@ -2790,6 +2837,36 @@ async def openai_chat_completions( if payload.tool_call_timeout is not None else 300, session_id = payload.session_id, + frequency_penalty = payload.frequency_penalty, + seed = payload.seed, + parallel_tool_calls = payload.parallel_tool_calls, + typical_p = payload.typical_p, + dry_multiplier = payload.dry_multiplier, + dry_base = payload.dry_base, + dry_allowed_length = payload.dry_allowed_length, + dry_penalty_last_n = payload.dry_penalty_last_n, + xtc_probability = payload.xtc_probability, + xtc_threshold = payload.xtc_threshold, + min_keep = payload.min_keep, + ignore_eos = payload.ignore_eos, + min_tokens = payload.min_tokens, + skip_special_tokens = payload.skip_special_tokens, + spaces_between_special_tokens = payload.spaces_between_special_tokens, + include_stop_str_in_output = payload.include_stop_str_in_output, + truncate_prompt_tokens = payload.truncate_prompt_tokens, + n_keep = payload.n_keep, + n_probs = payload.n_probs, + cache_prompt = payload.cache_prompt, + return_tokens = payload.return_tokens, + timings_per_token = payload.timings_per_token, + post_sampling_probs = payload.post_sampling_probs, + top_n_sigma = payload.top_n_sigma, + repeat_last_n = payload.repeat_last_n, + dynatemp_range = payload.dynatemp_range, + dynatemp_exponent = payload.dynatemp_exponent, + mirostat = payload.mirostat, + mirostat_tau = payload.mirostat_tau, + mirostat_eta = payload.mirostat_eta, ) _tool_sentinel = object() @@ -2955,10 +3032,41 @@ async def openai_chat_completions( max_tokens = payload.max_tokens, repetition_penalty = payload.repetition_penalty, presence_penalty = payload.presence_penalty, + stop = _clean_local_stop_list(payload.stop), cancel_event = cancel_event, enable_thinking = payload.enable_thinking, reasoning_effort = payload.reasoning_effort, preserve_thinking = payload.preserve_thinking, + frequency_penalty = payload.frequency_penalty, + seed = payload.seed, + parallel_tool_calls = payload.parallel_tool_calls, + typical_p = payload.typical_p, + dry_multiplier = payload.dry_multiplier, + dry_base = payload.dry_base, + dry_allowed_length = payload.dry_allowed_length, + dry_penalty_last_n = payload.dry_penalty_last_n, + xtc_probability = payload.xtc_probability, + xtc_threshold = payload.xtc_threshold, + min_keep = payload.min_keep, + ignore_eos = payload.ignore_eos, + min_tokens = payload.min_tokens, + skip_special_tokens = payload.skip_special_tokens, + spaces_between_special_tokens = payload.spaces_between_special_tokens, + include_stop_str_in_output = payload.include_stop_str_in_output, + truncate_prompt_tokens = payload.truncate_prompt_tokens, + n_keep = payload.n_keep, + n_probs = payload.n_probs, + cache_prompt = payload.cache_prompt, + return_tokens = payload.return_tokens, + timings_per_token = payload.timings_per_token, + post_sampling_probs = payload.post_sampling_probs, + top_n_sigma = payload.top_n_sigma, + repeat_last_n = payload.repeat_last_n, + dynatemp_range = payload.dynatemp_range, + dynatemp_exponent = payload.dynatemp_exponent, + mirostat = payload.mirostat, + mirostat_tau = payload.mirostat_tau, + mirostat_eta = payload.mirostat_eta, ) _gguf_sentinel = object() @@ -3298,6 +3406,7 @@ async def openai_chat_completions( else 300, session_id = payload.session_id, use_adapter = payload.use_adapter, + parallel_tool_calls = payload.parallel_tool_calls, ) _sf_tool_sentinel = object() @@ -4101,6 +4210,9 @@ def _build_chat_request( chat_kwargs["top_p"] = payload.top_p if payload.max_output_tokens is not None: chat_kwargs["max_tokens"] = payload.max_output_tokens + # Forward parallel_tool_calls from Responses caller through to llama-server. + if payload.parallel_tool_calls is not None: + chat_kwargs["parallel_tool_calls"] = payload.parallel_tool_calls chat_tools = _translate_responses_tools_to_chat(payload.tools) if chat_tools is not None: @@ -4110,13 +4222,7 @@ def _build_chat_request( if chat_tool_choice is not None: chat_kwargs["tool_choice"] = chat_tool_choice - req = ChatCompletionRequest(**chat_kwargs) - # `parallel_tool_calls` is not a first-class field on ChatCompletionRequest, - # but the model allows extras and _build_openai_passthrough_body forwards - # only explicitly-known fields. Llama-server does not currently implement - # parallel_tool_calls semantics, so we accept-and-ignore it on the - # Responses side to avoid breaking SDK clients that always send it. - return req + return ChatCompletionRequest(**chat_kwargs) def _chat_tool_calls_to_responses_output(tool_calls: list[dict]) -> list[dict]: @@ -4167,6 +4273,10 @@ async def _responses_non_streaming( msg = choices[0].get("message", {}) or {} text = msg.get("content", "") or "" tool_calls = msg.get("tool_calls") or [] + # parallel_tool_calls=False -> cap to 1 (llama.cpp flag isn't enforced; + # mirrors GGUF/Anthropic/safetensors paths). + if payload.parallel_tool_calls is False and tool_calls: + tool_calls = tool_calls[:1] usage_data = body.get("usage", {}) input_tokens = usage_data.get("prompt_tokens", 0) @@ -4288,6 +4398,11 @@ async def _responses_stream( tool_call_state: dict[int, dict] = {} # Text message lives at output_index 0; tool calls claim 1, 2, ... next_output_index = 1 + # parallel_tool_calls=False: latch the first tc index, drop the + # rest; llama.cpp flag isn't enforced by every jinja template + # (ggml-org/llama.cpp#22043). + serial_tool_calls = payload.parallel_tool_calls is False + first_serial_idx: Optional[int] = None def _snapshot_output() -> list[dict]: """Snapshot of all completed output items for response.completed.""" @@ -4404,6 +4519,11 @@ async def _responses_stream( for tc in delta.get("tool_calls") or []: idx = tc.get("index", 0) + if serial_tool_calls: + if first_serial_idx is None: + first_serial_idx = idx + if idx != first_serial_idx: + continue st = tool_call_state.get(idx) fn = tc.get("function") or {} if st is None: @@ -4773,6 +4893,15 @@ async def anthropic_messages( if openai_tool_choice is None: openai_tool_choice = "auto" + # Anthropic nests `disable_parallel_tool_use` under `tool_choice`; + # flip to OAI `parallel_tool_calls` so the local GGUF tool loop honors it. + # https://docs.claude.com/en/docs/agents-and-tools/tool-use/implement-tool-use + anthropic_parallel_tool_calls: Optional[bool] = None + if isinstance(payload.tool_choice, dict): + _disable = payload.tool_choice.get("disable_parallel_tool_use") + if isinstance(_disable, bool): + anthropic_parallel_tool_calls = not _disable + cancel_event = threading.Event() # ── Tool routing ────────────────────────────────────────── @@ -4869,6 +4998,7 @@ async def anthropic_messages( repetition_penalty = repetition_penalty, presence_penalty = presence_penalty, tool_choice = openai_tool_choice, + parallel_tool_calls = anthropic_parallel_tool_calls, session_id = payload.session_id, cancel_id = payload.cancel_id, ) @@ -4887,6 +5017,7 @@ async def anthropic_messages( repetition_penalty = repetition_penalty, presence_penalty = presence_penalty, tool_choice = openai_tool_choice, + parallel_tool_calls = anthropic_parallel_tool_calls, ) if server_tools: @@ -4978,6 +5109,7 @@ async def anthropic_messages( auto_heal_tool_calls = True, tool_call_timeout = 300, session_id = payload.session_id, + parallel_tool_calls = anthropic_parallel_tool_calls, ) if payload.stream: @@ -5229,6 +5361,36 @@ def _build_passthrough_payload( min_p = None, repetition_penalty = None, presence_penalty = None, + frequency_penalty = None, + seed = None, + parallel_tool_calls = None, + typical_p = None, + top_n_sigma = None, + repeat_last_n = None, + dynatemp_range = None, + dynatemp_exponent = None, + mirostat = None, + mirostat_tau = None, + mirostat_eta = None, + dry_multiplier = None, + dry_base = None, + dry_allowed_length = None, + dry_penalty_last_n = None, + xtc_probability = None, + xtc_threshold = None, + min_keep = None, + ignore_eos = None, + min_tokens = None, + skip_special_tokens = None, + spaces_between_special_tokens = None, + include_stop_str_in_output = None, + truncate_prompt_tokens = None, + n_keep = None, + n_probs = None, + cache_prompt = None, + return_tokens = None, + timings_per_token = None, + post_sampling_probs = None, tool_choice = "auto", response_format = None, chat_template_kwargs = None, @@ -5251,8 +5413,16 @@ def _build_passthrough_payload( else (backend_ctx or _DEFAULT_MAX_TOKENS_FLOOR) ) body["t_max_predict_ms"] = _DEFAULT_T_MAX_PREDICT_MS + # Strip empty stop entries (mirrors `_normalize_stop_for_provider`); + # stale `stop=["", "END"]` would 400 llama-server. if stop: - body["stop"] = stop + if isinstance(stop, str): + if stop: + body["stop"] = stop + elif isinstance(stop, list): + cleaned = [s for s in stop if isinstance(s, str) and s] + if cleaned: + body["stop"] = cleaned if min_p is not None: body["min_p"] = min_p if repetition_penalty is not None: @@ -5260,16 +5430,76 @@ def _build_passthrough_payload( body["repeat_penalty"] = repetition_penalty if presence_penalty is not None: body["presence_penalty"] = presence_penalty + # parallel_tool_calls is a no-op on llama-server today but forwarded + # for future support. `is not None` gate lets explicit 0/False through; + # llama-server ignores unknowns, Ollama drops non-OAI fields. + if frequency_penalty is not None: + body["frequency_penalty"] = frequency_penalty + if seed is not None: + body["seed"] = seed + if parallel_tool_calls is not None: + body["parallel_tool_calls"] = parallel_tool_calls + if typical_p is not None: + body["typical_p"] = typical_p + if top_n_sigma is not None: + body["top_n_sigma"] = top_n_sigma + if repeat_last_n is not None: + body["repeat_last_n"] = repeat_last_n + if dynatemp_range is not None: + body["dynatemp_range"] = dynatemp_range + if dynatemp_exponent is not None: + body["dynatemp_exponent"] = dynatemp_exponent + if mirostat is not None: + body["mirostat"] = mirostat + if mirostat_tau is not None: + body["mirostat_tau"] = mirostat_tau + if mirostat_eta is not None: + body["mirostat_eta"] = mirostat_eta + if dry_multiplier is not None: + body["dry_multiplier"] = dry_multiplier + if dry_base is not None: + body["dry_base"] = dry_base + if dry_allowed_length is not None: + body["dry_allowed_length"] = dry_allowed_length + if dry_penalty_last_n is not None: + body["dry_penalty_last_n"] = dry_penalty_last_n + if xtc_probability is not None: + body["xtc_probability"] = xtc_probability + if xtc_threshold is not None: + body["xtc_threshold"] = xtc_threshold + if min_keep is not None: + body["min_keep"] = min_keep + if ignore_eos is not None: + body["ignore_eos"] = ignore_eos + if min_tokens is not None: + body["min_tokens"] = min_tokens + if skip_special_tokens is not None: + body["skip_special_tokens"] = skip_special_tokens + if spaces_between_special_tokens is not None: + body["spaces_between_special_tokens"] = spaces_between_special_tokens + if include_stop_str_in_output is not None: + body["include_stop_str_in_output"] = include_stop_str_in_output + if truncate_prompt_tokens is not None: + body["truncate_prompt_tokens"] = truncate_prompt_tokens + if n_keep is not None: + body["n_keep"] = n_keep + if n_probs is not None: + body["n_probs"] = n_probs + if cache_prompt is not None: + body["cache_prompt"] = cache_prompt + if return_tokens is not None: + body["return_tokens"] = return_tokens + if timings_per_token is not None: + body["timings_per_token"] = timings_per_token + if post_sampling_probs is not None: + body["post_sampling_probs"] = post_sampling_probs if response_format is not None: - # llama-server applies a GBNF grammar derived from the JSON schema - # when response_format is present. Field is documented flat at the - # request root (tools/server/README.md), which is also what the - # OpenAI SDK produces by spreading extra_body into the body top. + # llama-server applies a GBNF grammar from the JSON schema. + # Field is documented flat at the request root. body["response_format"] = response_format if chat_template_kwargs is not None: - # Propagate reasoning / template overrides (e.g. enable_thinking) - # so llama-server renders the Jinja template in the mode the caller - # asked for instead of whatever default the model was loaded with. + # Reasoning / template overrides (e.g. enable_thinking) so + # llama-server renders the Jinja template in the requested mode. body["chat_template_kwargs"] = chat_template_kwargs return body @@ -5291,6 +5521,7 @@ async def _anthropic_passthrough_stream( repetition_penalty = None, presence_penalty = None, tool_choice = "auto", + parallel_tool_calls = None, session_id = None, cancel_id = None, ): @@ -5309,6 +5540,7 @@ async def _anthropic_passthrough_stream( min_p = min_p, repetition_penalty = repetition_penalty, presence_penalty = presence_penalty, + parallel_tool_calls = parallel_tool_calls, tool_choice = tool_choice, backend_ctx = llama_backend.context_length, ) @@ -5319,7 +5551,9 @@ async def _anthropic_passthrough_stream( _tracker.__enter__() async def _stream(): - emitter = AnthropicPassthroughEmitter() + emitter = AnthropicPassthroughEmitter( + parallel_tool_calls = parallel_tool_calls, + ) for line in emitter.start(message_id, model_name): yield line @@ -5443,6 +5677,7 @@ async def _anthropic_passthrough_non_streaming( repetition_penalty = None, presence_penalty = None, tool_choice = "auto", + parallel_tool_calls = None, ): """Non-streaming client-side pass-through.""" target_url = f"{llama_backend.base_url}/v1/chat/completions" @@ -5458,6 +5693,7 @@ async def _anthropic_passthrough_non_streaming( min_p = min_p, repetition_penalty = repetition_penalty, presence_penalty = presence_penalty, + parallel_tool_calls = parallel_tool_calls, tool_choice = tool_choice, backend_ctx = llama_backend.context_length, ) @@ -5484,6 +5720,10 @@ async def _anthropic_passthrough_non_streaming( content_blocks.append(AnthropicResponseTextBlock(text = text)) tool_calls = message.get("tool_calls") or [] + # parallel_tool_calls=False: cap to 1 tool_use block; llama.cpp flag + # isn't enforced by every jinja template (ggml-org/llama.cpp#22043). + if parallel_tool_calls is False and tool_calls: + tool_calls = tool_calls[:1] for tc in tool_calls: fn = tc.get("function") or {} try: @@ -5786,6 +6026,36 @@ def _build_openai_passthrough_body(payload, backend_ctx = None) -> dict: min_p = payload.min_p, repetition_penalty = payload.repetition_penalty, presence_penalty = payload.presence_penalty, + frequency_penalty = payload.frequency_penalty, + seed = payload.seed, + parallel_tool_calls = payload.parallel_tool_calls, + typical_p = payload.typical_p, + top_n_sigma = payload.top_n_sigma, + repeat_last_n = payload.repeat_last_n, + dynatemp_range = payload.dynatemp_range, + dynatemp_exponent = payload.dynatemp_exponent, + mirostat = payload.mirostat, + mirostat_tau = payload.mirostat_tau, + mirostat_eta = payload.mirostat_eta, + dry_multiplier = payload.dry_multiplier, + dry_base = payload.dry_base, + dry_allowed_length = payload.dry_allowed_length, + dry_penalty_last_n = payload.dry_penalty_last_n, + xtc_probability = payload.xtc_probability, + xtc_threshold = payload.xtc_threshold, + min_keep = payload.min_keep, + ignore_eos = payload.ignore_eos, + min_tokens = payload.min_tokens, + skip_special_tokens = payload.skip_special_tokens, + spaces_between_special_tokens = payload.spaces_between_special_tokens, + include_stop_str_in_output = payload.include_stop_str_in_output, + truncate_prompt_tokens = payload.truncate_prompt_tokens, + n_keep = payload.n_keep, + n_probs = payload.n_probs, + cache_prompt = payload.cache_prompt, + return_tokens = payload.return_tokens, + timings_per_token = payload.timings_per_token, + post_sampling_probs = payload.post_sampling_probs, tool_choice = tool_choice, response_format = _extract_response_format(payload), chat_template_kwargs = tpl_kwargs, diff --git a/studio/backend/tests/test_cpu_threads.py b/studio/backend/tests/test_cpu_threads.py index 0dbcbdb74b..1224941622 100644 --- a/studio/backend/tests/test_cpu_threads.py +++ b/studio/backend/tests/test_cpu_threads.py @@ -63,16 +63,18 @@ def test_cpu_thread_cap_is_opt_in(raw): # Anything that is not a positive integer raises a clear ValueError. -@pytest.mark.parametrize("raw", ["zero", "0", "-3", "1.5", "abc", "8a", "0x4", "1e3", "4 0"]) +@pytest.mark.parametrize( + "raw", ["zero", "0", "-3", "1.5", "abc", "8a", "0x4", "1e3", "4 0"] +) def test_cpu_thread_cap_requires_positive_integer(raw): - with pytest.raises(ValueError, match="must be a positive integer"): + with pytest.raises(ValueError, match = "must be a positive integer"): configure_cpu_threads({"UNSLOTH_CPU_THREADS": raw}) # env=None path uses real os.environ (production call from run.py / main.py). def test_cpu_thread_cap_uses_os_environ_when_env_is_none(monkeypatch): for variable in (*_THREAD_POOL_ENV_VARS, "UNSLOTH_CPU_THREADS"): - monkeypatch.delenv(variable, raising=False) + monkeypatch.delenv(variable, raising = False) monkeypatch.setenv("UNSLOTH_CPU_THREADS", "3") configure_cpu_threads() @@ -84,7 +86,7 @@ def test_cpu_thread_cap_uses_os_environ_when_env_is_none(monkeypatch): # Calling twice must not flip any seeded value. def test_cpu_thread_cap_idempotent(monkeypatch): for variable in (*_THREAD_POOL_ENV_VARS, "UNSLOTH_CPU_THREADS"): - monkeypatch.delenv(variable, raising=False) + monkeypatch.delenv(variable, raising = False) monkeypatch.setenv("UNSLOTH_CPU_THREADS", "5") configure_cpu_threads() @@ -138,9 +140,9 @@ def test_invalid_cpu_thread_cap_exits_without_traceback(entry_point): result = subprocess.run( [sys.executable, str(entry_point)], - env=env, - capture_output=True, - text=True, + env = env, + capture_output = True, + text = True, ) assert result.returncode == 1 diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py index 84f3e41998..331605b998 100644 --- a/studio/backend/tests/test_openai_tool_passthrough.py +++ b/studio/backend/tests/test_openai_tool_passthrough.py @@ -275,18 +275,19 @@ class TestChatCompletionRequestToolFields: assert req.stop is None def test_extra_fields_accepted(self): - # `frequency_penalty`, `seed`, `response_format` are not yet - # explicitly declared but must survive Pydantic parsing now that - # extra="allow" is set. + # ``response_format`` is still an undeclared OpenAI-side field; + # it must survive Pydantic parsing because extra="allow" is set. + # ``frequency_penalty`` and ``seed`` were promoted to explicit + # ChatCompletionRequest fields in the sampling-params PR, so + # they now ride the attribute path, not model_extra. req = self._make( frequency_penalty = 0.5, seed = 42, response_format = {"type": "json_object"}, ) - # Extras land in model_extra + assert req.frequency_penalty == 0.5 + assert req.seed == 42 assert req.model_extra is not None - assert req.model_extra.get("frequency_penalty") == 0.5 - assert req.model_extra.get("seed") == 42 assert req.model_extra.get("response_format") == {"type": "json_object"} def test_unsloth_extensions_still_work(self): diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py new file mode 100644 index 0000000000..c3115bf065 --- /dev/null +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -0,0 +1,1430 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""End-to-end routing tests for the new sampling parameters. + +Pins the per-provider gating contract added by the +expose-sampling-params PR: each of `frequency_penalty`, `seed`, `stop` +/ `stop_sequences`, `service_tier`, `parallel_tool_calls` only appears +on the outbound body when the upstream provider actually accepts it. + +The provider matrix is captured per docs: +- Anthropic Messages: accepts stop_sequences, service_tier + (auto|standard_only), disable_parallel_tool_use (inverted). REJECTS + frequency_penalty, seed, logprobs (silently dropped client-side). +- OpenAI Chat Completions (default OAI-compat branch): accepts every + field; OpenAI cloud uses `max_completion_tokens` rather than + `max_tokens`. +- OpenAI Responses (gpt-5.x / o3): rejects temperature, top_p, + frequency_penalty, seed, stop, logprobs. Accepts service_tier + (auto|default|flex|priority) and parallel_tool_calls. +""" + +import asyncio +import json + +import httpx +import pytest + +from core.inference import external_provider as ep_mod +from core.inference.external_provider import ExternalProviderClient + + +def _drive(coro): + # Explicit loop lifecycle + asyncgen shutdown so the httpx / + # MockTransport-backed async generators in the providers are + # finalised in this task instead of being collected later (which + # triggers the "aiter_text aclose was never awaited" warning the + # reviewer round noticed). + loop = asyncio.new_event_loop() + try: + result = loop.run_until_complete(coro) + loop.run_until_complete(loop.shutdown_asyncgens()) + return result + finally: + loop.close() + + +def _install_mock(monkeypatch, *, sse_payload: bytes | None = None) -> dict: + captured: dict = {} + + def handler(request: httpx.Request) -> httpx.Response: + try: + captured["body"] = json.loads(request.content.decode("utf-8")) + except json.JSONDecodeError: + captured["body"] = None + captured["url"] = str(request.url) + captured["headers"] = dict(request.headers) + return httpx.Response( + 200, + content = sse_payload + or (b'event: message_stop\ndata: {"type":"message_stop"}\n\n'), + headers = {"content-type": "text/event-stream"}, + ) + + monkeypatch.setattr( + ep_mod, + "_http_client", + httpx.AsyncClient(transport = httpx.MockTransport(handler)), + ) + return captured + + +# ── Anthropic ────────────────────────────────────────────────────────── + + +def _drive_anthropic(captured, **kwargs) -> dict: + async def run(): + client = ExternalProviderClient( + provider_type = "anthropic", + base_url = "https://api.anthropic.com/v1", + api_key = "sk-ant-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "claude-opus-4-7", + temperature = 0.7, + top_p = 0.95, + max_tokens = 64, + **kwargs, + ): + pass + await client.close() + + _drive(run()) + return captured["body"] + + +def test_anthropic_stop_sequences_forwarded_as_renamed_field(monkeypatch): + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, stop = ["END", "DONE"]) + assert body.get("stop_sequences") == ["END", "DONE"], body + # Anthropic does not have a `stop` field; the unrenamed key must not appear. + assert "stop" not in body, body + + +def test_anthropic_single_string_stop_is_wrapped(monkeypatch): + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, stop = "STOPHERE") + assert body.get("stop_sequences") == ["STOPHERE"], body + + +def test_anthropic_empty_stop_omitted(monkeypatch): + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, stop = []) + assert "stop_sequences" not in body, body + assert "stop" not in body, body + + +def test_anthropic_stop_sequences_dedup_and_drop_whitespace(monkeypatch): + """Anthropic 400s on any stop sequence that contains no non- + whitespace character (`stop_sequences: each stop sequence must + contain non-whitespace`). Empty strings, " ", "\\n", "\\n\\n", and + other whitespace-only chips are filtered out client-side so the + request reaches the wire. Duplicates are deduped to avoid wasting + slots against the cap. + """ + captured = _install_mock(monkeypatch) + body = _drive_anthropic( + captured, + stop = ["END", "", "END", "DONE", " ", "END", "\n\n", "\t"], + ) + # Order preserved on first sight, duplicates + every whitespace-only + # entry dropped. + assert body.get("stop_sequences") == ["END", "DONE"], body + + +def test_anthropic_single_whitespace_stop_string_dropped(monkeypatch): + """Single-string stop="\\n\\n" must not reach the wire either.""" + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, stop = "\n\n") + assert "stop_sequences" not in body, body + + +def test_anthropic_stop_sequences_truncated_to_16(monkeypatch): + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, stop = [f"S{i}" for i in range(20)]) + assert len(body.get("stop_sequences", [])) == 16, body + assert body["stop_sequences"][0] == "S0" + assert body["stop_sequences"][-1] == "S15" + + +def test_anthropic_service_tier_forwarded_when_valid(monkeypatch): + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, service_tier = "standard_only") + assert body.get("service_tier") == "standard_only", body + + +@pytest.mark.parametrize( + "bogus", ["flex", "priority", "scale", "default", "", "auto-foo"] +) +def test_anthropic_service_tier_unsupported_values_dropped(monkeypatch, bogus): + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, service_tier = bogus) + assert "service_tier" not in body, body + + +def _drive_anthropic_with_tools(captured, **kwargs) -> dict: + """Same as `_drive_anthropic` but enables a server-side tool + (`web_search`) so the request body carries `tools`. Needed to + exercise the `disable_parallel_tool_use` nesting path, which only + fires when there is at least one tool defined. + """ + enabled_tools = kwargs.pop("enabled_tools", None) or ["web_search"] + return _drive_anthropic(captured, enabled_tools = enabled_tools, **kwargs) + + +def test_anthropic_disable_parallel_tool_use_nested_under_tool_choice(monkeypatch): + """`disable_parallel_tool_use` must be a property of `tool_choice`, + NOT a top-level body field. Top-level placement is rejected with + `extraneous key [disable_parallel_tool_use] is not permitted`. See + https://platform.claude.com/docs/en/agents-and-tools/tool-use/implement-tool-use. + """ + captured = _install_mock(monkeypatch) + body = _drive_anthropic_with_tools(captured, parallel_tool_calls = False) + # Top-level placement is rejected with 400. + assert "disable_parallel_tool_use" not in body, body + assert "parallel_tool_calls" not in body, body + # Flag lives on tool_choice; default type is "auto". + tc = body.get("tool_choice") + assert isinstance(tc, dict), body + assert tc.get("disable_parallel_tool_use") is True, body + assert tc.get("type") == "auto", body + + +def test_anthropic_disable_parallel_tool_use_skipped_without_tools(monkeypatch): + """Without tools the flag is a no-op upstream; keep the body + minimal and never emit it at top level either. + """ + captured = _install_mock(monkeypatch) + body = _drive_anthropic(captured, parallel_tool_calls = False) + assert "disable_parallel_tool_use" not in body, body + assert "parallel_tool_calls" not in body, body + assert "tool_choice" not in body, body + + +def test_anthropic_parallel_tool_calls_default_not_sent(monkeypatch): + captured = _install_mock(monkeypatch) + body = _drive_anthropic_with_tools(captured, parallel_tool_calls = True) + # True is the upstream default; do not surface a tool_choice we + # would otherwise not have set, and definitely no top-level + # `disable_parallel_tool_use`. + assert "disable_parallel_tool_use" not in body, body + assert "parallel_tool_calls" not in body, body + tc = body.get("tool_choice") + if isinstance(tc, dict): + assert "disable_parallel_tool_use" not in tc, body + + +def test_anthropic_rejects_openai_only_knobs(monkeypatch): + """frequency_penalty / seed are dropped at the dispatch layer. + + Anthropic has no equivalent; the keyword args are not even forwarded + from stream_chat_completion to _stream_anthropic. This test pins + that no such field reaches the Messages body. + """ + captured = _install_mock(monkeypatch) + body = _drive_anthropic( + captured, + frequency_penalty = 1.5, + seed = 42, + ) + assert "frequency_penalty" not in body, body + assert "seed" not in body, body + + +# ── OpenAI Chat Completions (default OAI-compat) ───────────────────────── + + +def _drive_openai_compat(captured, **kwargs) -> dict: + """Send through the default OAI-compat branch (NOT /v1/responses). + + Use qwen so the dispatcher takes the default branch and the provider + inherits the default 16-stop cap (Mistral now caps at 4 per its + third-party shims). + """ + + async def run(): + client = ExternalProviderClient( + provider_type = "qwen", + base_url = "https://dashscope-intl.aliyuncs.com/compatible-mode/v1", + api_key = "test-key", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "qwen-plus", + temperature = 0.5, + top_p = 0.9, + max_tokens = 64, + **kwargs, + ): + pass + await client.close() + + _drive(run()) + return captured["body"] + + +def _oai_done_payload() -> bytes: + return b"data: [DONE]\n\n" + + +def test_openai_compat_forwards_frequency_penalty(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured, frequency_penalty = 1.25) + assert body.get("frequency_penalty") == 1.25, body + + +def test_openai_compat_forwards_seed(monkeypatch): + """qwen has no seed_field override so seed forwards as `seed`.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured, seed = 12345) + assert body.get("seed") == 12345, body + + +def test_mistral_renames_seed_to_random_seed(monkeypatch): + """Mistral's registry sets seed_field="random_seed" so the OAI seed + is renamed on the wire. https://docs.mistral.ai/api/endpoint/chat""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "mistral", + base_url = "https://api.mistral.ai/v1", + api_key = "mistral-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "mistral-small-latest", + temperature = 0.5, + top_p = 0.9, + max_tokens = 64, + seed = 12345, + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert body.get("random_seed") == 12345, body + assert "seed" not in body, body + + +def test_openai_compat_seed_field_default_is_seed(monkeypatch): + """Providers without a seed_field override get the OpenAI default.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "deepseek", + base_url = "https://api.deepseek.com/v1", + api_key = "ds-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "deepseek-chat", + temperature = 0.5, + top_p = 0.9, + max_tokens = 64, + seed = 7, + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert body.get("seed") == 7, body + assert "random_seed" not in body, body + + +def test_openai_compat_deepseek_stop_cap_is_16(monkeypatch): + """DeepSeek docs allow up to 16 stop sequences; the previous + 4-cap silently truncated valid configs.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "deepseek", + base_url = "https://api.deepseek.com/v1", + api_key = "ds-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "deepseek-chat", + temperature = 0.5, + top_p = 0.9, + max_tokens = 64, + stop = [f"S{i}" for i in range(20)], + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert len(body.get("stop", [])) == 16, body + + +def test_openai_compat_forwards_stop_array(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured, stop = ["END", "DONE"]) + assert body.get("stop") == ["END", "DONE"], body + # The default OAI-compat branch does not rename to stop_sequences. + assert "stop_sequences" not in body, body + + +def test_openai_compat_truncates_stop_to_default_cap(monkeypatch): + """Default OAI-compat cap is 16 (Qwen, DeepSeek, HuggingFace, custom); + OpenAI Chat / OpenRouter / Gemini / Mistral have tighter 4-entry caps.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured, stop = [f"s{i}" for i in range(20)]) + assert len(body.get("stop", [])) == 16, body + assert body["stop"][0] == "s0" + assert body["stop"][-1] == "s15" + + +def test_mistral_stop_cap_is_4(monkeypatch): + """Mistral's docs publish no max but third-party shims cap at 4; + match OpenAI Chat's cap to avoid silent upstream truncation.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "mistral", + base_url = "https://api.mistral.ai/v1", + api_key = "mistral-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "mistral-small-latest", + temperature = 0.5, + top_p = 0.9, + max_tokens = 64, + stop = [f"S{i}" for i in range(8)], + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert body.get("stop") == ["S0", "S1", "S2", "S3"], body + + +def test_openai_compat_stop_dedup_and_drop_empties(monkeypatch): + """Duplicates and empties shouldn't eat into the cap.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured, stop = ["END", "", "END", "DONE", "FIN", "END"]) + assert body.get("stop") == ["END", "DONE", "FIN"], body + + +def test_openai_compat_empty_stop_omitted(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured, stop = []) + assert "stop" not in body, body + + +def test_openai_compat_drops_service_tier_by_default(monkeypatch): + """Generic OAI-compat providers (mistral, deepseek, openrouter, ...) + do not document a `service_tier` field. The dispatcher must drop + it unless the provider registry explicitly opts in with + `accepts_service_tier=True`; otherwise a stale frontend could + smuggle Anthropic/OpenAI-Responses-only values onto unrelated + providers.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured, service_tier = "flex") + assert "service_tier" not in body, body + + +def test_openai_compat_forwards_parallel_tool_calls(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured, parallel_tool_calls = False) + assert body.get("parallel_tool_calls") is False, body + + +def test_openai_compat_omits_unset_optionals(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + body = _drive_openai_compat(captured) + # Optional knobs default to None / unset -> never appear. + assert "frequency_penalty" not in body, body + assert "seed" not in body, body + assert "stop" not in body, body + assert "service_tier" not in body, body + assert "parallel_tool_calls" not in body, body + + +# ── OpenAI Responses (gpt-5.x via /v1/responses) ───────────────────────── + + +def _responses_done_payload() -> bytes: + return ( + b"event: response.completed\n" + b'data: {"type":"response.completed","response":{"usage":{}}}\n\n' + ) + + +def _drive_openai_responses(captured, **kwargs) -> dict: + async def run(): + client = ExternalProviderClient( + provider_type = "openai", + base_url = "https://api.openai.com/v1", + api_key = "sk-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "gpt-5.5", + temperature = 1.0, + top_p = 1.0, + max_tokens = 64, + **kwargs, + ): + pass + await client.close() + + _drive(run()) + return captured["body"] + + +def test_openai_responses_drops_temperature_top_p(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) + body = _drive_openai_responses(captured) + assert "temperature" not in body, body + assert "top_p" not in body, body + + +def test_openai_responses_drops_frequency_penalty_seed_stop(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) + body = _drive_openai_responses( + captured, + frequency_penalty = 1.5, + seed = 99, + stop = ["END"], + ) + # Responses 400s on any of these; the dispatch must drop them + # before they hit the wire. + assert "frequency_penalty" not in body, body + assert "seed" not in body, body + assert "stop" not in body, body + assert "stop_sequences" not in body, body + + +def test_openai_responses_forwards_service_tier(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) + body = _drive_openai_responses(captured, service_tier = "priority") + assert body.get("service_tier") == "priority", body + + +@pytest.mark.parametrize("value", ["auto", "default", "flex", "priority"]) +def test_openai_responses_forwards_documented_service_tiers(monkeypatch, value): + """The live OpenAI Responses API reference lists `service_tier` as + `auto|default|flex|priority` for /v1/responses. Pin that every value + in the documented enum forwards untouched.""" + captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) + body = _drive_openai_responses(captured, service_tier = value) + assert body.get("service_tier") == value, body + + +@pytest.mark.parametrize("bogus", ["scale", "standard_only", "bogus", ""]) +def test_openai_responses_drops_undocumented_service_tier(monkeypatch, bogus): + """`scale` and `standard_only` are not in the documented Responses + request enum; drop them client-side so a stale frontend never + sends an upstream-rejected value.""" + captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) + body = _drive_openai_responses(captured, service_tier = bogus) + assert "service_tier" not in body, body + + +def test_openai_responses_forwards_parallel_tool_calls(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) + body = _drive_openai_responses(captured, parallel_tool_calls = False) + assert body.get("parallel_tool_calls") is False, body + + +def test_openai_responses_omits_unset_optionals(monkeypatch): + captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload()) + body = _drive_openai_responses(captured) + assert "service_tier" not in body, body + assert "parallel_tool_calls" not in body, body + + +# ── Schema-level smoke tests ───────────────────────────────────────────── + + +def test_chat_completion_request_accepts_new_sampling_fields(): + from models.inference import ChatCompletionRequest + + payload = ChatCompletionRequest.model_validate( + { + "messages": [{"role": "user", "content": "hi"}], + "frequency_penalty": -1.0, + "seed": 0, + "stop": ["END"], + "service_tier": "auto", + "parallel_tool_calls": True, + } + ) + assert payload.frequency_penalty == -1.0 + assert payload.seed == 0 + assert payload.stop == ["END"] + assert payload.service_tier == "auto" + assert payload.parallel_tool_calls is True + + +def test_chat_completion_request_rejects_bad_service_tier(): + import pydantic + from models.inference import ChatCompletionRequest + + with pytest.raises(pydantic.ValidationError): + ChatCompletionRequest.model_validate( + { + "messages": [{"role": "user", "content": "hi"}], + "service_tier": "bogus", + } + ) + + +def test_chat_completion_request_clamps_frequency_penalty_range(): + import pydantic + from models.inference import ChatCompletionRequest + + with pytest.raises(pydantic.ValidationError): + ChatCompletionRequest.model_validate( + { + "messages": [{"role": "user", "content": "hi"}], + "frequency_penalty": 3.0, + } + ) + with pytest.raises(pydantic.ValidationError): + ChatCompletionRequest.model_validate( + { + "messages": [{"role": "user", "content": "hi"}], + "frequency_penalty": -3.0, + } + ) + + +# ── Kimi web-search bypass forwards new sampling fields ──────────────── + + +def test_kimi_web_search_bypass_forwards_new_sampling_fields(monkeypatch): + """The Kimi $web_search path takes an early return into + `_stream_kimi_web_search` before the default OAI-compat body + builder runs; forwarding here keeps Kimi-with-search and + Kimi-without-search in lockstep.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "kimi", + base_url = "https://api.moonshot.ai/v1", + api_key = "kimi-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "kimi-k2.6", + temperature = 1.0, + top_p = 1.0, + max_tokens = 256, + enabled_tools = ["web_search"], + presence_penalty = 0.5, + frequency_penalty = 1.25, + seed = 7, + stop = ["END"], + parallel_tool_calls = False, + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + # Kimi locks these: stripped by body_omit in providers.py. + assert "frequency_penalty" not in body, body + assert "temperature" not in body, body + assert "top_p" not in body, body + assert "seed" not in body, body + assert "parallel_tool_calls" not in body, body + assert "presence_penalty" not in body, body + # Knobs not on Kimi's drop-list forward through the bypass. + assert body.get("stop") == ["END"], body + + +def test_kimi_web_search_uses_kimi_stop_cap_5(monkeypatch): + """Kimi documents a 5-stop max; the web-search bypass must honour + `provider_info["stop_max"]` rather than the OpenAI 4-cap or the + permissive default.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "kimi", + base_url = "https://api.moonshot.ai/v1", + api_key = "kimi-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "kimi-k2.6", + temperature = 1.0, + top_p = 1.0, + max_tokens = 256, + enabled_tools = ["web_search"], + stop = [f"S{i}" for i in range(10)], + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert len(body.get("stop", [])) == 5, body + assert body["stop"] == ["S0", "S1", "S2", "S3", "S4"], body + + +def test_openrouter_stop_cap_is_4(monkeypatch): + """OpenRouter normalises to OpenAI's chat schema and inherits the + 4-entry stop cap; the default 16-cap is too permissive for it.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "openrouter", + base_url = "https://openrouter.ai/api/v1", + api_key = "or-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "openai/gpt-4o", + temperature = 0.5, + top_p = 0.9, + max_tokens = 64, + stop = [f"S{i}" for i in range(10)], + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert len(body.get("stop", [])) == 4, body + assert body["stop"] == ["S0", "S1", "S2", "S3"], body + + +def test_gemini_stop_sequences_capped_to_5(monkeypatch): + """Native Gemini API forwards `stop` as generationConfig.stopSequences, + capped at 5 per https://ai.google.dev/api/generate-content#generationconfig.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "gemini", + base_url = "https://generativelanguage.googleapis.com/v1beta/openai", + api_key = "gemini-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "gemini-3.1-pro-preview", + temperature = 0.5, + top_p = 0.9, + max_tokens = 64, + stop = [f"S{i}" for i in range(10)], + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + gen_config = body.get("generationConfig", {}) + assert gen_config.get("stopSequences") == ["S0", "S1", "S2", "S3", "S4"], body + + +def test_openrouter_forwards_top_a(monkeypatch): + """OpenRouter exposes `top_a` (the tail-cut sampler). The frontend + capability map lets the value through; verify the OAI-compat body + builder actually carries it to the wire.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "openrouter", + base_url = "https://openrouter.ai/api/v1", + api_key = "or-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "openai/gpt-4o", + temperature = 0.5, + top_p = 0.9, + max_tokens = 64, + top_a = 0.25, + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert body.get("top_a") == 0.25, body + + +def test_vllm_forwards_output_shape_knobs(monkeypatch): + """vLLM accepts skip_special_tokens / spaces_between_special_tokens / + include_stop_str_in_output / truncate_prompt_tokens. Verify the + OAI-compat external proxy forwards them to the wire body.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "vllm", + base_url = "https://vllm.example.com/v1", + api_key = "vllm-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "Qwen/Qwen3-4B", + temperature = 0.5, + top_p = 0.9, + max_tokens = 64, + skip_special_tokens = False, + spaces_between_special_tokens = False, + include_stop_str_in_output = True, + truncate_prompt_tokens = 2048, + min_tokens = 8, + ignore_eos = True, + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert body.get("skip_special_tokens") is False, body + assert body.get("spaces_between_special_tokens") is False, body + assert body.get("include_stop_str_in_output") is True, body + assert body.get("truncate_prompt_tokens") == 2048, body + assert body.get("min_tokens") == 8, body + assert body.get("ignore_eos") is True, body + + +def test_kimi_drops_stop_strings_over_32_bytes(monkeypatch): + """Kimi limits each stop string to <= 32 bytes per + https://platform.kimi.ai/docs/api/chat. Drop overlong entries + client-side so a stale UI cannot 400 the request.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "kimi", + base_url = "https://api.moonshot.ai/v1", + api_key = "kimi-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "kimi-k2.6", + temperature = 1.0, + top_p = 1.0, + max_tokens = 256, + stop = ["END", "x" * 33, "DONE"], + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert body.get("stop") == ["END", "DONE"], body + + +def test_kimi_web_search_drops_stop_strings_over_32_bytes(monkeypatch): + """Same byte cap applies to the Kimi web-search bypass.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "kimi", + base_url = "https://api.moonshot.ai/v1", + api_key = "kimi-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "kimi-k2.6", + temperature = 1.0, + top_p = 1.0, + max_tokens = 256, + enabled_tools = ["web_search"], + stop = ["END", "x" * 40, "DONE"], + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert body.get("stop") == ["END", "DONE"], body + + +def test_kimi_default_path_uses_kimi_stop_cap_5(monkeypatch): + """The normal Kimi path must also honour the documented 5-cap.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "kimi", + base_url = "https://api.moonshot.ai/v1", + api_key = "kimi-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "kimi-k2.6", + temperature = 1.0, + top_p = 1.0, + max_tokens = 256, + stop = [f"S{i}" for i in range(10)], + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert len(body.get("stop", [])) == 5, body + + +# ── Local OpenAI passthrough forwards new sampling fields ────────────── + + +def test_local_openai_passthrough_forwards_new_sampling_fields(): + """`_build_openai_passthrough_body` forwards frequency_penalty, + seed, stop, and parallel_tool_calls to llama-server.""" + from models.inference import ChatCompletionRequest + from routes.inference import _build_openai_passthrough_body + + payload = ChatCompletionRequest.model_validate( + { + "messages": [{"role": "user", "content": "hi"}], + "stream": True, + "frequency_penalty": 1.25, + "seed": 123, + "stop": ["END"], + "parallel_tool_calls": False, + } + ) + body = _build_openai_passthrough_body(payload, backend_ctx = 4096) + assert body["frequency_penalty"] == 1.25, body + assert body["seed"] == 123, body + assert body["stop"] == ["END"], body + assert body["parallel_tool_calls"] is False, body + + +# ── Responses → ChatCompletions bridge preserves parallel_tool_calls ── + + +def test_responses_to_chat_bridge_preserves_parallel_tool_calls(): + """`_build_chat_request` (the /v1/responses to /v1/chat/completions + translator) must forward parallel_tool_calls so a Responses-API + caller's preference reaches llama-server.""" + from models.inference import ChatMessage, ResponsesRequest + from routes.inference import _build_chat_request, _build_openai_passthrough_body + + payload = ResponsesRequest( + input = "hi", + stream = True, + parallel_tool_calls = False, + ) + chat_req = _build_chat_request( + payload, + [ChatMessage(role = "user", content = "hi")], + stream = True, + ) + assert chat_req.parallel_tool_calls is False, chat_req + body = _build_openai_passthrough_body(chat_req, backend_ctx = 4096) + assert body["parallel_tool_calls"] is False, body + + +def test_responses_to_chat_bridge_omits_unset_parallel_tool_calls(): + """Unset parallel_tool_calls (None) must not appear on the + translated body; the upstream default is true everywhere so + forwarding None would over-specify.""" + from models.inference import ChatMessage, ResponsesRequest + from routes.inference import _build_chat_request, _build_openai_passthrough_body + + payload = ResponsesRequest(input = "hi", stream = True) + chat_req = _build_chat_request( + payload, + [ChatMessage(role = "user", content = "hi")], + stream = True, + ) + assert chat_req.parallel_tool_calls is None, chat_req + body = _build_openai_passthrough_body(chat_req, backend_ctx = 4096) + assert "parallel_tool_calls" not in body, body + + +# ── Backend ChatInferenceSettings schema accepts new fields ──────────── + + +def test_chat_settings_payload_accepts_new_sampling_keys(): + """ChatSettingsPayload has extra="forbid" so the new keys must be + listed explicitly; otherwise every settings save with any of them + 422s. Pin the round-trip for every persisted sampler the frontend + can emit (see PERSISTED_INFERENCE_PARAM_KEYS in chat-runtime-store.ts).""" + from routes.chat_history import ChatSettingsPayload + + inference = { + "frequencyPenalty": 0.7, + "seed": 42, + "stop": ["END"], + "serviceTier": "standard_only", + "parallelToolCalls": False, + "fastMode": True, + # Extended llama.cpp / vLLM / OpenRouter samplers. + "typicalP": 0.85, + "topNSigma": 2.5, + "repeatLastN": 64, + "dynatempRange": 0.3, + "dynatempExponent": 1.2, + "mirostat": 2, + "mirostatTau": 4.0, + "mirostatEta": 0.15, + "topA": 0.2, + "dryMultiplier": 0.8, + "dryBase": 1.75, + "dryAllowedLength": 2, + "dryPenaltyLastN": -1, + "xtcProbability": 0.5, + "xtcThreshold": 0.1, + "minKeep": 5, + "ignoreEos": True, + "minTokens": 10, + "skipSpecialTokens": False, + "spacesBetweenSpecialTokens": False, + "includeStopStrInOutput": True, + "truncatePromptTokens": 1024, + "nKeep": -1, + "nProbs": 5, + "cachePrompt": False, + "returnTokens": True, + "timingsPerToken": True, + "postSamplingProbs": True, + } + parsed = ChatSettingsPayload.model_validate({"inferenceParams": inference}) + ip = parsed.inferenceParams + assert ip is not None + for key, expected in inference.items(): + assert getattr(ip, key) == expected, f"{key} did not round-trip" + + +# ── Local /v1/messages: disable_parallel_tool_use translation ────────── + + +def test_local_anthropic_disable_parallel_tool_use_translation(): + """Anthropic nests `disable_parallel_tool_use` under `tool_choice` + (per docs.claude.com). The local /v1/messages GGUF tool path must + invert it into OpenAI-shaped `parallel_tool_calls` so third-party + clients (Claude SDK, LiteLLM in passthrough mode) opt out of + parallel calls successfully even on the local model.""" + + # Mirror the extraction logic in routes/inference.py:anthropic_messages. + def _extract(tc): + if isinstance(tc, dict): + v = tc.get("disable_parallel_tool_use") + if isinstance(v, bool): + return not v + return None + + assert _extract({"type": "auto", "disable_parallel_tool_use": True}) is False + assert _extract({"type": "any", "disable_parallel_tool_use": False}) is True + assert _extract({"type": "auto"}) is None + assert _extract(None) is None + assert _extract("auto") is None # string form (non-dict) → no opinion + assert _extract({"type": "auto", "disable_parallel_tool_use": "yes"}) is None + + +def test_anthropic_passthrough_emitter_serialises_tool_calls_on_opt_out(): + """When the Anthropic-compat passthrough is asked to disable + parallel tool calls, `AnthropicPassthroughEmitter.feed_chunk()` + must drop every streamed `delta.tool_calls` entry beyond the + first index, matching the GGUF agentic-loop client-side cap and + keeping the wire-side `disable_parallel_tool_use=true` honest + even when llama-server's jinja template ignores it.""" + from core.inference.anthropic_compat import AnthropicPassthroughEmitter + + emitter = AnthropicPassthroughEmitter(parallel_tool_calls = False) + emitter.start("msg_x", "test-model") + events = emitter.feed_chunk( + { + "choices": [ + { + "delta": { + "tool_calls": [ + { + "index": 0, + "id": "call_a", + "function": {"name": "first", "arguments": "{"}, + }, + { + "index": 1, + "id": "call_b", + "function": {"name": "second", "arguments": "{"}, + }, + ] + } + } + ] + } + ) + joined = "\n".join(events) + assert "first" in joined, joined + assert "second" not in joined, joined + + emitter_open = AnthropicPassthroughEmitter(parallel_tool_calls = True) + emitter_open.start("msg_y", "test-model") + events_open = emitter_open.feed_chunk( + { + "choices": [ + { + "delta": { + "tool_calls": [ + {"index": 0, "id": "a", "function": {"name": "x"}}, + {"index": 1, "id": "b", "function": {"name": "y"}}, + ] + } + } + ] + } + ) + joined_open = "\n".join(events_open) + assert "x" in joined_open and "y" in joined_open, joined_open + + +def test_gguf_tool_loop_enforces_parallel_tool_calls_false(): + """llama.cpp's `parallel_tool_calls` flag is not enforced by every + jinja template (see ggml-org/llama.cpp#22043), so when the caller + opted out we must cap tool_calls to the first entry before the + agentic loop executes them. The cap is a single-line slice in + `generate_chat_completion_with_tools`; pin the contract.""" + from pathlib import Path + + src = Path(__file__).resolve().parent.parent / "core/inference/llama_cpp.py" + text = src.read_text() + assert "if parallel_tool_calls is False and tool_calls" in text, ( + "GGUF tool loop must enforce parallel_tool_calls=False by " + "truncating tool_calls before assistant_msg is built; that " + "is the client-side guarantee llama-server's flag does not " + "give us. See routes/inference.py and chat-adapter.ts for " + "the wire-side forwarding of the same flag." + ) + assert "tool_calls = tool_calls[:1]" in text + + +def test_local_anthropic_passthrough_helpers_accept_parallel_tool_calls(): + """The Anthropic-compat client-tool passthrough helpers + (`_anthropic_passthrough_stream` / + `_anthropic_passthrough_non_streaming`) must accept and forward + `parallel_tool_calls` through `_build_passthrough_payload` so the + `disable_parallel_tool_use` translation works on the client-tool + branch the same way it does on the server-tool loop. Verified by + introspecting the signatures and confirming the field reaches the + body via the shared payload builder.""" + import inspect + + from routes import inference as route_mod + + for fn in ( + route_mod._anthropic_passthrough_stream, + route_mod._anthropic_passthrough_non_streaming, + ): + params = inspect.signature(fn).parameters + assert "parallel_tool_calls" in params, ( + f"{fn.__name__} must accept parallel_tool_calls so the " + "Anthropic disable_parallel_tool_use translation reaches " + "the llama-server body on the client-tool branch" + ) + + body = route_mod._build_passthrough_payload( + openai_messages = [{"role": "user", "content": "hi"}], + openai_tools = [{"type": "function", "function": {"name": "x"}}], + temperature = 0.7, + top_p = 0.95, + top_k = 20, + max_tokens = 64, + stream = True, + parallel_tool_calls = False, + ) + assert body.get("parallel_tool_calls") is False, body + + +def test_local_passthrough_forwards_extended_llama_cpp_samplers(): + """The local llama.cpp passthrough payload builder must forward + the extended sampler chain when set: top_n_sigma, repeat_last_n, + dynatemp_range/exponent, mirostat/mirostat_tau/mirostat_eta. Each + is gated `is not None` so a default-off value (e.g. mirostat=0) is + still forwarded explicitly when the caller opted in. + """ + from routes import inference as route_mod + + body = route_mod._build_passthrough_payload( + openai_messages = [{"role": "user", "content": "hi"}], + openai_tools = None, + temperature = 0.6, + top_p = 0.95, + top_k = 20, + max_tokens = 64, + stream = True, + top_n_sigma = 1.5, + repeat_last_n = 128, + dynatemp_range = 0.2, + dynatemp_exponent = 1.5, + mirostat = 2, + mirostat_tau = 5.0, + mirostat_eta = 0.1, + ) + assert body.get("top_n_sigma") == 1.5 + assert body.get("repeat_last_n") == 128 + assert body.get("dynatemp_range") == 0.2 + assert body.get("dynatemp_exponent") == 1.5 + assert body.get("mirostat") == 2 + assert body.get("mirostat_tau") == 5.0 + assert body.get("mirostat_eta") == 0.1 + + # Unset = absent from body so llama-server falls back to defaults. + body2 = route_mod._build_passthrough_payload( + openai_messages = [{"role": "user", "content": "hi"}], + openai_tools = None, + temperature = 0.6, + top_p = 0.95, + top_k = 20, + max_tokens = 64, + stream = True, + ) + for key in ( + "top_n_sigma", + "repeat_last_n", + "dynatemp_range", + "dynatemp_exponent", + "mirostat", + "mirostat_tau", + "mirostat_eta", + ): + assert key not in body2, body2 + + +def test_local_passthrough_forwards_dry_xtc_min_keep_eos_min_tokens(): + """The local llama.cpp passthrough payload builder must forward the + DRY (4-field) + XTC (2-field) + min_keep + ignore_eos + min_tokens + chain when set. Each is gated `is not None` so an explicit + upstream-default value (e.g. min_keep=0, ignore_eos=False) still + reaches the wire when the caller opted in. + """ + from routes import inference as route_mod + + body = route_mod._build_passthrough_payload( + openai_messages = [{"role": "user", "content": "hi"}], + openai_tools = None, + temperature = 0.6, + top_p = 0.95, + top_k = 20, + max_tokens = 64, + stream = True, + dry_multiplier = 0.8, + dry_base = 1.75, + dry_allowed_length = 3, + dry_penalty_last_n = -1, + xtc_probability = 0.5, + xtc_threshold = 0.1, + min_keep = 1, + ignore_eos = True, + min_tokens = 16, + ) + assert body.get("dry_multiplier") == 0.8 + assert body.get("dry_base") == 1.75 + assert body.get("dry_allowed_length") == 3 + assert body.get("dry_penalty_last_n") == -1 + assert body.get("xtc_probability") == 0.5 + assert body.get("xtc_threshold") == 0.1 + assert body.get("min_keep") == 1 + assert body.get("ignore_eos") is True + assert body.get("min_tokens") == 16 + + # Unset = absent from body. Matches the upstream "use default" + # contract: llama-server / vLLM apply their own defaults instead. + body2 = route_mod._build_passthrough_payload( + openai_messages = [{"role": "user", "content": "hi"}], + openai_tools = None, + temperature = 0.6, + top_p = 0.95, + top_k = 20, + max_tokens = 64, + stream = True, + ) + for key in ( + "dry_multiplier", + "dry_base", + "dry_allowed_length", + "dry_penalty_last_n", + "xtc_probability", + "xtc_threshold", + "min_keep", + "ignore_eos", + "min_tokens", + ): + assert key not in body2, body2 + + +def test_local_passthrough_forwards_vllm_output_and_llama_cpp_instrumentation(): + """Round-trip the 10 extra knobs added in round 4: + skip_special_tokens / spaces_between_special_tokens / + include_stop_str_in_output / truncate_prompt_tokens (vLLM + SamplingParams) + n_keep / n_probs / cache_prompt / return_tokens / + timings_per_token / post_sampling_probs (llama-server README). + Each is gated `is not None` so explicit upstream-default values + (skip_special_tokens=True, cache_prompt=True, etc) still reach the + wire when the caller opted in. + """ + from routes import inference as route_mod + + body = route_mod._build_passthrough_payload( + openai_messages = [{"role": "user", "content": "hi"}], + openai_tools = None, + temperature = 0.6, + top_p = 0.95, + top_k = 20, + max_tokens = 64, + stream = True, + skip_special_tokens = False, + spaces_between_special_tokens = False, + include_stop_str_in_output = True, + truncate_prompt_tokens = 4096, + n_keep = -1, + n_probs = 5, + cache_prompt = False, + return_tokens = True, + timings_per_token = True, + post_sampling_probs = True, + ) + assert body.get("skip_special_tokens") is False + assert body.get("spaces_between_special_tokens") is False + assert body.get("include_stop_str_in_output") is True + assert body.get("truncate_prompt_tokens") == 4096 + assert body.get("n_keep") == -1 + assert body.get("n_probs") == 5 + assert body.get("cache_prompt") is False + assert body.get("return_tokens") is True + assert body.get("timings_per_token") is True + assert body.get("post_sampling_probs") is True + + # Unset = absent from body so each backend applies its own default. + body2 = route_mod._build_passthrough_payload( + openai_messages = [{"role": "user", "content": "hi"}], + openai_tools = None, + temperature = 0.6, + top_p = 0.95, + top_k = 20, + max_tokens = 64, + stream = True, + ) + for key in ( + "skip_special_tokens", + "spaces_between_special_tokens", + "include_stop_str_in_output", + "truncate_prompt_tokens", + "n_keep", + "n_probs", + "cache_prompt", + "return_tokens", + "timings_per_token", + "post_sampling_probs", + ): + assert key not in body2, body2 + + +def test_local_passthrough_forwards_typical_p_when_set(): + """`typical_p` is a llama.cpp-specific sampler (`typ_p` in the + sampler chain). The local-llama-cpp passthrough payload builder must + forward it when set so the chat-adapter can opt in for local + backends without the field bleeding into external providers (whose + capability map gates it off).""" + from routes import inference as route_mod + + body = route_mod._build_passthrough_payload( + openai_messages = [{"role": "user", "content": "hi"}], + openai_tools = None, + temperature = 0.6, + top_p = 0.95, + top_k = 20, + max_tokens = 64, + stream = True, + typical_p = 0.7, + ) + assert body.get("typical_p") == 0.7, body + + # When unset, the field is omitted entirely so llama-server falls + # back to its 1.0 default. + body2 = route_mod._build_passthrough_payload( + openai_messages = [{"role": "user", "content": "hi"}], + openai_tools = None, + temperature = 0.6, + top_p = 0.95, + top_k = 20, + max_tokens = 64, + stream = True, + ) + assert "typical_p" not in body2, body2 + + +def test_anthropic_4_7_sampling_removed_regex_matches_expected_ids(): + """Pin the canonical Claude 4.7 model-id shape so the frontend + ANTHROPIC_4_7_SAMPLING_REMOVED_REGEX in + studio/frontend/src/features/chat/provider-capabilities.ts stays + in lockstep with the backend strip in + external_provider._stream_anthropic. + + Drift would mean the panel either silently strips a knob the user + moved (UI shows, wire drops) or the wire 400s after the user moved + a knob the UI should have hidden. Both are user-visible bugs. + """ + from core.inference.external_provider import ( + _ANTHROPIC_4_7_SAMPLING_REMOVED as RX, + ) + + # Only Opus shipped in the 4.7 generation per + # platform.claude.com/docs/en/about-claude/models/overview; Sonnet + # stops at 4.6 and Haiku at 4.5. Pin both directions explicitly so + # the regex never widens by accident. + should_match = [ + "claude-opus-4-7", + "claude-opus-4-7-20260418", + "claude-opus-4-7.1", + ] + should_not_match = [ + "claude-sonnet-4-7", + "claude-haiku-4-7", + "claude-opus-4-6", + "claude-sonnet-4-6", + "claude-haiku-4-5", + "claude-opus-4-71", + "claude-opus-5", + "claude-3-opus", + "gpt-4o", + ] + for mid in should_match: + assert RX.match(mid), f"{mid!r} should match 4.7 sampling-removed regex" + for mid in should_not_match: + assert not RX.match(mid), f"{mid!r} should NOT match 4.7 regex" + + +def test_deepseek_payload_omits_seed_and_parallel_tool_calls(): + """DeepSeek's published /chat/completions schema lists + messages/model/thinking/max_tokens/response_format/stop/stream/ + temperature/top_p/tools/tool_choice/logprobs/top_logprobs/user_id + only. `seed` and `parallel_tool_calls` are not in the schema; the + capability bucket hides them so the chat-adapter never sends them. + Source: + https://api-docs.deepseek.com/api/create-chat-completion + """ + # Frontend capability flags are the source of truth. Re-derive them + # by reading the TS file as text (the backend has no JS engine) and + # confirm the deepseek bucket has seed:false + parallelToolCalls:false. + from pathlib import Path + + src = ( + Path(__file__).resolve().parents[2] + / "frontend" + / "src" + / "features" + / "chat" + / "provider-capabilities.ts" + ).read_text(encoding = "utf-8") + deepseek_idx = src.index(" deepseek: {") + end = src.index("},", deepseek_idx) + bucket = src[deepseek_idx:end] + assert "seed: false" in bucket, bucket + assert "parallelToolCalls: false" in bucket, bucket diff --git a/studio/frontend/src/components/ui/stop-sequences-input.tsx b/studio/frontend/src/components/ui/stop-sequences-input.tsx new file mode 100644 index 0000000000..e9adbc894f --- /dev/null +++ b/studio/frontend/src/components/ui/stop-sequences-input.tsx @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { cn } from "@/lib/utils"; +import { XIcon } from "lucide-react"; +import { type KeyboardEvent, useState } from "react"; + +/** Chips editor for stop/stop_sequences. Enter or comma commits; Backspace + * on empty deletes the last. OpenAI Chat caps at 4; pass Infinity to disable. */ +export interface StopSequencesInputProps { + value: string[]; + onChange: (next: string[]) => void; + maxEntries?: number; + disabled?: boolean; + placeholder?: string; + className?: string; + "aria-label"?: string; +} + +export function StopSequencesInput({ + value, + onChange, + maxEntries = 4, + disabled, + placeholder = "Add stop sequence", + className, + "aria-label": ariaLabel, +}: StopSequencesInputProps) { + const [draft, setDraft] = useState(""); + const atCap = value.length >= maxEntries; + + function commitDraft() { + // Preserve whitespace exactly (stops are byte-exact); llama-server + // accepts "\n\n" for blank-line halts. Anthropic strips whitespace + // entries on the wire. + if (!draft) return; + if (atCap) return; + if (value.includes(draft)) { + setDraft(""); + return; + } + onChange([...value, draft]); + setDraft(""); + } + + function removeChip(index: number) { + if (disabled) return; + onChange(value.filter((_, i) => i !== index)); + } + + function handleKeyDown(event: KeyboardEvent) { + if (disabled) return; + if (event.key === "Enter" || event.key === ",") { + event.preventDefault(); + commitDraft(); + return; + } + if (event.key === "Backspace" && !draft && value.length > 0) { + event.preventDefault(); + onChange(value.slice(0, -1)); + } + } + + return ( +
+ {value.map((entry, index) => ( + + {entry} + {!disabled ? ( + + ) : null} + + ))} + setDraft(event.target.value)} + onKeyDown={handleKeyDown} + onBlur={commitDraft} + placeholder={atCap ? `Max ${maxEntries} stops` : placeholder} + disabled={disabled || atCap} + aria-label={ariaLabel || placeholder} + className={cn( + "h-6 min-w-[8ch] flex-1 border-0 bg-transparent p-0 text-sm shadow-none", + "focus-visible:ring-0 focus-visible:border-0", + )} + /> +
+ ); +} diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index a74a4a2dea..11ef3b4700 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -1806,6 +1806,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { ); const externalCapabilities = getProviderCapabilities( externalProvider?.providerType, + externalSelection?.modelId, ); const externalReasoningCaps: ReturnType< typeof getExternalReasoningCapabilities @@ -1990,6 +1991,166 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { ...(externalCapabilities?.presencePenalty ? { presence_penalty: params.presencePenalty } : {}), + // Optional sampling extras. Per-provider gates live in + // provider-capabilities.ts; backend drops unknown fields. + ...(externalCapabilities?.frequencyPenalty + ? { frequency_penalty: params.frequencyPenalty } + : {}), + ...(externalCapabilities?.seed && params.seed !== null + ? { seed: params.seed } + : {}), + ...(externalCapabilities?.stop && params.stop.length > 0 + ? { stop: params.stop } + : {}), + ...(externalCapabilities?.serviceTier && params.serviceTier + ? { service_tier: params.serviceTier } + : {}), + // Upstream default is true on every shipped provider; + // forward only on explicit opt-out. + ...(externalCapabilities?.parallelToolCalls && + params.parallelToolCalls === false + ? { parallel_tool_calls: false } + : {}), + // llama.cpp / vLLM / OpenRouter extras: cap-flag plus + // non-default value gates so only meaningful knobs hit wire. + ...(externalCapabilities?.typicalP && + params.typicalP !== null && + params.typicalP !== 1 + ? { typical_p: params.typicalP } + : {}), + ...(externalCapabilities?.topNSigma && + params.topNSigma !== null && + params.topNSigma !== -1 + ? { top_n_sigma: params.topNSigma } + : {}), + ...(externalCapabilities?.repeatLastN && + params.repeatLastN !== null + ? { repeat_last_n: params.repeatLastN } + : {}), + // Dynatemp: range>0 unlocks both fields. + ...(externalCapabilities?.dynatempRange && + params.dynatempRange !== null && + params.dynatempRange > 0 + ? { + dynatemp_range: params.dynatempRange, + ...(externalCapabilities?.dynatempExponent && + params.dynatempExponent !== null + ? { dynatemp_exponent: params.dynatempExponent } + : {}), + } + : {}), + // Mirostat: mode!=0 unlocks tau + eta. + ...(externalCapabilities?.mirostat && + params.mirostat !== null && + params.mirostat !== 0 + ? { + mirostat: params.mirostat, + ...(externalCapabilities?.mirostatTau && + params.mirostatTau !== null + ? { mirostat_tau: params.mirostatTau } + : {}), + ...(externalCapabilities?.mirostatEta && + params.mirostatEta !== null + ? { mirostat_eta: params.mirostatEta } + : {}), + } + : {}), + ...(externalCapabilities?.topA && + params.topA !== null && + params.topA > 0 + ? { top_a: params.topA } + : {}), + // DRY: multiplier>0 unlocks the 4-field chain. + ...(externalCapabilities?.dryMultiplier && + params.dryMultiplier !== null && + params.dryMultiplier > 0 + ? { + dry_multiplier: params.dryMultiplier, + ...(externalCapabilities?.dryBase && + params.dryBase !== null + ? { dry_base: params.dryBase } + : {}), + ...(externalCapabilities?.dryAllowedLength && + params.dryAllowedLength !== null + ? { dry_allowed_length: params.dryAllowedLength } + : {}), + ...(externalCapabilities?.dryPenaltyLastN && + params.dryPenaltyLastN !== null + ? { dry_penalty_last_n: params.dryPenaltyLastN } + : {}), + } + : {}), + // XTC: probability>0 unlocks threshold. + ...(externalCapabilities?.xtcProbability && + params.xtcProbability !== null && + params.xtcProbability > 0 + ? { + xtc_probability: params.xtcProbability, + ...(externalCapabilities?.xtcThreshold && + params.xtcThreshold !== null + ? { xtc_threshold: params.xtcThreshold } + : {}), + } + : {}), + ...(externalCapabilities?.minKeep && + params.minKeep !== null && + params.minKeep > 0 + ? { min_keep: params.minKeep } + : {}), + ...(externalCapabilities?.ignoreEos && params.ignoreEos === true + ? { ignore_eos: true } + : {}), + ...(externalCapabilities?.minTokens && + params.minTokens !== null && + params.minTokens > 0 + ? { min_tokens: params.minTokens } + : {}), + // vLLM output-shape: default true for skip/spaces, false + // for include-stop. Forward only on user opt-out. + ...(externalCapabilities?.skipSpecialTokens && + params.skipSpecialTokens === false + ? { skip_special_tokens: false } + : {}), + ...(externalCapabilities?.spacesBetweenSpecialTokens && + params.spacesBetweenSpecialTokens === false + ? { spaces_between_special_tokens: false } + : {}), + ...(externalCapabilities?.includeStopStrInOutput && + params.includeStopStrInOutput === true + ? { include_stop_str_in_output: true } + : {}), + ...(externalCapabilities?.truncatePromptTokens && + params.truncatePromptTokens !== null && + params.truncatePromptTokens > 0 + ? { truncate_prompt_tokens: params.truncatePromptTokens } + : {}), + // n_keep accepts -1 (keep all), so the gate is != 0. + ...(externalCapabilities?.nKeep && + params.nKeep !== null && + params.nKeep !== 0 + ? { n_keep: params.nKeep } + : {}), + ...(externalCapabilities?.nProbs && + params.nProbs !== null && + params.nProbs > 0 + ? { n_probs: params.nProbs } + : {}), + ...(externalCapabilities?.cachePrompt && + params.cachePrompt === false + ? { cache_prompt: false } + : {}), + ...(externalCapabilities?.returnTokens && + params.returnTokens === true + ? { return_tokens: true } + : {}), + ...(externalCapabilities?.timingsPerToken && + params.timingsPerToken === true + ? { timings_per_token: true } + : {}), + ...(externalCapabilities?.postSamplingProbs && + params.postSamplingProbs === true + ? { post_sampling_probs: true } + : {}), // Compose the enabled_tools list from the active pills; // backend maps each name to the provider's tool schema. ...(webSearchEnabledForThisTurn || @@ -2045,9 +2206,8 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { isPromptCacheTtl(externalProvider.promptCacheTtl) ? { prompt_cache_ttl: externalProvider.promptCacheTtl } : {}), - // Anthropic fast mode (Opus 4.6 / 4.7 only); backend - // silently drops on unsupported models as a second - // line of defence. + // Fast mode (Anthropic Opus 4.6 / 4.7). Backend drops on + // unsupported models as second defence. ...(params.fastMode && providerSupportsFastMode( externalProvider.providerType, @@ -2080,6 +2240,107 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { min_p: params.minP, repetition_penalty: params.repetitionPenalty, presence_penalty: params.presencePenalty, + // llama-server (_build_passthrough_payload) accepts OAI + // extras and ignores unknown; parallel_tool_calls default + // false upstream so forward unconditionally. + ...(params.frequencyPenalty !== 0 + ? { frequency_penalty: params.frequencyPenalty } + : {}), + ...(params.seed !== null ? { seed: params.seed } : {}), + ...(params.stop.length > 0 ? { stop: params.stop } : {}), + ...(params.typicalP !== null && params.typicalP !== 1 + ? { typical_p: params.typicalP } + : {}), + ...(params.topNSigma !== null && params.topNSigma !== -1 + ? { top_n_sigma: params.topNSigma } + : {}), + ...(params.repeatLastN !== null + ? { repeat_last_n: params.repeatLastN } + : {}), + ...(params.dynatempRange !== null && params.dynatempRange > 0 + ? { + dynatemp_range: params.dynatempRange, + ...(params.dynatempExponent !== null + ? { dynatemp_exponent: params.dynatempExponent } + : {}), + } + : {}), + ...(params.mirostat !== null && params.mirostat !== 0 + ? { + mirostat: params.mirostat, + ...(params.mirostatTau !== null + ? { mirostat_tau: params.mirostatTau } + : {}), + ...(params.mirostatEta !== null + ? { mirostat_eta: params.mirostatEta } + : {}), + } + : {}), + // DRY: multiplier>0 unlocks the 4-field chain. + ...(params.dryMultiplier !== null && params.dryMultiplier > 0 + ? { + dry_multiplier: params.dryMultiplier, + ...(params.dryBase !== null + ? { dry_base: params.dryBase } + : {}), + ...(params.dryAllowedLength !== null + ? { dry_allowed_length: params.dryAllowedLength } + : {}), + ...(params.dryPenaltyLastN !== null + ? { dry_penalty_last_n: params.dryPenaltyLastN } + : {}), + } + : {}), + // XTC: probability>0 unlocks threshold. + ...(params.xtcProbability !== null && params.xtcProbability > 0 + ? { + xtc_probability: params.xtcProbability, + ...(params.xtcThreshold !== null + ? { xtc_threshold: params.xtcThreshold } + : {}), + } + : {}), + ...(params.minKeep !== null && params.minKeep > 0 + ? { min_keep: params.minKeep } + : {}), + ...(params.ignoreEos === true ? { ignore_eos: true } : {}), + ...(params.minTokens !== null && params.minTokens > 0 + ? { min_tokens: params.minTokens } + : {}), + // Forward only on non-default; per-backend cap-gates wire visibility. + ...(params.skipSpecialTokens === false + ? { skip_special_tokens: false } + : {}), + ...(params.spacesBetweenSpecialTokens === false + ? { spaces_between_special_tokens: false } + : {}), + ...(params.includeStopStrInOutput === true + ? { include_stop_str_in_output: true } + : {}), + ...(params.truncatePromptTokens !== null && + params.truncatePromptTokens > 0 + ? { truncate_prompt_tokens: params.truncatePromptTokens } + : {}), + ...(params.nKeep !== null && params.nKeep !== 0 + ? { n_keep: params.nKeep } + : {}), + ...(params.nProbs !== null && params.nProbs > 0 + ? { n_probs: params.nProbs } + : {}), + ...(params.cachePrompt === false ? { cache_prompt: false } : {}), + ...(params.returnTokens === true ? { return_tokens: true } : {}), + ...(params.timingsPerToken === true + ? { timings_per_token: true } + : {}), + ...(params.postSamplingProbs === true + ? { post_sampling_probs: true } + : {}), + // Forward only on explicit opt-out (default true on every + // backend; default omit keeps wire-shape stable for users + // who never opened the new settings panel). + ...(params.parallelToolCalls === false + ? { parallel_tool_calls: false } + : {}), image_base64: imageBase64, audio_base64: audioBase64, cancel_id: cancelId, diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx index c4a5f34273..c8002d0a6c 100644 --- a/studio/frontend/src/features/chat/chat-page.tsx +++ b/studio/frontend/src/features/chat/chat-page.tsx @@ -698,7 +698,10 @@ export function ChatPage(): ReactElement { const provider = externalProvidersForChat.find( (p) => p.id === selection.providerId, ); - const baseCapabilities = getProviderCapabilities(provider?.providerType); + const baseCapabilities = getProviderCapabilities( + provider?.providerType, + selection.modelId, + ); if (!baseCapabilities) return baseCapabilities; const anthropicThinkingEnabled = provider?.providerType === "anthropic" && diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index 8c852b8189..ab6d68e3b2 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -87,13 +87,17 @@ import { type ProviderCapabilities, getExternalMaxOutputTokens, getExternalMinOutputTokens, + getProviderStopMax, + getServiceTierOptions, providerSupportsBuiltinCodeExecution, providerSupportsFastMode, } from "./provider-capabilities"; +import { StopSequencesInput } from "@/components/ui/stop-sequences-input"; import { useChatRuntimeStore } from "./stores/chat-runtime-store"; import { ChatMcpServersDialog } from "./chat-mcp-servers-dialog"; import { listMcpServers } from "./api/mcp-servers-api"; -import type { InferenceParams } from "./types/runtime"; +import type { InferenceParams, ServiceTier } from "./types/runtime"; +import { Input } from "@/components/ui/input"; export { defaultInferenceParams, type Preset } from "./presets/preset-policy"; export type { InferenceParams } from "./types/runtime"; @@ -129,21 +133,11 @@ export function InfoHint({ children }: { children: ReactNode }) { ); } -/** - * Editable numeric value display. - * - * Renders as a single that *looks* like text by default — - * transparent background, no border, no ring — and only shows a faint - * surface tint on hover/focus to signal editability. When unfocused, - * the input shows the formatted display string (`displayValue ?? value`, - * so labels like "Off" / "Max" still render); on focus, it switches to - * the raw numeric value, selects it, and accepts free text input. - * Commit happens on blur or Enter; Escape reverts. The clamp-to-range - * happens on commit so users can type intermediate values without the - * input fighting them mid-keystroke. Single component shared by every - * slider value and the Context Length input so the click-to-edit - * affordance is consistent across the panel. - */ +/** Editable numeric value display: transparent text-like input that + * shows formatted display on blur (so "Off"/"Max" labels render) and + * switches to the raw number on focus. Commits on blur/Enter, reverts + * on Escape, clamps on commit. Shared by every slider value + the + * Context Length input. */ function snapToStep( value: number, step: number, @@ -407,10 +401,14 @@ export function ChatSettingsPanel({ externalProviderType = null, onReloadModel, }: ChatSettingsPanelProps) { - // For non-external (local) models we show every knob — providerCapabilities - // is only consulted when `isExternalModel` is true. An external model with an - // unknown provider falls back to the OpenAI-compat shape via - // getProviderCapabilities, so these flags never undercount support. + const isMobile = useIsMobile(); + const isGguf = useChatRuntimeStore((s) => s.activeGgufVariant) != null; + // Local models show every knob (providerCapabilities only consulted + // when isExternalModel; unknown providers fall back to OPENAI_COMPAT_BASE). + // GGUF llama-server honours frequency_penalty/seed/stop/parallel_tool_calls; + // HF transformers path doesn't, so hide on local non-GGUF (stale params + // are harmlessly ignored by the safetensors worker). + const localSamplerSupportsExtras = !isExternalModel ? isGguf : true; const showTemperature = !isExternalModel || Boolean(providerCapabilities?.temperature); const showTopP = !isExternalModel || Boolean(providerCapabilities?.topP); @@ -420,8 +418,60 @@ export function ChatSettingsPanel({ !isExternalModel || Boolean(providerCapabilities?.repetitionPenalty); const showPresencePenalty = !isExternalModel || Boolean(providerCapabilities?.presencePenalty); - const isMobile = useIsMobile(); - const isGguf = useChatRuntimeStore((s) => s.activeGgufVariant) != null; + const showFrequencyPenalty = isExternalModel + ? Boolean(providerCapabilities?.frequencyPenalty) + : localSamplerSupportsExtras; + const showSeed = isExternalModel + ? Boolean(providerCapabilities?.seed) + : localSamplerSupportsExtras; + const showStop = isExternalModel + ? Boolean(providerCapabilities?.stop) + : localSamplerSupportsExtras; + const showServiceTier = + isExternalModel && Boolean(providerCapabilities?.serviceTier); + const showParallelToolCalls = isExternalModel + ? Boolean(providerCapabilities?.parallelToolCalls) + : localSamplerSupportsExtras; + // Extended samplers: external uses cap flag, local is GGUF-only. + const capAdv = (k: keyof ProviderCapabilities): boolean => + isExternalModel + ? Boolean(providerCapabilities?.[k]) + : localSamplerSupportsExtras; + const advCaps = { + typicalP: capAdv("typicalP"), + topNSigma: capAdv("topNSigma"), + repeatLastN: capAdv("repeatLastN"), + dynatempRange: capAdv("dynatempRange"), + dynatempExponent: capAdv("dynatempExponent"), + mirostat: capAdv("mirostat"), + mirostatTau: capAdv("mirostatTau"), + mirostatEta: capAdv("mirostatEta"), + topA: capAdv("topA"), + dryMultiplier: capAdv("dryMultiplier"), + dryBase: capAdv("dryBase"), + dryAllowedLength: capAdv("dryAllowedLength"), + dryPenaltyLastN: capAdv("dryPenaltyLastN"), + xtcProbability: capAdv("xtcProbability"), + xtcThreshold: capAdv("xtcThreshold"), + minKeep: capAdv("minKeep"), + ignoreEos: capAdv("ignoreEos"), + minTokens: capAdv("minTokens"), + skipSpecialTokens: capAdv("skipSpecialTokens"), + spacesBetweenSpecialTokens: capAdv("spacesBetweenSpecialTokens"), + includeStopStrInOutput: capAdv("includeStopStrInOutput"), + truncatePromptTokens: capAdv("truncatePromptTokens"), + nKeep: capAdv("nKeep"), + nProbs: capAdv("nProbs"), + cachePrompt: capAdv("cachePrompt"), + returnTokens: capAdv("returnTokens"), + timingsPerToken: capAdv("timingsPerToken"), + postSamplingProbs: capAdv("postSamplingProbs"), + }; + const showAdvancedSamplingSection = Object.values(advCaps).some(Boolean); + // Per-provider stop cap; backend re-trims on the wire if a stale + // UI sends more than the upstream accepts. + const stopMaxEntries = getProviderStopMax(externalProviderType); + const serviceTierOptions = getServiceTierOptions(externalProviderType); const hasModelContent = !isExternalModel && (isGguf || Boolean(params.checkpoint)); const speculativeType = useChatRuntimeStore((s) => s.speculativeType); @@ -1291,6 +1341,143 @@ export function ChatSettingsPanel({ info="Penalizes any token that has already appeared at least once, encouraging the model to introduce new topics. 0 = off." /> ) : null} + {showFrequencyPenalty ? ( + + ) : null} + {showSeed ? ( +
+
+ + Seed + + + Best-effort determinism. OpenAI Chat and OAI-compat + local backends honor it; OpenAI Responses and Anthropic + silently drop it. + +
+ { + const raw = event.target.value; + if (raw === "") { + set("seed")(null); + return; + } + const parsed = Number.parseInt(raw, 10); + if (Number.isFinite(parsed)) { + set("seed")(parsed); + } + }} + className="h-8 w-[124px] shrink-0 text-right font-mono text-xs" + aria-label="Seed" + /> +
+ ) : null} + {showStop ? ( +
+
+ + Stop sequences + + + Strings that halt generation. Enter or comma to commit. + Maps to `stop_sequences` (Anthropic) / `stop` (OpenAI, + cap 4). + +
+ +
+ ) : null} + {showServiceTier ? ( +
+
+ + Service tier + + + Provider routing tier. `auto` = provider default. + OpenAI: flex / priority / scale. Anthropic: + `standard_only` opts out of Priority Tier. + +
+ +
+ ) : null} + {showParallelToolCalls ? ( +
+
+ + Parallel tool calls + + + Allow multiple tool calls per turn (default). + Anthropic uses inverse `disable_parallel_tool_use`. + +
+ +
+ ) : null} {!isExternalModel && !isGguf && ( )} + {showAdvancedSamplingSection ? ( + +
+ {advCaps.typicalP ? ( + + set("typicalP")(v >= 1 ? null : v) + } + displayValue={ + params.typicalP == null || params.typicalP >= 1 + ? "Off" + : undefined + } + info="llama.cpp `typ_p`. Locally typical sampling. 1.0 = off." + /> + ) : null} + {advCaps.topNSigma ? ( + + set("topNSigma")(v <= -1 ? null : v) + } + displayValue={ + params.topNSigma == null || params.topNSigma <= -1 + ? "Off" + : undefined + } + info="llama.cpp `top_n_sigma`. Sigma-based truncation. -1 = off." + /> + ) : null} + {advCaps.repeatLastN ? ( + + set("repeatLastN")(v === 0 ? null : v) + } + displayValue={ + params.repeatLastN == null + ? "Off" + : params.repeatLastN === -1 + ? "Ctx" + : undefined + } + info="llama.cpp `repeat_last_n`. Token window the repetition penalty considers. 0 = off, -1 = full context." + /> + ) : null} + {advCaps.dynatempRange ? ( + + set("dynatempRange")(v === 0 ? null : v) + } + displayValue={ + params.dynatempRange == null || params.dynatempRange === 0 + ? "Off" + : undefined + } + info="llama.cpp `dynatemp_range`. Dynamic temperature swing around base temperature. 0 = off." + /> + ) : null} + {advCaps.dynatempExponent ? ( + set("dynatempExponent")(v)} + info="llama.cpp `dynatemp_exponent`. Curve exponent, pairs with Dynatemp Range." + /> + ) : null} + {advCaps.mirostat ? ( + + set("mirostat")(v === 0 ? null : v) + } + displayValue={ + params.mirostat == null || params.mirostat === 0 + ? "Off" + : params.mirostat === 1 + ? "v1" + : "v2" + } + info="llama.cpp `mirostat`. Target-entropy sampler. 0 = off, 1 = Mirostat v1, 2 = Mirostat v2." + /> + ) : null} + {advCaps.mirostatTau ? ( + set("mirostatTau")(v)} + info="llama.cpp `mirostat_tau`. Target entropy. Higher = more diverse." + /> + ) : null} + {advCaps.mirostatEta ? ( + set("mirostatEta")(v)} + info="llama.cpp `mirostat_eta`. Learning rate for the entropy controller." + /> + ) : null} + {advCaps.topA ? ( + + set("topA")(v === 0 ? null : v) + } + displayValue={ + params.topA == null || params.topA === 0 ? "Off" : undefined + } + info="OpenRouter `top_a`. Tail-cut sampler scaled by the top token's probability. 0 = off." + /> + ) : null} + {advCaps.dryMultiplier ? ( + + set("dryMultiplier")(v === 0 ? null : v) + } + displayValue={ + params.dryMultiplier == null || params.dryMultiplier === 0 + ? "Off" + : undefined + } + info="llama.cpp DRY master switch (unlocks base / allowed length / penalty last N). 0 = off." + /> + ) : null} + {advCaps.dryBase && (params.dryMultiplier ?? 0) > 0 ? ( + set("dryBase")(v)} + info="llama.cpp `dry_base`. Exponential base for the DRY penalty. Default 1.75." + /> + ) : null} + {advCaps.dryAllowedLength && (params.dryMultiplier ?? 0) > 0 ? ( + set("dryAllowedLength")(v)} + info="llama.cpp `dry_allowed_length`. Repeats up to this length are not penalised. Default 2." + /> + ) : null} + {advCaps.dryPenaltyLastN && (params.dryMultiplier ?? 0) > 0 ? ( + + set("dryPenaltyLastN")(v === 0 ? null : v) + } + displayValue={ + params.dryPenaltyLastN == null + ? "Off" + : params.dryPenaltyLastN === -1 + ? "Ctx" + : undefined + } + info="llama.cpp `dry_penalty_last_n`. Token window the DRY penalty considers. 0 = off, -1 = full context." + /> + ) : null} + {advCaps.xtcProbability ? ( + + set("xtcProbability")(v === 0 ? null : v) + } + displayValue={ + params.xtcProbability == null || params.xtcProbability === 0 + ? "Off" + : undefined + } + info="llama.cpp XTC (eXclude Top Choices). Master switch. 0 = off." + /> + ) : null} + {advCaps.xtcThreshold && (params.xtcProbability ?? 0) > 0 ? ( + set("xtcThreshold")(v)} + info="llama.cpp `xtc_threshold`. Minimum probability for a token to be removable by XTC. Default 0.1." + /> + ) : null} + {advCaps.minKeep ? ( + + set("minKeep")(v === 0 ? null : v) + } + displayValue={ + params.minKeep == null || params.minKeep === 0 + ? "Off" + : undefined + } + info="llama.cpp `min_keep`. Minimum tokens retained past all sampler filters." + /> + ) : null} + {advCaps.minTokens ? ( + + set("minTokens")(v === 0 ? null : v) + } + displayValue={ + params.minTokens == null || params.minTokens === 0 + ? "Off" + : undefined + } + info="llama.cpp + vLLM. Minimum tokens before stop / EOS can fire." + /> + ) : null} + {advCaps.truncatePromptTokens ? ( + + set("truncatePromptTokens")(v === 0 ? null : v) + } + displayValue={ + params.truncatePromptTokens == null || + params.truncatePromptTokens === 0 + ? "Off" + : undefined + } + info="vLLM `truncate_prompt_tokens`. Left-truncate the prompt to this many tokens. 0 = off." + /> + ) : null} + {advCaps.nKeep ? ( + + set("nKeep")(v === 0 ? null : v) + } + displayValue={ + params.nKeep == null + ? "Off" + : params.nKeep === -1 + ? "All" + : undefined + } + info="llama.cpp `n_keep`. Tokens to retain when the context is shifted. 0 = off, -1 = keep all." + /> + ) : null} + {advCaps.nProbs ? ( + + set("nProbs")(v === 0 ? null : v) + } + displayValue={ + params.nProbs == null || params.nProbs === 0 + ? "Off" + : undefined + } + info="llama.cpp `n_probs`. Return the top-N token probabilities per token (diagnostic)." + /> + ) : null} + {advCaps.ignoreEos ? ( +
+
+ + Ignore EOS + + + llama.cpp + vLLM. Keep generating past the model's + end-of-sequence token. Useful for forcing long replies. + +
+ set("ignoreEos")(v ? true : null)} + aria-label="Ignore EOS" + /> +
+ ) : null} + {advCaps.skipSpecialTokens ? ( +
+
+ + Skip Special Tokens + + + vLLM `skip_special_tokens` (default on). Off keeps + chat-template markers like `<|im_end|>` in output. + +
+ + set("skipSpecialTokens")(v ? null : false) + } + aria-label="Skip special tokens" + /> +
+ ) : null} + {advCaps.spacesBetweenSpecialTokens ? ( +
+
+ + Spaces Between Special Tokens + + + vLLM `spaces_between_special_tokens`. Default on. + +
+ + set("spacesBetweenSpecialTokens")(v ? null : false) + } + aria-label="Spaces between special tokens" + /> +
+ ) : null} + {advCaps.includeStopStrInOutput ? ( +
+
+ + Include Stop String + + + vLLM `include_stop_str_in_output`. Echo the matched stop + string back in the response (useful for agentic tools). + +
+ + set("includeStopStrInOutput")(v ? true : null) + } + aria-label="Include stop string in output" + /> +
+ ) : null} + {advCaps.cachePrompt ? ( +
+
+ + Cache Prompt + + + llama.cpp `cache_prompt`. Default on. Reuses the KV cache + across requests with shared prefixes. + +
+ + set("cachePrompt")(v ? null : false) + } + aria-label="Cache prompt" + /> +
+ ) : null} + {advCaps.returnTokens ? ( +
+
+ + Return Tokens + + + llama.cpp `return_tokens`. Include the raw token ids in + the response (debug). + +
+ + set("returnTokens")(v ? true : null) + } + aria-label="Return tokens" + /> +
+ ) : null} + {advCaps.timingsPerToken ? ( +
+
+ + Timings Per Token + + + llama.cpp `timings_per_token`. Per-token wall-clock + timings in the response (perf debug). + +
+ + set("timingsPerToken")(v ? true : null) + } + aria-label="Timings per token" + /> +
+ ) : null} + {advCaps.postSamplingProbs ? ( +
+
+ + Post-Sampling Probs + + + llama.cpp `post_sampling_probs`. Report the + post-sampling distribution (sampler debug). + +
+ + set("postSamplingProbs")(v ? true : null) + } + aria-label="Post-sampling probs" + /> +
+ ) : null} +
+ + ) : null} + {!isExternalModel ? (
diff --git a/studio/frontend/src/features/chat/presets/preset-policy.ts b/studio/frontend/src/features/chat/presets/preset-policy.ts index 4efbb74f11..54407803be 100644 --- a/studio/frontend/src/features/chat/presets/preset-policy.ts +++ b/studio/frontend/src/features/chat/presets/preset-policy.ts @@ -13,6 +13,11 @@ export interface Preset { params: InferenceParams; } +// Fields that belong to a preset. Sampling knobs are included so a +// user can save a preset that fixes their preferred decoding style. +// Operational knobs (serviceTier, parallelToolCalls) and per-request +// determinism state (seed) are intentionally excluded so switching +// presets does not silently change request routing. export type PresetOwnedParams = Pick< InferenceParams, | "temperature" @@ -21,6 +26,8 @@ export type PresetOwnedParams = Pick< | "minP" | "repetitionPenalty" | "presencePenalty" + | "frequencyPenalty" + | "stop" | "maxTokens" | "systemPrompt" >; @@ -103,11 +110,22 @@ export function getPresetOwnedParams( minP: params.minP, repetitionPenalty: params.repetitionPenalty, presencePenalty: params.presencePenalty, + frequencyPenalty: params.frequencyPenalty, + stop: params.stop, maxTokens: params.maxTokens, systemPrompt: params.systemPrompt, }; } +function stopArraysEqual(a: string[], b: string[]): boolean { + if (a === b) return true; + if (a.length !== b.length) return false; + for (let i = 0; i < a.length; i += 1) { + if (a[i] !== b[i]) return false; + } + return true; +} + export function isSamePresetConfig( a: InferenceParams, b: InferenceParams, @@ -121,6 +139,8 @@ export function isSamePresetConfig( left.minP === right.minP && left.repetitionPenalty === right.repetitionPenalty && left.presencePenalty === right.presencePenalty && + left.frequencyPenalty === right.frequencyPenalty && + stopArraysEqual(left.stop, right.stop) && left.maxTokens === right.maxTokens && left.systemPrompt === right.systemPrompt ); diff --git a/studio/frontend/src/features/chat/provider-capabilities.ts b/studio/frontend/src/features/chat/provider-capabilities.ts index f550279826..58341850d9 100644 --- a/studio/frontend/src/features/chat/provider-capabilities.ts +++ b/studio/frontend/src/features/chat/provider-capabilities.ts @@ -1,34 +1,126 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -/** - * Per-provider sampling parameter capability matrix. - * - * Values are derived from each provider's published chat-completion docs as of - * 2026-05. They describe which of our UI knobs map cleanly onto the provider's - * request body; the panel hides params a provider does not accept so users - * cannot dial a value that gets silently dropped or rejected. - * - * "Local" models (anything that is not an external provider) are represented by - * a null capability — every knob renders for them. - */ - +// Per-provider sampling capability matrix sourced from each provider's +// chat-completion docs (2026-05); panel hides params the active +// provider would silently drop or reject. +// New knobs: default to false on every SaaS bucket; only local + +// openrouter expose llama.cpp samplers. export interface ProviderCapabilities { - /** - * Temperature sampling. Reasoning-class models (OpenAI's gpt-5.x / o3 via - * /v1/responses) reject this with `Unsupported parameter`. - */ + /** OpenAI gpt-5.x / o-series reject via /v1/responses. */ temperature: boolean; - /** Nucleus (top_p) sampling. Same restriction as `temperature` on OpenAI. */ topP: boolean; - /** top-k token sampling (only Anthropic on the providers we ship). */ + /** Anthropic only among SaaS providers. */ topK: boolean; - /** min-p token cutoff (no SaaS provider currently exposes this). */ minP: boolean; - /** Repetition penalty (no SaaS provider currently exposes this). */ repetitionPenalty: boolean; - /** OpenAI-style presence penalty. */ presencePenalty: boolean; + /** OAI Chat only; rejected by Responses + Anthropic. */ + frequencyPenalty: boolean; + /** OAI Chat + OAI-compat. Responses + Anthropic drop. */ + seed: boolean; + /** Not accepted by Responses; mapped to `stop_sequences` on Anthropic. */ + stop: boolean; + /** Per-provider enum, see getServiceTierOptions. */ + serviceTier: boolean; + /** Anthropic inverts to `disable_parallel_tool_use`. */ + parallelToolCalls: boolean; + /** llama.cpp `typ_p`. */ + typicalP: boolean; + /** llama.cpp `top_n_sigma`. */ + topNSigma: boolean; + /** llama.cpp `repeat_last_n`. */ + repeatLastN: boolean; + /** llama.cpp `dynatemp_range`. */ + dynatempRange: boolean; + /** llama.cpp `dynatemp_exponent`. */ + dynatempExponent: boolean; + /** llama.cpp `mirostat` (0/1/2). */ + mirostat: boolean; + mirostatTau: boolean; + mirostatEta: boolean; + /** OpenRouter `top_a`. https://openrouter.ai/docs/api/reference/parameters */ + topA: boolean; + /** llama.cpp DRY (4 fields). dryMultiplier is the master switch. */ + dryMultiplier: boolean; + dryBase: boolean; + dryAllowedLength: boolean; + dryPenaltyLastN: boolean; + /** llama.cpp XTC (2 fields). xtcProbability is the master switch. */ + xtcProbability: boolean; + xtcThreshold: boolean; + /** llama.cpp `min_keep`. */ + minKeep: boolean; + /** llama.cpp + vLLM. Ollama OAI translator drops it. */ + ignoreEos: boolean; + /** llama.cpp + vLLM. Ollama OAI translator drops it. */ + minTokens: boolean; + /** vLLM only. */ + skipSpecialTokens: boolean; + spacesBetweenSpecialTokens: boolean; + /** vLLM only. Useful for agentic tools. */ + includeStopStrInOutput: boolean; + /** vLLM only. Left-truncate the prompt. */ + truncatePromptTokens: boolean; + /** llama.cpp `n_keep` / `n_probs`. */ + nKeep: boolean; + nProbs: boolean; + /** llama.cpp `cache_prompt`. */ + cachePrompt: boolean; + /** llama.cpp debug flags. */ + returnTokens: boolean; + timingsPerToken: boolean; + postSamplingProbs: boolean; +} + +/** + * Per-provider stop-sequence max count. Mirrors backend `stop_max`. + * openai 4 (Chat hard cap; Responses drops stop) + * anthropic 16 (client-side guard, docs no max) + * kimi 5 (https://platform.kimi.ai/docs/api/chat) + * deepseek 16 (https://api-docs.deepseek.com/api/create-chat-completion) + * mistral 16, gemini 4, openrouter 4, default 16 (ollama/vllm/llama.cpp/custom) + */ +const PROVIDER_STOP_MAX: Record = { + openai: 4, + anthropic: 16, + kimi: 5, + deepseek: 16, + mistral: 16, + gemini: 4, + openrouter: 4, +}; + +export function getProviderStopMax( + providerType: string | null | undefined, +): number { + if (!providerType) return 16; // local backends + return PROVIDER_STOP_MAX[providerType] ?? 16; +} + +export type ServiceTierOption = + | "auto" + | "default" + | "flex" + | "priority" + | "scale" + | "standard_only"; + +/** + * Legal `service_tier` per provider. anthropic=auto|standard_only; + * openai (/v1/responses)=auto|default|flex|priority (scale excluded + * though SDK lists it); others fall through to auto|default. + */ +export function getServiceTierOptions( + providerType: string | null | undefined, +): readonly ServiceTierOption[] { + if (providerType === "anthropic") { + return ["auto", "standard_only"] as const; + } + if (providerType === "openai") { + return ["auto", "default", "flex", "priority"] as const; + } + return ["auto", "default"] as const; } export type ExternalReasoningCapabilities = { @@ -76,47 +168,45 @@ export function clampReasoningEffortToLevels( */ export const EXTERNAL_MAX_OUTPUT_TOKENS = 32768; -/** - * Per-model max-output caps from each provider's docs: - * OpenAI: developers.openai.com/api/docs/models/gpt-5.5 - * Anthropic: platform.claude.com/docs/en/about-claude/models - * Gemini: ai.google.dev/gemini-api/docs/models/gemini-3.1-pro-preview - * DeepSeek: api-docs.deepseek.com/quick_start/pricing (V4 family) - * Local-model path is unaffected. - */ +// Per-model max-output caps (verified May 2026). Longer prefixes first +// so .startsWith() picks the specific id over the family root. +// OpenAI: developers.openai.com/api/docs/models/ +// Anthropic: platform.claude.com/docs/en/about-claude/models/overview +// Gemini: ai.google.dev/gemini-api/docs/models +// DeepSeek: api-docs.deepseek.com/quick_start/pricing const EXTERNAL_MAX_OUTPUT_TOKENS_BY_MODEL: Array<{ providerType: string; prefixes: readonly string[]; cap: number; }> = [ - // OpenAI - { providerType: "openai", prefixes: ["gpt-5.5-pro", "gpt-5.5"], cap: 128000 }, - { providerType: "openai", prefixes: ["gpt-5.4-pro", "gpt-5.4"], cap: 65536 }, - { providerType: "openai", prefixes: ["gpt-5.3"], cap: 16384 }, - // Anthropic + { providerType: "openai", prefixes: ["gpt-5.3-chat-latest", "gpt-5.1-chat"], cap: 16384 }, + { providerType: "openai", prefixes: ["gpt-5"], cap: 128000 }, + { providerType: "openai", prefixes: ["o1", "o3", "o4", "codex-mini"], cap: 100000 }, + // Anthropic Opus 4.6 + 4.7 ship 128k Max output; Sonnet 4.5/4.6/4 + + // Opus 4.5 + Haiku 4.5 ship 64k; Opus 4.1 / Opus 4 fall through to + // the 32k EXTERNAL_MAX_OUTPUT_TOKENS default. { providerType: "anthropic", - prefixes: ["claude-opus-4-7"], + prefixes: ["claude-opus-4-7", "claude-opus-4-6"], cap: 128000, }, { providerType: "anthropic", prefixes: [ - "claude-opus-4-6", "claude-sonnet-4-6", "claude-opus-4-5", "claude-sonnet-4-5", "claude-haiku-4-5", + "claude-sonnet-4", ], cap: 64000, }, - // Gemini { providerType: "gemini", prefixes: ["gemini-3", "gemini-pro", "gemini-flash"], cap: 65536, }, - // DeepSeek (V4: deepseek-chat / deepseek-reasoner alias V4-flash). + // V4: deepseek-chat / deepseek-reasoner alias V4-flash. { providerType: "deepseek", prefixes: ["deepseek"], cap: 384000 }, ]; @@ -160,33 +250,13 @@ function _inferProviderFromOpenrouterId( return null; } -/** - * Whether the external provider offers a built-in web-search tool that the - * model invokes server-side. When `true`, the chat composer's Search button - * is available for that provider and the chat-adapter forwards - * `enable_tools: true, enabled_tools: ["web_search"]` on the request — the - * backend routes the call through the provider's tool schema: - * - OpenAI: `tools: [{type: "web_search"}]` on /v1/responses - * - Anthropic: `tools: [{type: "web_search_20250305", name: "web_search", - * max_uses: 5}]` on /v1/messages - * - OpenRouter: `plugins: [{id: "web"}]` on /v1/chat/completions (the - * router's universal web-search shape; works for every - * underlying model including the `openrouter/free` router). - * - Kimi: `tools: [{type: "builtin_function", function: {name: - * "$web_search"}}]` with `thinking: {type: - * "disabled"}`. Requires a client round-trip: - * the first call returns the search args; the backend - * echoes them back as a role=tool message; the second - * call streams the answer. Handled in - * _stream_kimi_web_search on the backend. - * - * Mistral is intentionally excluded: their `web_search` connector lives on - * the Agents API (`/v1/agents` + `/v1/conversations`), not chat completions, - * and returns `"WebSearchTool connector is not supported"` if injected into - * /v1/chat/completions. Wiring it would require a dedicated Agents streaming - * path. Gemini's grounded-search can be added with the same pattern when - * matching backend translation lands. - */ +// Gates the composer's Search button. Backend maps +// enable_tools:["web_search"] to each provider's tool schema: +// OpenAI: tools:[{type:"web_search"}] on /v1/responses +// Anthropic: tools:[{type:"web_search_20250305", max_uses:5}] +// OpenRouter: plugins:[{id:"web"}] +// Kimi: $web_search builtin (2-call via _stream_kimi_web_search) +// Mistral excluded (web_search lives on Agents API, 400s on /v1/chat). export function providerSupportsBuiltinWebSearch( providerType: string | null | undefined, modelId?: string | null | undefined, @@ -218,24 +288,18 @@ export function providerSupportsBuiltinWebSearch( ); } -/** - * Whether the external provider exposes a server-side web_fetch tool - * (single URL, text or PDF) emitting a document block. Anthropic-only - * today (`web_fetch_20250910` / `web_fetch_20260209`). Gates the - * composer's standalone Fetch pill, independent of Search. - */ +// Anthropic-only server-side web_fetch tool +// (web_fetch_20250910 / _20260209). Gates the composer's Fetch pill. export function providerSupportsBuiltinWebFetch( providerType: string | null | undefined, ): boolean { return providerType === "anthropic"; } -/** - * Whether the active provider + model supports Anthropic fast-mode - * (`speed: "fast"` + `fast-mode-2026-02-01` header). Opus 4.6 / 4.7 - * only per https://platform.claude.com/docs/en/build-with-claude/fast-mode. - * Backend silently drops on unsupported models as a second defence. - */ +// Anthropic fast-mode (`speed:"fast"` + fast-mode-2026-02-01 header). +// Opus 4.6 / 4.7 only per +// https://platform.claude.com/docs/en/build-with-claude/fast-mode. +// Backend silently drops on unsupported models as a second defence. const ANTHROPIC_FAST_MODE_MODEL_PREFIXES = [ "claude-opus-4-7", "claude-opus-4-6", @@ -247,38 +311,21 @@ export function providerSupportsFastMode( ): boolean { if (providerType !== "anthropic") return false; if (!modelId) return false; - // Family boundary ("" or "-") required so IDs like "claude-opus-4-70" - // / "claude-opus-4-7b" do not match. + // Family boundary required so "claude-opus-4-70" doesn't match. return ANTHROPIC_FAST_MODE_MODEL_PREFIXES.some( (prefix) => modelId === prefix || modelId.startsWith(`${prefix}-`), ); } -/** - * Whether the selected external provider/model exposes a server-side - * code-execution tool. Two providers ship one today: - * - * - **Anthropic** (`code_execution_20250825`): Python + bash + - * str_replace-based file edits inside a 5 GB sandboxed container - * per request. Documented at - * https://platform.claude.com/docs/en/agents-and-tools/tool-use/code-execution-tool - * - * - **OpenAI cloud** (`shell` on /v1/responses): bash inside a - * reusable container; we auto-create one on the first turn of a - * chat thread and reference it on subsequent turns via the - * thread's stored `openaiCodeExecContainerId`. Documented at - * https://developers.openai.com/api/docs/guides/tools-shell - * - * Returns false for every other provider. The backend additionally - * gates the OpenAI shell tool on `is_openai_cloud` so custom - * OpenAI-compat servers (ollama / llama.cpp / vLLM) that also report - * `provider_type="openai"` never receive the tool — but in practice - * none of those catalogs surface the `gpt-5.5` ids anyway, so the - * frontend prefix match is enough. - * - * v1 wires the tools themselves; file uploads (Anthropic - * `container_upload` / OpenAI `input_file`) are a deliberate follow-up. - */ +// Server-side code-execution tools: +// Anthropic code_execution_20250825 (Python + bash + str_replace in +// a 5 GB sandbox). +// OpenAI cloud `shell` on /v1/responses (bash in a reusable container +// referenced via openaiCodeExecContainerId across turns). +// Backend also gates OpenAI on is_openai_cloud so custom OAI-compat +// servers reporting provider_type="openai" can't accidentally get the +// shell tool. File uploads (container_upload / input_file) are +// follow-up work. const ANTHROPIC_CODE_EXECUTION_MODEL_PREFIXES = [ "claude-opus-4-7", "claude-opus-4-6", @@ -356,18 +403,8 @@ export function providerSupportsBuiltinCodeExecution( return false; } -/** - * Whether the selected external provider/model exposes OpenAI's - * Responses-API server-side image_generation tool. Lit on for OpenAI - * cloud (`api.openai.com`) when the picked model is a Responses-API - * family id (gpt-5.x today). The backend additionally gates on - * `is_openai_cloud`; mirror that here so the pill is hidden on custom - * OpenAI-compat backends (ollama / llama.cpp / vLLM) that report - * `provider_type="openai"` but would 400 on a `{type:"image_generation"}` - * tool. See backend/core/inference/external_provider.py near line 2770 - * for the dispatch and backend/tests/test_openai_image_generation.py - * for the round-trip coverage. - */ +// OpenAI Responses-API image_generation. OpenAI cloud + +// Responses-family ids only; backend mirrors via is_openai_cloud. const OPENAI_IMAGE_GENERATION_MODEL_PREFIXES = [ "gpt-5.5-pro", "gpt-5.5", @@ -457,19 +494,8 @@ function geminiImageModelAllowsGoogleSearch(modelId: string): boolean { ); } -/** - * Per-provider minimum on the outbound max_tokens. Kimi's docs require - * `max_tokens >= 16000` whenever a thinking model is in use so the - * reasoning_content and final answer both fit in the budget — anything - * lower truncates the response mid-stream. Other providers don't have a - * documented floor, so they fall through to the generic min of 64 in - * the slider. - * - * The chat-adapter resolves the effective floor on send and bumps the - * outbound max_tokens up to this value if the user's stored maxTokens - * sits below it. The settings panel reflects the same floor as the - * slider min so the displayed value never drifts from what's sent. - */ +// Per-provider min on outbound max_tokens. Kimi thinking needs >=16000 +// (truncates mid-stream below); chat-adapter bumps on send. const EXTERNAL_MIN_OUTPUT_TOKENS_BY_PROVIDER: Record = { kimi: 16000, }; @@ -488,37 +514,334 @@ const OPENAI_COMPAT_BASE: ProviderCapabilities = { minP: false, repetitionPenalty: false, presencePenalty: true, + frequencyPenalty: true, + seed: true, + stop: true, + serviceTier: false, + parallelToolCalls: true, + typicalP: false, + topNSigma: false, + repeatLastN: false, + dynatempRange: false, + dynatempExponent: false, + mirostat: false, + mirostatTau: false, + mirostatEta: false, + topA: false, + dryMultiplier: false, + dryBase: false, + dryAllowedLength: false, + dryPenaltyLastN: false, + xtcProbability: false, + xtcThreshold: false, + minKeep: false, + ignoreEos: false, + minTokens: false, + skipSpecialTokens: false, + spacesBetweenSpecialTokens: false, + includeStopStrInOutput: false, + truncatePromptTokens: false, + nKeep: false, + nProbs: false, + cachePrompt: false, + returnTokens: false, + timingsPerToken: false, + postSamplingProbs: false, }; -const ALL_SUPPORTED: ProviderCapabilities = { +// Unsloth's first-party llama-server runtime (provider type `llama_cpp`) +// plus the permissive `custom` preset. Exposes the full llama.cpp +// sampler chain (typical_p / top_n_sigma / mirostat / dynatemp / +// repeat_last_n) per the upstream server README. Not used for vLLM or +// Ollama; see VLLM_OLLAMA_CAPABILITIES below. +const LLAMA_CPP_CAPABILITIES: ProviderCapabilities = { temperature: true, topP: true, topK: true, minP: true, repetitionPenalty: true, presencePenalty: true, + frequencyPenalty: true, + seed: true, + stop: true, + serviceTier: false, + parallelToolCalls: true, + typicalP: true, + topNSigma: true, + repeatLastN: true, + dynatempRange: true, + dynatempExponent: true, + mirostat: true, + mirostatTau: true, + mirostatEta: true, + topA: false, + dryMultiplier: true, + dryBase: true, + dryAllowedLength: true, + dryPenaltyLastN: true, + xtcProbability: true, + xtcThreshold: true, + minKeep: true, + ignoreEos: true, + minTokens: true, + skipSpecialTokens: false, + spacesBetweenSpecialTokens: false, + includeStopStrInOutput: false, + truncatePromptTokens: false, + nKeep: true, + nProbs: true, + cachePrompt: true, + returnTokens: true, + timingsPerToken: true, + postSamplingProbs: true, }; +// vLLM SamplingParams: OAI subset + top_k/min_p/repetition_penalty/seed +// + the 4 vLLM-only output-shape knobs. No DRY / XTC / mirostat / +// dynatemp / typical_p / min_keep / n_keep / n_probs / cache_prompt / +// debug flags (none in SamplingParams). +const VLLM_CAPABILITIES: ProviderCapabilities = { + ...LLAMA_CPP_CAPABILITIES, + typicalP: false, + topNSigma: false, + repeatLastN: false, + dynatempRange: false, + dynatempExponent: false, + mirostat: false, + mirostatTau: false, + mirostatEta: false, + dryMultiplier: false, + dryBase: false, + dryAllowedLength: false, + dryPenaltyLastN: false, + xtcProbability: false, + xtcThreshold: false, + minKeep: false, + skipSpecialTokens: true, + spacesBetweenSpecialTokens: true, + includeStopStrInOutput: true, + truncatePromptTokens: true, + nKeep: false, + nProbs: false, + cachePrompt: false, + returnTokens: false, + timingsPerToken: false, + postSamplingProbs: false, +}; + +// Ollama OAI translator (openai/openai.go FromChatRequest) only copies +// the documented OpenAI subset on /v1/chat/completions: top_k, min_p, +// repetition_penalty, ignore_eos, min_tokens, and the 4 vLLM output +// knobs all silently drop on this path. (Native /api/chat would forward +// them via `options`, but Studio uses /v1.) +const OLLAMA_CAPABILITIES: ProviderCapabilities = { + ...VLLM_CAPABILITIES, + topK: false, + minP: false, + repetitionPenalty: false, + ignoreEos: false, + minTokens: false, + skipSpecialTokens: false, + spacesBetweenSpecialTokens: false, + includeStopStrInOutput: false, + truncatePromptTokens: false, +}; + +// OpenRouter is a router-of-routers: gateway accepts a wider set of +// OAI-style fields than any single upstream and silently drops what +// the chosen route doesn't. Surface the full documented set (incl. +// top_a) and leave llama.cpp-only knobs off. +// https://openrouter.ai/docs/api/reference/parameters +const OPENROUTER_CAPABILITIES: ProviderCapabilities = { + temperature: true, + topP: true, + topK: true, + minP: true, + repetitionPenalty: true, + presencePenalty: true, + frequencyPenalty: true, + seed: true, + stop: true, + serviceTier: false, + parallelToolCalls: true, + typicalP: false, + topNSigma: false, + repeatLastN: false, + dynatempRange: false, + dynatempExponent: false, + mirostat: false, + mirostatTau: false, + mirostatEta: false, + topA: true, + dryMultiplier: false, + dryBase: false, + dryAllowedLength: false, + dryPenaltyLastN: false, + xtcProbability: false, + xtcThreshold: false, + minKeep: false, + ignoreEos: false, + minTokens: false, + skipSpecialTokens: false, + spacesBetweenSpecialTokens: false, + includeStopStrInOutput: false, + truncatePromptTokens: false, + nKeep: false, + nProbs: false, + cachePrompt: false, + returnTokens: false, + timingsPerToken: false, + postSamplingProbs: false, +}; + +// OpenAI reasoning class via /v1/responses: temperature fixed at 1, +// top_p ignored, 400s on presence/frequency_penalty/seed. Chat-class +// (gpt-4o etc) keeps the full surface even via /v1/responses. Both +// drop `stop` (Responses doesn't surface it). +// https://platform.openai.com/docs/guides/reasoning +const OPENAI_REASONING_CAPABILITIES: ProviderCapabilities = { + temperature: false, + topP: false, + topK: false, + minP: false, + repetitionPenalty: false, + presencePenalty: false, + frequencyPenalty: false, + seed: false, + stop: false, + serviceTier: true, + parallelToolCalls: true, + typicalP: false, + topNSigma: false, + repeatLastN: false, + dynatempRange: false, + dynatempExponent: false, + mirostat: false, + mirostatTau: false, + mirostatEta: false, + topA: false, + dryMultiplier: false, + dryBase: false, + dryAllowedLength: false, + dryPenaltyLastN: false, + xtcProbability: false, + xtcThreshold: false, + minKeep: false, + ignoreEos: false, + minTokens: false, + skipSpecialTokens: false, + spacesBetweenSpecialTokens: false, + includeStopStrInOutput: false, + truncatePromptTokens: false, + nKeep: false, + nProbs: false, + cachePrompt: false, + returnTokens: false, + timingsPerToken: false, + postSamplingProbs: false, +}; +const OPENAI_CHAT_CAPABILITIES: ProviderCapabilities = { + temperature: true, + topP: true, + topK: false, + minP: false, + repetitionPenalty: false, + presencePenalty: true, + frequencyPenalty: true, + seed: true, + // Responses API does not surface `stop` even for non-reasoning models; + // tracked in OpenAI's Responses-vs-ChatCompletions migration notes. + stop: false, + serviceTier: true, + parallelToolCalls: true, + typicalP: false, + topNSigma: false, + repeatLastN: false, + dynatempRange: false, + dynatempExponent: false, + mirostat: false, + mirostatTau: false, + mirostatEta: false, + topA: false, + dryMultiplier: false, + dryBase: false, + dryAllowedLength: false, + dryPenaltyLastN: false, + xtcProbability: false, + xtcThreshold: false, + minKeep: false, + ignoreEos: false, + minTokens: false, + skipSpecialTokens: false, + spacesBetweenSpecialTokens: false, + includeStopStrInOutput: false, + truncatePromptTokens: false, + nKeep: false, + nProbs: false, + cachePrompt: false, + returnTokens: false, + timingsPerToken: false, + postSamplingProbs: false, +}; + +// Prefix list for OpenAI reasoning-class model ids. Kept in sync with +// OPENAI_REASONING_MODELS below (used for reasoning_effort capability). +// Longest prefixes first so "gpt-5.5-pro" wins over "gpt-5.5". +const OPENAI_REASONING_MODEL_PREFIXES = [ + "gpt-5.5-pro", + "gpt-5.5", + "gpt-5.4-pro", + "gpt-5.4", + "gpt-5.3-chat-latest", + "gpt-5.3-codex", + "gpt-5.3", + "gpt-5.2", + "gpt-5.1", + "gpt-5", + "o1", + "o3", + "o4", +] as const; + +function isOpenAIReasoningModelId(modelId: string | null | undefined): boolean { + const normalized = modelId?.trim().toLowerCase() ?? ""; + if (!normalized) return false; + return OPENAI_REASONING_MODEL_PREFIXES.some((p) => normalized.startsWith(p)); +} + +// Mirror of backend _ANTHROPIC_4_7_SAMPLING_REMOVED. Opus 4.7 removed +// temperature/top_p/top_k; only Opus shipped in 4.7. The -4-7[-.]/EOL +// anchor keeps future families (claude-opus-5 etc) unaffected. +const ANTHROPIC_4_7_SAMPLING_REMOVED_REGEX = /^claude-opus-4-7(?:[-.]|$)/i; + +function isClaude47SamplingRemoved(modelId: string | null | undefined): boolean { + const normalized = modelId?.trim().toLowerCase() ?? ""; + if (!normalized) return false; + return ANTHROPIC_4_7_SAMPLING_REMOVED_REGEX.test(normalized); +} + +// DeepSeek reasoner ids silently ignore temperature/top_p/presence/ +// frequency and 400 on logprobs per the reasoning_model guide. Prefix +// match covers future revisions (deepseek-reasoner-2027 etc). +const DEEPSEEK_REASONING_MODEL_PREFIXES = [ + "deepseek-reasoner", + "deepseek-r1", +] as const; + +function isDeepSeekReasoningModelId(modelId: string | null | undefined): boolean { + const normalized = modelId?.trim().toLowerCase() ?? ""; + if (!normalized) return false; + return DEEPSEEK_REASONING_MODEL_PREFIXES.some((p) => normalized.startsWith(p)); +} + const PROVIDER_CAPABILITIES: Record = { - // OpenAI's flagship models (gpt-5.x / o3 / gpt-4.5) are reasoning-class - // models served via /v1/responses, which rejects temperature, top_p, and - // presence/frequency penalty. See backend - // external_provider._stream_openai_responses for the proxy. - openai: { - temperature: false, - topP: false, - topK: false, - minP: false, - repetitionPenalty: false, - presencePenalty: false, - }, - // Anthropic's Messages API accepts top_k on 3.x and 4.5/4.6, but Claude - // 4.7 (Opus/Sonnet/Haiku) deprecated it and returns 400 if it is set. - // We surface top_k in the panel for all Anthropic providers and let the - // backend strip it per-model — see _stream_anthropic in - // studio/backend/core/inference/external_provider.py. - // Presence/frequency penalty is not part of the Messages API on any - // Claude generation. + // Default to reasoning-class; getProviderCapabilities upgrades + // non-reasoning ids (gpt-4o etc) to OPENAI_CHAT_CAPABILITIES. + openai: OPENAI_REASONING_CAPABILITIES, + // Messages API: temperature/top_p/top_k/stop_sequences/service_tier + // (auto|standard_only)/disable_parallel_tool_use. Opus 4.7 strips + // temperature/top_p/top_k via the regex above. No presence/frequency + // penalty / seed / logprobs on any Claude generation. anthropic: { temperature: true, topP: true, @@ -526,14 +849,44 @@ const PROVIDER_CAPABILITIES: Record = { minP: false, repetitionPenalty: false, presencePenalty: false, + frequencyPenalty: false, + seed: false, + stop: true, + serviceTier: true, + parallelToolCalls: true, + typicalP: false, + topNSigma: false, + repeatLastN: false, + dynatempRange: false, + dynatempExponent: false, + mirostat: false, + mirostatTau: false, + mirostatEta: false, + topA: false, + dryMultiplier: false, + dryBase: false, + dryAllowedLength: false, + dryPenaltyLastN: false, + xtcProbability: false, + xtcThreshold: false, + minKeep: false, + ignoreEos: false, + minTokens: false, + skipSpecialTokens: false, + spacesBetweenSpecialTokens: false, + includeStopStrInOutput: false, + truncatePromptTokens: false, + nKeep: false, + nProbs: false, + cachePrompt: false, + returnTokens: false, + timingsPerToken: false, + postSamplingProbs: false, }, mistral: OPENAI_COMPAT_BASE, - // Gemini's native generationConfig accepts temperature, topP, topK and - // presencePenalty (plus a separate frequencyPenalty we do not surface - // today). minP and repetitionPenalty are not part of the contract -- - // see https://ai.google.dev/api/rest/v1beta/GenerationConfig. Backend - // request shaping lives in _stream_gemini in - // studio/backend/core/inference/external_provider.py. + // Gemini generationConfig: temperature/topP/topK/presencePenalty/ + // frequencyPenalty/seed/stopSequences. No minP/repetitionPenalty. + // https://ai.google.dev/api/rest/v1beta/GenerationConfig gemini: { temperature: true, topP: true, @@ -541,13 +894,43 @@ const PROVIDER_CAPABILITIES: Record = { minP: false, repetitionPenalty: false, presencePenalty: true, + frequencyPenalty: false, + seed: false, + stop: true, + serviceTier: false, + parallelToolCalls: false, + typicalP: false, + topNSigma: false, + repeatLastN: false, + dynatempRange: false, + dynatempExponent: false, + mirostat: false, + mirostatTau: false, + mirostatEta: false, + topA: false, + dryMultiplier: false, + dryBase: false, + dryAllowedLength: false, + dryPenaltyLastN: false, + xtcProbability: false, + xtcThreshold: false, + minKeep: false, + ignoreEos: false, + minTokens: false, + skipSpecialTokens: false, + spacesBetweenSpecialTokens: false, + includeStopStrInOutput: false, + truncatePromptTokens: false, + nKeep: false, + nProbs: false, + cachePrompt: false, + returnTokens: false, + timingsPerToken: false, + postSamplingProbs: false, }, - // Kimi k2.5/k2.6 are reasoning-class — the API locks temperature and - // top_p to fixed defaults and 400s on any other value: - // "invalid temperature: only 1 is allowed for this model". - // Hide both sliders so the user is not offered knobs the model - // silently overrides. Backend additionally strips these fields via - // PROVIDER_REGISTRY['kimi']['body_omit']. + // Kimi K2.x locks temperature + top_p ("only 1 is allowed for this + // model"); seed + parallel_tool_calls aren't in the Chat schema + // (platform.kimi.ai/docs/api/chat). Backend strips via body_omit. kimi: { temperature: false, topP: false, @@ -555,8 +938,45 @@ const PROVIDER_CAPABILITIES: Record = { minP: false, repetitionPenalty: false, presencePenalty: true, + // K2.x 400s on non-default frequency_penalty; backend strips too. + frequencyPenalty: false, + seed: false, + stop: true, + serviceTier: false, + parallelToolCalls: false, + typicalP: false, + topNSigma: false, + repeatLastN: false, + dynatempRange: false, + dynatempExponent: false, + mirostat: false, + mirostatTau: false, + mirostatEta: false, + topA: false, + dryMultiplier: false, + dryBase: false, + dryAllowedLength: false, + dryPenaltyLastN: false, + xtcProbability: false, + xtcThreshold: false, + minKeep: false, + ignoreEos: false, + minTokens: false, + skipSpecialTokens: false, + spacesBetweenSpecialTokens: false, + includeStopStrInOutput: false, + truncatePromptTokens: false, + nKeep: false, + nProbs: false, + cachePrompt: false, + returnTokens: false, + timingsPerToken: false, + postSamplingProbs: false, }, - // DeepSeek deprecated presence/frequency penalty in their current docs. + // DeepSeek schema (api-docs.deepseek.com/api/create-chat-completion) + // lists temperature/top_p/stop only; no seed or parallel_tool_calls. + // Presence/frequency are deprecated. Reasoner ids additionally ignore + // temperature/top_p; getProviderCapabilities downshifts them. deepseek: { temperature: true, topP: true, @@ -564,38 +984,80 @@ const PROVIDER_CAPABILITIES: Record = { minP: false, repetitionPenalty: false, presencePenalty: false, + frequencyPenalty: false, + seed: false, + stop: true, + serviceTier: false, + parallelToolCalls: false, + typicalP: false, + topNSigma: false, + repeatLastN: false, + dynatempRange: false, + dynatempExponent: false, + mirostat: false, + mirostatTau: false, + mirostatEta: false, + topA: false, + dryMultiplier: false, + dryBase: false, + dryAllowedLength: false, + dryPenaltyLastN: false, + xtcProbability: false, + xtcThreshold: false, + minKeep: false, + ignoreEos: false, + minTokens: false, + skipSpecialTokens: false, + spacesBetweenSpecialTokens: false, + includeStopStrInOutput: false, + truncatePromptTokens: false, + nKeep: false, + nProbs: false, + cachePrompt: false, + returnTokens: false, + timingsPerToken: false, + postSamplingProbs: false, }, qwen: OPENAI_COMPAT_BASE, huggingface: OPENAI_COMPAT_BASE, - // OpenRouter silently drops params the target model does not support, so we - // surface every knob and let the gateway handle the per-model fan-out. - openrouter: ALL_SUPPORTED, - // Local OpenAI-compatible connections are proxied through the OpenAI backend - // path, but vLLM/Ollama/llama.cpp users often want top_k / min_p / - // repetition controls, so be permissive. - custom: ALL_SUPPORTED, - vllm: ALL_SUPPORTED, - ollama: ALL_SUPPORTED, - llama_cpp: ALL_SUPPORTED, + openrouter: OPENROUTER_CAPABILITIES, + // llama_cpp + custom: first-party llama-server, full chain. + // vllm: OAI subset + top_k/min_p/repetition_penalty/seed. + // ollama: stricter (OAI translator drops top_k/min_p/rep_pen too). + custom: LLAMA_CPP_CAPABILITIES, + llama_cpp: LLAMA_CPP_CAPABILITIES, + vllm: VLLM_CAPABILITIES, + ollama: OLLAMA_CAPABILITIES, }; const DEFAULT_EXTERNAL_CAPABILITIES = OPENAI_COMPAT_BASE; -/** - * Resolve the capability set for an external provider. Returns `null` for - * a local model (i.e. when `providerType` is null/undefined), which callers - * should treat as "every knob applies". - */ +// Per-model overrides: openai+chat-class -> OPENAI_CHAT_CAPABILITIES; +// anthropic claude-opus-4-7 strips temp/top_p/top_k; deepseek reasoner +// hides temp/top_p. Local (no providerType) returns null = every knob. export function getProviderCapabilities( providerType: string | null | undefined, + modelId?: string | null | undefined, ): ProviderCapabilities | null { if (!providerType) return null; - return PROVIDER_CAPABILITIES[providerType] ?? DEFAULT_EXTERNAL_CAPABILITIES; + const base = PROVIDER_CAPABILITIES[providerType] ?? DEFAULT_EXTERNAL_CAPABILITIES; + if (providerType === "openai" && modelId && !isOpenAIReasoningModelId(modelId)) { + return OPENAI_CHAT_CAPABILITIES; + } + if (providerType === "anthropic" && isClaude47SamplingRemoved(modelId)) { + return { ...base, temperature: false, topP: false, topK: false }; + } + if (providerType === "deepseek" && isDeepSeekReasoningModelId(modelId)) { + return { ...base, temperature: false, topP: false }; + } + return base; } const DEFAULT_EFFORT_LEVELS = ["low", "medium", "high"] as const; +// OpenRouter ids with no non-reasoning mode. (google/gemini-pro-latest +// was dropped: gateway 404s; don't re-pin to a versioned id that may +// rotate again.) const OPENROUTER_MANDATORY_REASONING_MODELS = new Set([ - "google/gemini-pro-latest", "baidu/cobuddy:free", "inclusionai/ring-2.6-1t:free", "deepseek/deepseek-r1", @@ -626,6 +1088,9 @@ const NO_REASONING_CAPS: ReasoningCaps = { reasoningEffortLevels: DEFAULT_EFFORT_LEVELS, }; +// Longest prefixes first (find() must match before the bare-family +// fallback). Levels per platform.claude.com/docs/en/about-claude/models/overview. +// 4.5 line maps to budget_tokens; sonnet-4/opus-4 retire 2026-06-15. const ANTHROPIC_REASONING_MODELS = [ { prefixes: ["claude-opus-4-7"], @@ -637,7 +1102,10 @@ const ANTHROPIC_REASONING_MODELS = [ }, { prefixes: ["claude-opus-4-5", "claude-sonnet-4-5", "claude-haiku-4-5"], - // Backend maps semantic levels to manual budget_tokens. + levels: ["none", "low", "medium", "high"], + }, + { + prefixes: ["claude-opus-4-1", "claude-opus-4", "claude-sonnet-4"], levels: ["none", "low", "medium", "high"], }, ] as const; @@ -681,17 +1149,26 @@ const OPENAI_REASONING_MODELS = [ levels: ["medium"], }, { + // gpt-5.3-codex enum is low/medium/high/xhigh only per dev page. prefixes: ["gpt-5.3-codex"], + supportsOff: false, + levels: ["low", "medium", "high", "xhigh"], + }, + { + // Azure footnote ^7^: minimal supported only on original gpt-5. + // Listed before the bare gpt-5 entry so the longer match wins. + prefixes: ["gpt-5.1", "gpt-5.2"], supportsOff: true, levels: ["none", "low", "medium", "high", "xhigh"], }, { - prefixes: ["gpt-5", "gpt-5.1", "gpt-5.2"], + prefixes: ["gpt-5"], supportsOff: false, levels: ["minimal", "low", "medium", "high"], }, { - prefixes: ["o3"], + // o-series all accept low/medium/high per dev pages + Azure table. + prefixes: ["o1", "o3", "o4", "codex-mini"], supportsOff: false, levels: DEFAULT_EFFORT_LEVELS, }, @@ -867,19 +1344,27 @@ function resolveGeminiReasoningCapabilities( } function resolveMistralReasoningCapabilities(modelId: string): ExternalReasoningCapabilities { - if (modelId === "magistral-medium-latest") { - return withReasoningEffortStyle({ + // magistral-*: native always-on (422 on reasoning_effort). + // mistral-{small,medium,vibe-cli}-latest: none/low/medium/high. + // https://docs.mistral.ai/studio-api/conversations/reasoning + if ( + modelId === "magistral-medium-latest" || + modelId === "magistral-small-latest" + ) { + return withEnableThinkingStyle({ supportsReasoning: true, - supportsReasoningOff: false, - // Native reasoning model: present baseline as Medium in the UI. - reasoningEffortLevels: ["medium", "high"] as const, + reasoningAlwaysOn: true, }); } - if (modelId === "mistral-small-latest" || modelId === "mistral-vibe-cli-latest") { + if ( + modelId === "mistral-small-latest" || + modelId === "mistral-medium-latest" || + modelId === "mistral-vibe-cli-latest" + ) { return withReasoningEffortStyle({ supportsReasoning: true, supportsReasoningOff: true, - reasoningEffortLevels: ["none", "high"] as const, + reasoningEffortLevels: ["none", "low", "medium", "high"] as const, }); } return withEnableThinkingStyle(); @@ -892,7 +1377,7 @@ export interface ExternalReasoningResolveOptions { baseUrl?: string | null; } -// vLLM has no per-model reasoning signal on OpenAI-compat — pin via user toggle. +// vLLM has no per-model reasoning signal on OpenAI-compat; pin via user toggle. function resolveConnectionLevelReasoning( normalizedProvider: string, options: ExternalReasoningResolveOptions | undefined, @@ -906,11 +1391,9 @@ function resolveConnectionLevelReasoning( return null; } -/** - * resolve external-model thinking capabilities. - * provider-specific matching lives in the OpenAI/Anthropic resolvers. - * other providers default to no reasoning controls. - */ +// Provider-specific matching lives in the per-provider resolvers +// (resolveOpenAI / Anthropic / Kimi / Mistral...). Unknown providers +// default to no reasoning controls. export function getExternalReasoningCapabilities( providerType: string | null | undefined, modelId: string | null | undefined, @@ -952,9 +1435,8 @@ export function getExternalReasoningCapabilities( const isOpenRouterProvider = normalizedProvider === "openrouter"; if (isOpenRouterProvider) { // OpenRouter's unified `reasoning` parameter is accepted on every - // chat-completion request; the gateway silently no-ops for models - // that don't reason. Mandatory-reasoning ids are handled by the - // early guard above; everything else exposes a toggleable control. + // request; gateway no-ops for non-reasoning models. Mandatory ids + // already handled above; everything else exposes a toggle. return { supportsReasoning: true, reasoningStyle: "enable_thinking", diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts index 09c06f92d8..ddb3d71aa4 100644 --- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts +++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts @@ -406,11 +406,45 @@ const PERSISTED_INFERENCE_PARAM_KEYS = [ "minP", "repetitionPenalty", "presencePenalty", + "frequencyPenalty", + "seed", + "stop", + "serviceTier", + "parallelToolCalls", "maxSeqLength", "maxTokens", "systemPrompt", "trustRemoteCode", "fastMode", + // Extended llama.cpp / vLLM / OpenRouter samplers exposed by PR #5711. + "typicalP", + "topNSigma", + "repeatLastN", + "dynatempRange", + "dynatempExponent", + "mirostat", + "mirostatTau", + "mirostatEta", + "topA", + "dryMultiplier", + "dryBase", + "dryAllowedLength", + "dryPenaltyLastN", + "xtcProbability", + "xtcThreshold", + "minKeep", + "ignoreEos", + "minTokens", + "skipSpecialTokens", + "spacesBetweenSpecialTokens", + "includeStopStrInOutput", + "truncatePromptTokens", + "nKeep", + "nProbs", + "cachePrompt", + "returnTokens", + "timingsPerToken", + "postSamplingProbs", ] as const satisfies readonly PersistedInferenceParamKey[]; const SCALAR_SETTING_KEYS = [ diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index d313b43438..520acea25c 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -314,12 +314,81 @@ export interface OpenAIChatCompletionsRequest { * the Anthropic provider with `code_execution` in `enabled_tools`. */ anthropic_code_exec_container_id?: string | null; - /** - * Anthropic fast-mode toggle. Opus 4.6 / 4.7 only; backend drops - * silently on every other model + provider. See - * https://platform.claude.com/docs/en/build-with-claude/fast-mode - */ + /** OpenAI Chat only. Range -2..2. */ + frequency_penalty?: number; + /** OAI Chat + most OAI-compat. Responses + Anthropic drop. */ + seed?: number; + /** OAI Chat caps at 4; Anthropic mapped to `stop_sequences`. */ + stop?: string[]; + /** Per-provider enum (see getServiceTierOptions); external_provider.py drops unsupported values. */ + service_tier?: + | "auto" + | "default" + | "flex" + | "priority" + | "scale" + | "standard_only"; + /** Anthropic inverts to `disable_parallel_tool_use`. */ + parallel_tool_calls?: boolean; + /** llama.cpp `typ_p`. 1.0 disables. */ + typical_p?: number; + /** llama.cpp `top_n_sigma`. -1 disables. */ + top_n_sigma?: number; + /** llama.cpp `repeat_last_n`. 0 disables, -1 = ctx-size. */ + repeat_last_n?: number; + /** llama.cpp `dynatemp_range`. 0 disables. */ + dynatemp_range?: number; + /** llama.cpp `dynatemp_exponent`. Pairs with dynatemp_range. */ + dynatemp_exponent?: number; + /** llama.cpp `mirostat` (0/1/2). 0 disables. */ + mirostat?: number; + mirostat_tau?: number; + mirostat_eta?: number; + /** OpenRouter `top_a`. https://openrouter.ai/docs/api/reference/parameters */ + top_a?: number; + /** Anthropic Opus 4.6 / 4.7 only. https://platform.claude.com/docs/en/build-with-claude/fast-mode */ fast_mode?: boolean | null; + /** + * llama.cpp DRY sampler (4 fields). `dry_multiplier=0` disables. + * https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md + */ + dry_multiplier?: number; + /** Default 1.75. */ + dry_base?: number; + /** Default 2. */ + dry_allowed_length?: number; + /** 0 disables, -1 = ctx-size. */ + dry_penalty_last_n?: number; + /** llama.cpp XTC. 0 disables. */ + xtc_probability?: number; + /** Default 0.1. */ + xtc_threshold?: number; + /** llama.cpp `min_keep`. */ + min_keep?: number; + /** Continue past EOS. llama.cpp + vLLM. */ + ignore_eos?: boolean; + /** Min tokens before stop / EOS. llama.cpp + vLLM. */ + min_tokens?: number; + /** vLLM only. */ + skip_special_tokens?: boolean; + /** vLLM only. */ + spaces_between_special_tokens?: boolean; + /** vLLM only. Useful for agentic tools. */ + include_stop_str_in_output?: boolean; + /** vLLM only. Left-truncate the prompt. */ + truncate_prompt_tokens?: number; + /** llama.cpp `n_keep`. -1 = keep all. */ + n_keep?: number; + /** llama.cpp `n_probs`. */ + n_probs?: number; + /** llama.cpp `cache_prompt`. */ + cache_prompt?: boolean; + /** llama.cpp `return_tokens` (debug). */ + return_tokens?: boolean; + /** llama.cpp `timings_per_token` (perf debug). */ + timings_per_token?: boolean; + /** llama.cpp `post_sampling_probs` (sampler debug). */ + post_sampling_probs?: boolean; } export interface OpenAIChatDelta { diff --git a/studio/frontend/src/features/chat/types/runtime.ts b/studio/frontend/src/features/chat/types/runtime.ts index 4c44ee1e9c..056749c43e 100644 --- a/studio/frontend/src/features/chat/types/runtime.ts +++ b/studio/frontend/src/features/chat/types/runtime.ts @@ -1,6 +1,16 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 +export type ServiceTier = + | "auto" + | "default" + | "flex" + | "priority" + | "scale" + | "standard_only"; + +// null = field omitted from wire (provider default). +// Per-provider gates in provider-capabilities.ts. export interface InferenceParams { temperature: number; topP: number; @@ -8,17 +18,77 @@ export interface InferenceParams { minP: number; repetitionPenalty: number; presencePenalty: number; + /** OpenAI Chat only; rejected by Responses + Anthropic. */ + frequencyPenalty: number; + /** Determinism seed. OpenAI Chat + most OAI-compat backends only. */ + seed: number | null; + /** OAI Chat `stop` / Anthropic `stop_sequences`. OAI caps at 4. */ + stop: string[]; + /** Per-provider enum via `getServiceTierOptions`. `null` = provider default. */ + serviceTier: ServiceTier | null; + /** Anthropic inverts to `disable_parallel_tool_use`. */ + parallelToolCalls: boolean; + /** llama.cpp `typ_p`. 1.0 disables. */ + typicalP: number | null; + /** llama.cpp `top_n_sigma`. -1 disables. */ + topNSigma: number | null; + /** llama.cpp `repeat_last_n`. 0 disables, -1 = ctx-size. */ + repeatLastN: number | null; + /** llama.cpp `dynatemp_range`. 0 disables. */ + dynatempRange: number | null; + /** llama.cpp `dynatemp_exponent`. Pairs with dynatempRange. */ + dynatempExponent: number | null; + /** llama.cpp `mirostat` (0/1/2). 0 disables. */ + mirostat: number | null; + mirostatTau: number | null; + mirostatEta: number | null; + /** OpenRouter `top_a`. Range [0, 1]. */ + topA: number | null; + /** llama.cpp DRY: multiplier is master switch (0 disables 4-field chain). See llama.cpp/tools/server/README.md. */ + dryMultiplier: number | null; + /** Default 1.75. */ + dryBase: number | null; + /** Default 2. */ + dryAllowedLength: number | null; + /** 0 disables, -1 = ctx-size. */ + dryPenaltyLastN: number | null; + /** llama.cpp XTC: probability is the master switch (0 disables). */ + xtcProbability: number | null; + /** Default 0.1. */ + xtcThreshold: number | null; + /** llama.cpp `min_keep`: min tokens past all filters. 0 disables. */ + minKeep: number | null; + /** Continue past EOS. llama.cpp + vLLM. */ + ignoreEos: boolean | null; + /** Min tokens before stop / EOS can fire. llama.cpp + vLLM. */ + minTokens: number | null; + /** vLLM only. Default true; forward only when false. */ + skipSpecialTokens: boolean | null; + /** vLLM only. Default true; forward only when false. */ + spacesBetweenSpecialTokens: boolean | null; + /** vLLM only. Useful for agentic tools needing the matched stop string echoed. */ + includeStopStrInOutput: boolean | null; + /** vLLM only. Left-truncate the prompt. */ + truncatePromptTokens: number | null; + /** llama.cpp `n_keep`. 0 disables, -1 = keep all. */ + nKeep: number | null; + /** llama.cpp `n_probs`: top-N token probabilities per token. */ + nProbs: number | null; + /** llama.cpp `cache_prompt`. Default true; forward only when false. */ + cachePrompt: boolean | null; + /** llama.cpp `return_tokens` (debug). */ + returnTokens: boolean | null; + /** llama.cpp `timings_per_token` (perf debug). */ + timingsPerToken: boolean | null; + /** llama.cpp `post_sampling_probs` (sampler debug). */ + postSamplingProbs: boolean | null; maxSeqLength: number; maxTokens: number; systemPrompt: string; checkpoint: string; - /** Allow loading models with custom code (e.g. NVIDIA Nemotron). Only enable for repos you trust. */ + /** Trust custom model code (e.g. NVIDIA Nemotron). Only for trusted repos. */ trustRemoteCode?: boolean; - /** - * Anthropic fast-mode toggle. Opus 4.6 / 4.7 only; higher OTPS at - * 6x standard Opus pricing. Default false. - * https://platform.claude.com/docs/en/build-with-claude/fast-mode - */ + /** Anthropic Opus 4.6 / 4.7 only. 6x pricing for higher OTPS. */ fastMode?: boolean; } @@ -29,6 +99,39 @@ export const DEFAULT_INFERENCE_PARAMS: InferenceParams = { minP: 0.01, repetitionPenalty: 1.0, presencePenalty: 0.0, + frequencyPenalty: 0.0, + seed: null, + stop: [], + serviceTier: null, + parallelToolCalls: true, + typicalP: null, + topNSigma: null, + repeatLastN: null, + dynatempRange: null, + dynatempExponent: null, + mirostat: null, + mirostatTau: null, + mirostatEta: null, + topA: null, + dryMultiplier: null, + dryBase: null, + dryAllowedLength: null, + dryPenaltyLastN: null, + xtcProbability: null, + xtcThreshold: null, + minKeep: null, + ignoreEos: null, + minTokens: null, + skipSpecialTokens: null, + spacesBetweenSpecialTokens: null, + includeStopStrInOutput: null, + truncatePromptTokens: null, + nKeep: null, + nProbs: null, + cachePrompt: null, + returnTokens: null, + timingsPerToken: null, + postSamplingProbs: null, maxSeqLength: 4096, maxTokens: 8192, systemPrompt: "", diff --git a/studio/frontend/src/features/chat/utils/chat-settings-storage.ts b/studio/frontend/src/features/chat/utils/chat-settings-storage.ts index 4e93a20bff..dd20d812f2 100644 --- a/studio/frontend/src/features/chat/utils/chat-settings-storage.ts +++ b/studio/frontend/src/features/chat/utils/chat-settings-storage.ts @@ -40,10 +40,23 @@ const NUMERIC_INFERENCE_FIELDS = [ "minP", "repetitionPenalty", "presencePenalty", + "frequencyPenalty", "maxSeqLength", "maxTokens", ] as const satisfies readonly (keyof PersistedInferenceParams)[]; +// `seed` is nullable so it skips NUMERIC_INFERENCE_FIELDS' finite-number filter. +// Keep in sync with ServiceTier (../types/runtime.ts) and getServiceTierOptions. +// "scale" stays for forward-compat with legacy persisted data. +const VALID_SERVICE_TIERS = new Set([ + "auto", + "default", + "flex", + "priority", + "scale", + "standard_only", +]); + const CHAT_PRESET_SOURCES = new Set([ "builtin-default", "custom", @@ -140,6 +153,87 @@ function sanitizeInferenceParams( if (typeof value.trustRemoteCode === "boolean") { params.trustRemoteCode = value.trustRemoteCode; } + // seed: nullable integer (null = no seed on the wire). + if (value.seed === null) { + params.seed = null; + } else if (typeof value.seed === "number" && Number.isInteger(value.seed)) { + params.seed = value.seed; + } + // stop: cap at 16 (Anthropic widest); backend re-truncates per provider. + // Empty array MUST persist or clearing the last chip is reverted on reload. + if (Array.isArray(value.stop)) { + const stops = value.stop.filter((s): s is string => typeof s === "string"); + params.stop = stops.slice(0, 16); + } + if (value.serviceTier === null) { + params.serviceTier = null; + } else if ( + typeof value.serviceTier === "string" && + VALID_SERVICE_TIERS.has(value.serviceTier) + ) { + params.serviceTier = value.serviceTier as PersistedInferenceParams["serviceTier"]; + } + if (typeof value.parallelToolCalls === "boolean") { + params.parallelToolCalls = value.parallelToolCalls; + } + // typicalP: nullable float (null = no typ_p on wire, matches + // llama-server default 1.0). Mirrors seed handling. + if (value.typicalP === null) { + params.typicalP = null; + } else if ( + typeof value.typicalP === "number" && + Number.isFinite(value.typicalP) + ) { + params.typicalP = value.typicalP; + } + // Nullable numeric samplers (same handling as typicalP/seed). + for (const key of [ + "topNSigma", + "repeatLastN", + "dynatempRange", + "dynatempExponent", + "mirostat", + "mirostatTau", + "mirostatEta", + "topA", + "dryMultiplier", + "dryBase", + "dryAllowedLength", + "dryPenaltyLastN", + "xtcProbability", + "xtcThreshold", + "minKeep", + "minTokens", + "truncatePromptTokens", + "nKeep", + "nProbs", + ] as const) { + const raw = value[key]; + if (raw === null) { + (params as Record)[key] = null; + } else if (typeof raw === "number" && Number.isFinite(raw)) { + (params as Record)[key] = raw; + } + } + // Nullable booleans (ignoreEos, skip/spaces special-tokens, include-stop, + // cache_prompt, return_tokens, timings_per_token, post_sampling_probs). + for (const key of [ + "ignoreEos", + "skipSpecialTokens", + "spacesBetweenSpecialTokens", + "includeStopStrInOutput", + "cachePrompt", + "returnTokens", + "timingsPerToken", + "postSamplingProbs", + ] as const) { + const raw = value[key]; + if (raw === null) { + (params as Record)[key] = null; + } else if (typeof raw === "boolean") { + (params as Record)[key] = raw; + } + } // Mirror trustRemoteCode handling so the toggle survives reload // and the /api/chat/settings round-trip. if (typeof value.fastMode === "boolean") {