Tighten comments across PR 5711 (no behaviour change)
Audited every comment added by this PR; condensed multi-paragraph docstrings and inline blocks to 1-2 lines where the WHY survives. Files touched: 6 backend + 7 frontend. Net -148 lines. - external_provider.py: -80 (docstring + stop_sequences + compaction) - chat-settings-sheet.tsx: -70 (InfoHint tooltips + section headers) - routes/inference.py: -54 (parallel_tool_calls cap comments) - provider-capabilities.ts: -108 (stop-cap table, gemini bucket, service_tier resolver, max-output cap header) - chat-adapter.ts: -27 (sampling forwarding stanza) - providers.py: -24 (kimi reasoning class, deepseek aliases) - llama_cpp.py: -24 (payload builder shared header) - models/inference.py: -28 (Pydantic Field descriptions) - chat-settings-storage.ts: -23 (nullable handling comments) - stop-sequences-input.tsx: -23 (JSDoc + chip-key + draft-commit) - anthropic_compat.py: -9 (serial tool-call gate) - types/runtime.ts: -12 (DRY JSDoc, null convention header) - types/api.ts: -6 (service_tier JSDoc) Tightening rules applied: - Removed em-dashes everywhere a comment was rewritten. - Kept every external doc URL (Anthropic, OpenAI, vLLM, llama.cpp, Ollama, OpenRouter, Mistral, DeepSeek, Gemini, Kimi). - Collapsed "silently dropped on unsupported routes" prose since the bucket-comment at the top of each capability table already states it. - Removed restated rationale prose in comment blocks where the field name plus the master rule already encodes the WHY. 393/393 backend tests pass (test_sampling_params_routing 65, plus anthropic / openai / gemini / llama-server suites). Frontend tsc + vite build clean.
This commit is contained in:
Daniel Han
parent
0cf2097a81
commit
1add4dbd0e
13 changed files with 170 additions and 318 deletions
|
|
@ -410,11 +410,10 @@ class AnthropicPassthroughEmitter:
|
|||
self._tool_call_states: dict = {} # delta index -> {block_index, id, name}
|
||||
self._usage: dict = {}
|
||||
self._stop_reason: str = "end_turn"
|
||||
# When the caller opted out of parallel tool calls (Anthropic
|
||||
# `tool_choice.disable_parallel_tool_use=true` mapped to local
|
||||
# `parallel_tool_calls=false`), only emit the first tool-call
|
||||
# index and drop later siblings. llama.cpp may not enforce the
|
||||
# flag on every jinja template (ggml-org/llama.cpp#22043).
|
||||
# parallel_tool_calls=False (Anthropic
|
||||
# tool_choice.disable_parallel_tool_use=true): emit only the first
|
||||
# tool-call index; llama.cpp's flag isn't enforced by every jinja
|
||||
# template (ggml-org/llama.cpp#22043).
|
||||
self._serial_tool_calls: bool = parallel_tool_calls is False
|
||||
self._first_tool_call_idx: Optional[int] = None
|
||||
|
||||
|
|
|
|||
|
|
@ -81,15 +81,10 @@ def _normalize_stop_for_provider(
|
|||
def _is_openai_family_cloud(base_url: Optional[str]) -> bool:
|
||||
"""True iff ``base_url`` points at OpenAI cloud or Azure OpenAI Foundry.
|
||||
|
||||
Host-anchored to avoid subdomain-injection bypass
|
||||
(https://evil.com/api.openai.com/v1, https://api.openai.com.attacker.com/v1).
|
||||
Used to gate cloud-only Responses-API extensions
|
||||
(prompt_cache_retention, context_management compaction, container
|
||||
shell tool) that 400 on non-cloud OAI-compat servers.
|
||||
|
||||
Azure Foundry uses <resource>.openai.azure.com; match via endswith
|
||||
with the leading dot so the apex `openai.azure.com` can't slip
|
||||
through (no apex Foundry endpoint exists).
|
||||
Host-anchored against subdomain-injection (api.openai.com.attacker.com).
|
||||
Gates Responses-API extensions (prompt_cache_retention, context_management,
|
||||
container shell) that 400 on non-cloud OAI-compat servers. Azure Foundry
|
||||
matches via .openai.azure.com suffix; leading dot blocks the apex.
|
||||
"""
|
||||
if not base_url:
|
||||
return False
|
||||
|
|
@ -917,19 +912,14 @@ class ExternalProviderClient:
|
|||
For OpenAI-compatible providers, lines are forwarded verbatim.
|
||||
For Anthropic, the native Messages API SSE is translated to OpenAI format.
|
||||
|
||||
``top_k`` and ``presence_penalty`` are forwarded only when the caller
|
||||
supplies a value the provider accepts — the frontend's
|
||||
provider-capability map already filters these per provider, so we
|
||||
treat them as opt-in here.
|
||||
|
||||
Optional sampling extras (``top_k``, ``presence_penalty``,
|
||||
``frequency_penalty``, ``seed``, ``stop``, ``service_tier``,
|
||||
``parallel_tool_calls`` follow the same rule: the per-provider
|
||||
stream helpers silently drop fields the upstream API does not
|
||||
accept (e.g. Responses rejects all of seed / frequency / stop;
|
||||
Anthropic does not implement seed / frequency / logprobs).
|
||||
``parallel_tool_calls``) are opt-in. Per-provider helpers silently
|
||||
drop fields the upstream rejects (Responses: seed/freq/stop;
|
||||
Anthropic: seed/freq/logprobs).
|
||||
|
||||
``fast_mode`` only applies to Anthropic Opus 4.6 / 4.7 (silently
|
||||
dropped elsewhere); adds the beta header and ``speed: "fast"``.
|
||||
``fast_mode``: Anthropic Opus 4.6/4.7 only; adds the beta header
|
||||
and ``speed: "fast"``.
|
||||
"""
|
||||
# tool_choice="none" hard-disables hosted/builtin tools across
|
||||
# every provider so enabled_tools cannot accidentally bill or leak.
|
||||
|
|
@ -1054,9 +1044,8 @@ class ExternalProviderClient:
|
|||
else:
|
||||
body["max_tokens"] = max_tokens
|
||||
|
||||
# Optional sampling extensions. Only forwarded when the caller
|
||||
# passed a value. Per-provider rename / cap via `seed_field` /
|
||||
# `stop_max`; `body_omit` strips fields the upstream rejects.
|
||||
# Optional sampling extras; `seed_field` renames seed (Mistral),
|
||||
# `body_omit` strips upstream-rejected fields.
|
||||
from core.inference.providers import get_provider_info
|
||||
|
||||
provider_info = get_provider_info(self.provider_type) or {}
|
||||
|
|
@ -1069,9 +1058,8 @@ class ExternalProviderClient:
|
|||
normalized_stop = _normalize_stop_for_provider(stop, provider_info)
|
||||
if normalized_stop:
|
||||
body["stop"] = normalized_stop
|
||||
# service_tier is OpenAI Chat-only on the generic OAI-compat
|
||||
# branch; opt-in via `accepts_service_tier=True` on the registry
|
||||
# entry. Anthropic and Responses handle it in their own helpers.
|
||||
# service_tier is OAI-Chat-only here (accepts_service_tier registry
|
||||
# opt-in); Anthropic/Responses branches set it themselves.
|
||||
if service_tier is not None and provider_info.get(
|
||||
"accepts_service_tier", False
|
||||
):
|
||||
|
|
@ -1468,10 +1456,8 @@ class ExternalProviderClient:
|
|||
if max_tokens is not None:
|
||||
body["max_tokens"] = max_tokens
|
||||
|
||||
# The default OAI-compat body construction is skipped because
|
||||
# this helper returns early. Apply the same provider-aware
|
||||
# sampling / stop logic here so kimi-with-search matches
|
||||
# kimi-without-search.
|
||||
# Kimi-with-search returns early before the default OAI-compat
|
||||
# body build; re-apply provider-aware sampling/stop here.
|
||||
from core.inference.providers import get_provider_info
|
||||
|
||||
provider_info = get_provider_info(self.provider_type) or {}
|
||||
|
|
@ -2141,27 +2127,17 @@ class ExternalProviderClient:
|
|||
if top_k is not None and top_k > 0 and not sampling_removed:
|
||||
body["top_k"] = top_k
|
||||
|
||||
# Optional sampling extensions. Anthropic has no
|
||||
# frequency_penalty / seed / logprobs equivalents so they are
|
||||
# never forwarded here. The two body-level knobs Anthropic
|
||||
# accepts land here:
|
||||
# stop -> stop_sequences (renamed, ws-stripped)
|
||||
# service_tier -> service_tier (auto|standard_only only)
|
||||
# parallel_tool_calls inversion is applied after the tools
|
||||
# wiring below because Anthropic requires it nested under
|
||||
# tool_choice.
|
||||
# Anthropic body-knob mapping: stop -> stop_sequences (ws-stripped,
|
||||
# dedup), service_tier (auto|standard_only). parallel_tool_calls is
|
||||
# handled after tools wiring (Anthropic nests under tool_choice).
|
||||
if stop:
|
||||
sequences: list[str]
|
||||
if isinstance(stop, str):
|
||||
sequences = [stop] if stop.strip() else []
|
||||
else:
|
||||
# Dedupe + drop whitespace-only entries. Anthropic
|
||||
# rejects any sequence with no non-whitespace char
|
||||
# ("stop_sequences: each stop sequence must contain
|
||||
# non-whitespace"), so "", " ", "\n", "\n\n" are all
|
||||
# filtered. The 16-cap is a client-side guard; the
|
||||
# docs do not publish a max, but every SDK treats 16
|
||||
# as a sane ceiling (Bedrock is the outlier at 8191).
|
||||
# Anthropic rejects whitespace-only stop_sequences ("must
|
||||
# contain non-whitespace"); 16-cap is a client-side guard
|
||||
# (undocumented max; every SDK uses 16, Bedrock at 8191).
|
||||
sequences = list(
|
||||
dict.fromkeys(s for s in stop if isinstance(s, str) and s.strip())
|
||||
)
|
||||
|
|
@ -2371,9 +2347,8 @@ class ExternalProviderClient:
|
|||
if anthropic_code_exec_container_id:
|
||||
body["container"] = anthropic_code_exec_container_id
|
||||
|
||||
# parallel_tool_calls=False maps to disable_parallel_tool_use=True
|
||||
# nested under tool_choice (top-level placement is rejected).
|
||||
# Without tools the flag is a no-op so we skip the block.
|
||||
# parallel_tool_calls=False -> tool_choice.disable_parallel_tool_use=True
|
||||
# (top-level 400s; no-op without tools).
|
||||
# https://platform.claude.com/docs/en/agents-and-tools/tool-use/implement-tool-use
|
||||
if parallel_tool_calls is False and body.get("tools"):
|
||||
tc = body.get("tool_choice")
|
||||
|
|
@ -5435,11 +5410,8 @@ class ExternalProviderClient:
|
|||
"input": input_items,
|
||||
"stream": True,
|
||||
}
|
||||
# Responses accepts auto|default|flex|priority per the live
|
||||
# docs. The openai-python SDK type happens to include "scale"
|
||||
# too but the public Responses reference does not, so drop it
|
||||
# here to avoid a 400. Scale Tier is still selectable on Chat
|
||||
# Completions backends. parallel_tool_calls default is true.
|
||||
# Responses: auto|default|flex|priority (SDK type lists "scale"
|
||||
# but server 400s; Scale Tier stays on Chat Completions).
|
||||
if service_tier in ("auto", "default", "flex", "priority"):
|
||||
body["service_tier"] = service_tier
|
||||
if parallel_tool_calls is not None:
|
||||
|
|
|
|||
|
|
@ -4316,10 +4316,8 @@ class LlamaCppBackend:
|
|||
else (self._effective_context_length or _DEFAULT_MAX_TOKENS_FLOOR)
|
||||
)
|
||||
payload["t_max_predict_ms"] = _DEFAULT_T_MAX_PREDICT_MS
|
||||
# Strip empty / non-string stop entries before forwarding to
|
||||
# llama-server (matches `_normalize_stop_for_provider` for the
|
||||
# external path). Without this a stale `stop=["", "END"]` can
|
||||
# 400 the upstream.
|
||||
# Strip empty/non-string stop entries (mirrors
|
||||
# `_normalize_stop_for_provider`); stale `stop=["", "END"]` 400s.
|
||||
if stop:
|
||||
if isinstance(stop, str):
|
||||
payload["stop"] = stop
|
||||
|
|
@ -4327,9 +4325,8 @@ class LlamaCppBackend:
|
|||
_cleaned = [s for s in stop if isinstance(s, str) and s]
|
||||
if _cleaned:
|
||||
payload["stop"] = _cleaned
|
||||
# Each field gated `is not None` so explicit 0 / 0.0 / False
|
||||
# values reach the wire. llama-server silently ignores fields
|
||||
# it doesn't recognise.
|
||||
# `is not None` gate so explicit 0/False reach the wire;
|
||||
# llama-server ignores unknown fields.
|
||||
if frequency_penalty is not None:
|
||||
payload["frequency_penalty"] = frequency_penalty
|
||||
if seed is not None:
|
||||
|
|
@ -5195,12 +5192,9 @@ class LlamaCppBackend:
|
|||
_accumulated_predicted_ms += _it.get("predicted_ms", 0)
|
||||
_accumulated_predicted_n += _it.get("predicted_n", 0)
|
||||
|
||||
# When the caller opted out of parallel tool calls
|
||||
# (parallel_tool_calls=False), enforce at most one call
|
||||
# per assistant turn even if llama-server emitted more.
|
||||
# llama.cpp's parallel_tool_calls flag isn't enforced by
|
||||
# every jinja template (see ggml-org/llama.cpp#22043),
|
||||
# so this client-side cap is the only guarantee.
|
||||
# parallel_tool_calls=False: client-side cap to 1
|
||||
# (llama.cpp flag isn't enforced by every jinja template,
|
||||
# ggml-org/llama.cpp#22043).
|
||||
if parallel_tool_calls is False and tool_calls:
|
||||
tool_calls = tool_calls[:1]
|
||||
|
||||
|
|
@ -5414,8 +5408,8 @@ class LlamaCppBackend:
|
|||
_cleaned = [s for s in stop if isinstance(s, str) and s]
|
||||
if _cleaned:
|
||||
stream_payload["stop"] = _cleaned
|
||||
# Match the per-iteration tool loop above so sampling behavior
|
||||
# stays consistent when the cap-exhausted final-answer pass runs.
|
||||
# Match per-iteration tool-loop sampling for the cap-exhausted
|
||||
# final-answer pass.
|
||||
if frequency_penalty is not None:
|
||||
stream_payload["frequency_penalty"] = frequency_penalty
|
||||
if seed is not None:
|
||||
|
|
|
|||
|
|
@ -137,19 +137,17 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = {
|
|||
r"gemini-pro-latest|gemini-flash-latest|gemini-flash-lite-latest"
|
||||
r")$"
|
||||
),
|
||||
# Gemini's OpenAI-compatible layer inherits OpenAI's 4-stop cap
|
||||
# (https://ai.google.dev/gemini-api/docs/openai). Without the
|
||||
# explicit cap the default 16 leaks through and the upstream
|
||||
# silently drops the overflow.
|
||||
# Gemini OAI-compat inherits OpenAI's 4-stop cap; default 16
|
||||
# silently truncates upstream.
|
||||
# https://ai.google.dev/gemini-api/docs/openai
|
||||
"stop_max": 4,
|
||||
},
|
||||
"deepseek": {
|
||||
"display_name": "DeepSeek",
|
||||
"base_url": "https://api.deepseek.com/v1",
|
||||
# Legacy aliases (deepseek-chat / deepseek-reasoner) retire
|
||||
# 2026-07-24 per https://api-docs.deepseek.com/updates. Surface
|
||||
# the new canonical ids (deepseek-v4-flash / deepseek-v4-pro)
|
||||
# alongside so the picker keeps working on cutover.
|
||||
# deepseek-chat / deepseek-reasoner retire 2026-07-24; list
|
||||
# v4-pro / v4-flash alongside for cutover.
|
||||
# https://api-docs.deepseek.com/updates
|
||||
"default_models": [
|
||||
"deepseek-v4-pro",
|
||||
"deepseek-v4-flash",
|
||||
|
|
@ -217,13 +215,9 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = {
|
|||
"auth_prefix": "Bearer ",
|
||||
"notes": "Moonshot API key. China: use base URL https://api.moonshot.cn/v1",
|
||||
"model_id_allowlist": re.compile(r"^kimi-k2\.[56]$"),
|
||||
# Both k2.6 and k2.5 are reasoning-class. The API rejects
|
||||
# custom sampling ("invalid temperature: only 1 is allowed for
|
||||
# this model", same for top_p). frequency_penalty follows the
|
||||
# same lock on those models. seed and parallel_tool_calls are
|
||||
# not in Kimi's documented chat schema; strip them too so a
|
||||
# stale client or direct API caller cannot smuggle them onto
|
||||
# the wire and 400 the request.
|
||||
# k2.5/k2.6 are reasoning-class: API locks temperature=1, top_p=1,
|
||||
# frequency_penalty; seed and parallel_tool_calls are undocumented
|
||||
# and 400.
|
||||
"body_omit": (
|
||||
"temperature",
|
||||
"top_p",
|
||||
|
|
|
|||
|
|
@ -876,10 +876,8 @@ class ChatCompletionRequest(BaseModel):
|
|||
seed: Optional[int] = Field(
|
||||
None,
|
||||
description = (
|
||||
"Best-effort determinism seed. Forwarded to OpenAI Chat "
|
||||
"Completions and OpenAI-compatible local backends. The "
|
||||
"Responses family rejects it server-side and Anthropic does "
|
||||
"not implement it, so it is silently dropped on those routes."
|
||||
"Best-effort determinism seed. Forwarded to OpenAI Chat and "
|
||||
"OAI-compat local backends; dropped on Anthropic and OpenAI Responses."
|
||||
),
|
||||
)
|
||||
service_tier: Optional[
|
||||
|
|
@ -887,22 +885,18 @@ class ChatCompletionRequest(BaseModel):
|
|||
] = Field(
|
||||
None,
|
||||
description = (
|
||||
"Provider service tier. Anthropic accepts only `auto` and "
|
||||
"`standard_only`; OpenAI Chat accepts "
|
||||
"`auto|default|flex|priority|scale`; OpenAI Responses accepts "
|
||||
"`auto|default|flex|priority`. Unsupported values per provider "
|
||||
"are dropped in the per-provider stream helper instead of "
|
||||
"422'ing here so a stale frontend never breaks a request."
|
||||
"Provider service tier. Anthropic: auto|standard_only. "
|
||||
"OpenAI Chat: auto|default|flex|priority|scale. "
|
||||
"OpenAI Responses: auto|default|flex|priority. "
|
||||
"Unsupported values are dropped per provider rather than 422'd here."
|
||||
),
|
||||
)
|
||||
parallel_tool_calls: Optional[bool] = Field(
|
||||
None,
|
||||
description = (
|
||||
"Whether the provider may dispatch tool calls in parallel. "
|
||||
"OpenAI: forwarded as `parallel_tool_calls`. Anthropic: "
|
||||
"inverted into `disable_parallel_tool_use` on the Messages "
|
||||
"body. Default `None` preserves each provider's upstream "
|
||||
"default (which is `true` everywhere today)."
|
||||
"Allow parallel tool calls. Forwarded as `parallel_tool_calls` "
|
||||
"on OpenAI; inverted to `disable_parallel_tool_use` on Anthropic. "
|
||||
"None preserves upstream default (currently true everywhere)."
|
||||
),
|
||||
)
|
||||
typical_p: Optional[float] = Field(
|
||||
|
|
@ -958,8 +952,8 @@ class ChatCompletionRequest(BaseModel):
|
|||
None,
|
||||
ge = 0.0,
|
||||
description = (
|
||||
"llama.cpp DRY multiplier. 0 disables the 4-field chain "
|
||||
"(dry_base / dry_allowed_length / dry_penalty_last_n). Local only."
|
||||
"llama.cpp DRY multiplier. 0 disables the dry_base / "
|
||||
"dry_allowed_length / dry_penalty_last_n chain. Local only."
|
||||
),
|
||||
)
|
||||
dry_base: Optional[float] = Field(
|
||||
|
|
|
|||
|
|
@ -240,12 +240,9 @@ studio_router = APIRouter()
|
|||
|
||||
|
||||
def _clean_local_stop_list(stop) -> Optional[list[str]]:
|
||||
"""Strip empty / non-string entries from a stop sequence input.
|
||||
|
||||
Mirrors `_normalize_stop_for_provider` (external_provider.py) so
|
||||
local llama-server callers cannot ship `stop=["", "END"]` and
|
||||
get a 400. Returns `None` when nothing survives, so the caller
|
||||
can omit the field entirely.
|
||||
"""Strip empty/non-string stop entries; returns None when empty so
|
||||
callers can omit. Mirrors `_normalize_stop_for_provider` so
|
||||
`stop=["", "END"]` cannot 400 llama-server.
|
||||
"""
|
||||
if isinstance(stop, str):
|
||||
return [stop] if stop else None
|
||||
|
|
@ -4185,9 +4182,7 @@ def _build_chat_request(
|
|||
chat_kwargs["top_p"] = payload.top_p
|
||||
if payload.max_output_tokens is not None:
|
||||
chat_kwargs["max_tokens"] = payload.max_output_tokens
|
||||
# parallel_tool_calls is first-class on ChatCompletionRequest and
|
||||
# the OpenAI-compat passthrough builder forwards it. Translate it
|
||||
# so a Responses API caller's preference reaches llama-server.
|
||||
# Forward parallel_tool_calls from Responses caller through to llama-server.
|
||||
if payload.parallel_tool_calls is not None:
|
||||
chat_kwargs["parallel_tool_calls"] = payload.parallel_tool_calls
|
||||
|
||||
|
|
@ -4250,9 +4245,8 @@ async def _responses_non_streaming(
|
|||
msg = choices[0].get("message", {}) or {}
|
||||
text = msg.get("content", "") or ""
|
||||
tool_calls = msg.get("tool_calls") or []
|
||||
# Match the cap applied on GGUF / Anthropic / safetensors tool
|
||||
# paths: when the caller opted out of parallel tool calls, surface
|
||||
# at most one. llama.cpp may not enforce the flag.
|
||||
# parallel_tool_calls=False -> cap to 1 (llama.cpp flag isn't enforced;
|
||||
# mirrors GGUF/Anthropic/safetensors paths).
|
||||
if payload.parallel_tool_calls is False and tool_calls:
|
||||
tool_calls = tool_calls[:1]
|
||||
|
||||
|
|
@ -4376,10 +4370,9 @@ async def _responses_stream(
|
|||
tool_call_state: dict[int, dict] = {}
|
||||
# Text message lives at output_index 0; tool calls claim 1, 2, ...
|
||||
next_output_index = 1
|
||||
# When the caller opted out of parallel tool calls, latch the
|
||||
# first index we see and drop subsequent siblings — mirrors the
|
||||
# GGUF agentic-loop / Anthropic-passthrough caps; llama.cpp may
|
||||
# not enforce the upstream flag (ggml-org/llama.cpp#22043).
|
||||
# parallel_tool_calls=False: latch the first tc index, drop the
|
||||
# rest; llama.cpp flag isn't enforced by every jinja template
|
||||
# (ggml-org/llama.cpp#22043).
|
||||
serial_tool_calls = payload.parallel_tool_calls is False
|
||||
first_serial_idx: Optional[int] = None
|
||||
|
||||
|
|
@ -4872,10 +4865,9 @@ async def anthropic_messages(
|
|||
if openai_tool_choice is None:
|
||||
openai_tool_choice = "auto"
|
||||
|
||||
# Anthropic nests `disable_parallel_tool_use` inside `tool_choice`
|
||||
# (https://docs.claude.com/en/docs/agents-and-tools/tool-use/implement-tool-use).
|
||||
# Flip it into the OpenAI-shaped `parallel_tool_calls` toggle so the
|
||||
# local GGUF tool loop respects clients that opt out of parallel calls.
|
||||
# Anthropic nests `disable_parallel_tool_use` under `tool_choice`;
|
||||
# flip to OAI `parallel_tool_calls` so the local GGUF tool loop honors it.
|
||||
# https://docs.claude.com/en/docs/agents-and-tools/tool-use/implement-tool-use
|
||||
anthropic_parallel_tool_calls: Optional[bool] = None
|
||||
if isinstance(payload.tool_choice, dict):
|
||||
_disable = payload.tool_choice.get("disable_parallel_tool_use")
|
||||
|
|
@ -5393,10 +5385,8 @@ def _build_passthrough_payload(
|
|||
else (backend_ctx or _DEFAULT_MAX_TOKENS_FLOOR)
|
||||
)
|
||||
body["t_max_predict_ms"] = _DEFAULT_T_MAX_PREDICT_MS
|
||||
# Strip empty / non-string stop entries before forwarding to
|
||||
# llama-server; the external-provider helper does this via
|
||||
# `_normalize_stop_for_provider`, and the local path needs the same
|
||||
# defensive shape so a stale `stop=["", "END"]` cannot 400 upstream.
|
||||
# Strip empty stop entries (mirrors `_normalize_stop_for_provider`);
|
||||
# stale `stop=["", "END"]` would 400 llama-server.
|
||||
if stop:
|
||||
if isinstance(stop, str):
|
||||
if stop:
|
||||
|
|
@ -5412,12 +5402,9 @@ def _build_passthrough_payload(
|
|||
body["repeat_penalty"] = repetition_penalty
|
||||
if presence_penalty is not None:
|
||||
body["presence_penalty"] = presence_penalty
|
||||
# llama-server's /v1/chat/completions accepts the standard OpenAI
|
||||
# fields. parallel_tool_calls is a no-op on llama-server today but
|
||||
# forwarded so a future release picks it up automatically.
|
||||
# Each field below gated `is not None` so explicit 0 / False reach
|
||||
# the wire; llama-server silently ignores unknown fields, Ollama's
|
||||
# OAI translator drops everything outside the OAI subset.
|
||||
# parallel_tool_calls is a no-op on llama-server today but forwarded
|
||||
# for future support. `is not None` gate lets explicit 0/False through;
|
||||
# llama-server ignores unknowns, Ollama drops non-OAI fields.
|
||||
if frequency_penalty is not None:
|
||||
body["frequency_penalty"] = frequency_penalty
|
||||
if seed is not None:
|
||||
|
|
@ -5705,11 +5692,8 @@ async def _anthropic_passthrough_non_streaming(
|
|||
content_blocks.append(AnthropicResponseTextBlock(text = text))
|
||||
|
||||
tool_calls = message.get("tool_calls") or []
|
||||
# Mirror the GGUF agentic-loop client-side cap: when the caller
|
||||
# opted out of parallel tool calls, surface at most one tool_use
|
||||
# block even if llama-server returned more than one. llama.cpp may
|
||||
# not enforce the flag on every jinja template (see
|
||||
# ggml-org/llama.cpp#22043).
|
||||
# parallel_tool_calls=False: cap to 1 tool_use block; llama.cpp flag
|
||||
# isn't enforced by every jinja template (ggml-org/llama.cpp#22043).
|
||||
if parallel_tool_calls is False and tool_calls:
|
||||
tool_calls = tool_calls[:1]
|
||||
for tc in tool_calls:
|
||||
|
|
|
|||
|
|
@ -7,14 +7,8 @@ import { cn } from "@/lib/utils";
|
|||
import { XIcon } from "lucide-react";
|
||||
import { type KeyboardEvent, useState } from "react";
|
||||
|
||||
/**
|
||||
* Chips editor for the `stop` / `stop_sequences` array.
|
||||
*
|
||||
* Commit a chip with Enter or comma; press Backspace on an empty input
|
||||
* to delete the most recent chip. Caps at `maxEntries` -- OpenAI Chat
|
||||
* Completions documents a hard cap of 4 stop sequences; Anthropic
|
||||
* Messages accepts arbitrarily many. Pass `Infinity` to disable the cap.
|
||||
*/
|
||||
/** Chips editor for stop/stop_sequences. Enter or comma commits; Backspace
|
||||
* on empty deletes the last. OpenAI Chat caps at 4; pass Infinity to disable. */
|
||||
export interface StopSequencesInputProps {
|
||||
value: string[];
|
||||
onChange: (next: string[]) => void;
|
||||
|
|
@ -38,12 +32,9 @@ export function StopSequencesInput({
|
|||
const atCap = value.length >= maxEntries;
|
||||
|
||||
function commitDraft() {
|
||||
// Reject only the empty draft; preserve whitespace exactly (stop
|
||||
// matching is byte-exact). OpenAI-compat / llama-server backends
|
||||
// accept whitespace-only stops like "\n\n" for blank-line halts;
|
||||
// pasting such a value into the input should round-trip rather
|
||||
// than be silently dropped. Anthropic's helper strips whitespace
|
||||
// entries on the wire so a chip that's invalid there cannot 400.
|
||||
// Preserve whitespace exactly (stops are byte-exact); llama-server
|
||||
// accepts "\n\n" for blank-line halts. Anthropic strips whitespace
|
||||
// entries on the wire.
|
||||
if (!draft) return;
|
||||
if (atCap) return;
|
||||
if (value.includes(draft)) {
|
||||
|
|
@ -85,9 +76,7 @@ export function StopSequencesInput({
|
|||
>
|
||||
{value.map((entry, index) => (
|
||||
<Badge
|
||||
// Stop sequences are not guaranteed unique across edits (the
|
||||
// user could enter "END" twice if the previous one was just
|
||||
// deleted), so combine value + index for a stable key.
|
||||
// Not unique across edits (re-typed value); combine value+index for a stable key.
|
||||
key={`${entry}-${index}`}
|
||||
variant="secondary"
|
||||
className="gap-1 pl-2 pr-1"
|
||||
|
|
|
|||
|
|
@ -1991,10 +1991,8 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
...(externalCapabilities?.presencePenalty
|
||||
? { presence_penalty: params.presencePenalty }
|
||||
: {}),
|
||||
// Optional sampling extensions. Each gate is per-provider
|
||||
// (see provider-capabilities.ts); the backend additionally
|
||||
// drops fields the upstream API does not accept, so a
|
||||
// stale frontend cannot 400 the request.
|
||||
// Optional sampling extras. Per-provider gates live in
|
||||
// provider-capabilities.ts; backend drops unknown fields.
|
||||
...(externalCapabilities?.frequencyPenalty
|
||||
? { frequency_penalty: params.frequencyPenalty }
|
||||
: {}),
|
||||
|
|
@ -2007,16 +2005,14 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
...(externalCapabilities?.serviceTier && params.serviceTier
|
||||
? { service_tier: params.serviceTier }
|
||||
: {}),
|
||||
// Forward parallel_tool_calls when the user explicitly
|
||||
// turned it off (the upstream default is `true` on every
|
||||
// provider we ship, so default true is a no-op).
|
||||
// Upstream default is true on every shipped provider;
|
||||
// forward only on explicit opt-out.
|
||||
...(externalCapabilities?.parallelToolCalls &&
|
||||
params.parallelToolCalls === false
|
||||
? { parallel_tool_calls: false }
|
||||
: {}),
|
||||
// llama.cpp / vLLM / OpenRouter extras. Each is gated by
|
||||
// (a) the active provider's capability flag and (b) a
|
||||
// non-default value, so only meaningful knobs hit the wire.
|
||||
// llama.cpp / vLLM / OpenRouter extras: cap-flag plus
|
||||
// non-default value gates so only meaningful knobs hit wire.
|
||||
...(externalCapabilities?.typicalP &&
|
||||
params.typicalP !== null &&
|
||||
params.typicalP !== 1
|
||||
|
|
@ -2244,10 +2240,9 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
min_p: params.minP,
|
||||
repetition_penalty: params.repetitionPenalty,
|
||||
presence_penalty: params.presencePenalty,
|
||||
// llama-server accepts the standard OAI extensions via
|
||||
// _build_passthrough_payload and silently ignores unknown
|
||||
// fields. parallel_tool_calls defaults to false upstream so
|
||||
// we forward unconditionally to honour the default-on UI.
|
||||
// llama-server (_build_passthrough_payload) accepts OAI
|
||||
// extras and ignores unknown; parallel_tool_calls default
|
||||
// false upstream so forward unconditionally.
|
||||
...(params.frequencyPenalty !== 0
|
||||
? { frequency_penalty: params.frequencyPenalty }
|
||||
: {}),
|
||||
|
|
@ -2312,9 +2307,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
...(params.minTokens !== null && params.minTokens > 0
|
||||
? { min_tokens: params.minTokens }
|
||||
: {}),
|
||||
// Forward only when value diverges from upstream default;
|
||||
// per-backend capability gating decides whether the wire
|
||||
// even sees these.
|
||||
// Forward only on non-default; per-backend cap-gates wire visibility.
|
||||
...(params.skipSpecialTokens === false
|
||||
? { skip_special_tokens: false }
|
||||
: {}),
|
||||
|
|
|
|||
|
|
@ -413,15 +413,11 @@ export function ChatSettingsPanel({
|
|||
}: ChatSettingsPanelProps) {
|
||||
const isMobile = useIsMobile();
|
||||
const isGguf = useChatRuntimeStore((s) => s.activeGgufVariant) != null;
|
||||
// For non-external (local) models we show every knob — providerCapabilities
|
||||
// is only consulted when `isExternalModel` is true. An external model with an
|
||||
// unknown provider falls back to the OpenAI-compat shape via
|
||||
// getProviderCapabilities, so these flags never undercount support.
|
||||
// GGUF (llama-server) honours frequency_penalty/seed/stop/parallel_tool_calls;
|
||||
// the safetensors / HF transformers path does not, so we hide those toggles
|
||||
// on local non-GGUF backends to keep the UI honest. Anything still set in
|
||||
// params from a prior GGUF session is harmlessly ignored by the safetensors
|
||||
// worker, so this is purely a presentation gate.
|
||||
// Local models show every knob (providerCapabilities only consulted
|
||||
// when isExternalModel; unknown providers fall back to OPENAI_COMPAT_BASE).
|
||||
// GGUF llama-server honours frequency_penalty/seed/stop/parallel_tool_calls;
|
||||
// HF transformers path doesn't, so hide on local non-GGUF (stale params
|
||||
// are harmlessly ignored by the safetensors worker).
|
||||
const localSamplerSupportsExtras = !isExternalModel ? isGguf : true;
|
||||
const showTemperature =
|
||||
!isExternalModel || Boolean(providerCapabilities?.temperature);
|
||||
|
|
@ -446,9 +442,7 @@ export function ChatSettingsPanel({
|
|||
const showParallelToolCalls = isExternalModel
|
||||
? Boolean(providerCapabilities?.parallelToolCalls)
|
||||
: localSamplerSupportsExtras;
|
||||
// Extended llama.cpp / vLLM / OpenRouter samplers. Same gate as the
|
||||
// core knobs above: external → providerCapabilities flag; local →
|
||||
// GGUF-only (safetensors transformers ignores these).
|
||||
// Extended samplers: external uses cap flag, local is GGUF-only.
|
||||
const capAdv = (k: keyof ProviderCapabilities): boolean =>
|
||||
isExternalModel
|
||||
? Boolean(providerCapabilities?.[k])
|
||||
|
|
@ -484,9 +478,8 @@ export function ChatSettingsPanel({
|
|||
postSamplingProbs: capAdv("postSamplingProbs"),
|
||||
};
|
||||
const showAdvancedSamplingSection = Object.values(advCaps).some(Boolean);
|
||||
// Per-provider stop cap from provider-capabilities.ts; backend
|
||||
// re-trims on the wire if a stale UI sends more than the upstream
|
||||
// accepts.
|
||||
// Per-provider stop cap; backend re-trims on the wire if a stale
|
||||
// UI sends more than the upstream accepts.
|
||||
const stopMaxEntries = getProviderStopMax(externalProviderType);
|
||||
const serviceTierOptions = getServiceTierOptions(externalProviderType);
|
||||
const hasModelContent =
|
||||
|
|
@ -1379,10 +1372,9 @@ export function ChatSettingsPanel({
|
|||
Seed
|
||||
</span>
|
||||
<InfoHint>
|
||||
Best-effort determinism seed. Same seed + prompt =
|
||||
same output (approximately). OpenAI Chat Completions
|
||||
and OpenAI-compat local backends honor it; OpenAI
|
||||
Responses and Anthropic silently drop it.
|
||||
Best-effort determinism. OpenAI Chat and OAI-compat
|
||||
local backends honor it; OpenAI Responses and Anthropic
|
||||
silently drop it.
|
||||
</InfoHint>
|
||||
</div>
|
||||
<Input
|
||||
|
|
@ -1413,11 +1405,9 @@ export function ChatSettingsPanel({
|
|||
Stop sequences
|
||||
</span>
|
||||
<InfoHint>
|
||||
Strings that halt generation as soon as the model
|
||||
emits them. Enter a value and press Enter or comma
|
||||
to commit a chip. Backend translates to
|
||||
`stop_sequences` on Anthropic and `stop` on OpenAI
|
||||
Chat (capped at 4 entries).
|
||||
Strings that halt generation. Enter or comma to commit.
|
||||
Maps to `stop_sequences` (Anthropic) / `stop` (OpenAI,
|
||||
cap 4).
|
||||
</InfoHint>
|
||||
</div>
|
||||
<StopSequencesInput
|
||||
|
|
@ -1435,22 +1425,16 @@ export function ChatSettingsPanel({
|
|||
Service tier
|
||||
</span>
|
||||
<InfoHint>
|
||||
Provider routing tier. `auto` (default) lets the
|
||||
provider choose. `flex` / `priority` / `scale` route
|
||||
to higher-latency-tolerant or premium queues on
|
||||
OpenAI; `standard_only` opts out of Anthropic's
|
||||
priority tier.
|
||||
Provider routing tier. `auto` = provider default.
|
||||
OpenAI: flex / priority / scale. Anthropic:
|
||||
`standard_only` opts out of Priority Tier.
|
||||
</InfoHint>
|
||||
</div>
|
||||
<Select
|
||||
value={params.serviceTier ?? "auto"}
|
||||
onValueChange={(value) => {
|
||||
// Store every selected tier verbatim, including the
|
||||
// explicit "auto". On Anthropic the docs distinguish
|
||||
// omitting service_tier (provider default) from
|
||||
// setting "auto" (opt into Priority Tier when
|
||||
// available); preserve the user's choice through to
|
||||
// the adapter so the wire reflects it.
|
||||
// Store "auto" verbatim: Anthropic distinguishes
|
||||
// omitted (provider default) from auto (Priority Tier opt-in).
|
||||
const allowed: readonly ServiceTier[] = serviceTierOptions;
|
||||
if (allowed.includes(value as ServiceTier)) {
|
||||
set("serviceTier")(value as ServiceTier);
|
||||
|
|
@ -1480,11 +1464,8 @@ export function ChatSettingsPanel({
|
|||
Parallel tool calls
|
||||
</span>
|
||||
<InfoHint>
|
||||
When on, the model may dispatch multiple tool calls
|
||||
in a single turn (default). Turn off to force one
|
||||
tool call at a time. Anthropic implements this as
|
||||
`disable_parallel_tool_use`; OpenAI as
|
||||
`parallel_tool_calls`.
|
||||
Allow multiple tool calls per turn (default).
|
||||
Anthropic uses inverse `disable_parallel_tool_use`.
|
||||
</InfoHint>
|
||||
</div>
|
||||
<Switch
|
||||
|
|
@ -1623,7 +1604,7 @@ export function ChatSettingsPanel({
|
|||
max={5}
|
||||
step={0.1}
|
||||
onChange={(v) => set("dynatempExponent")(v)}
|
||||
info="llama.cpp `dynatemp_exponent`. Curve exponent for dynamic temperature. Paired with Dynatemp Range."
|
||||
info="llama.cpp `dynatemp_exponent`. Curve exponent, pairs with Dynatemp Range."
|
||||
/>
|
||||
) : null}
|
||||
{advCaps.mirostat ? (
|
||||
|
|
@ -1699,7 +1680,7 @@ export function ChatSettingsPanel({
|
|||
? "Off"
|
||||
: undefined
|
||||
}
|
||||
info="llama.cpp DRY sampler. Master switch for the 4-field DRY chain (base / allowed length / penalty last N). 0 = off."
|
||||
info="llama.cpp DRY master switch (unlocks base / allowed length / penalty last N). 0 = off."
|
||||
/>
|
||||
) : null}
|
||||
{advCaps.dryBase && (params.dryMultiplier ?? 0) > 0 ? (
|
||||
|
|
@ -1892,9 +1873,8 @@ export function ChatSettingsPanel({
|
|||
Skip Special Tokens
|
||||
</span>
|
||||
<InfoHint>
|
||||
vLLM `skip_special_tokens`. Default on. Turn off to keep
|
||||
chat-template markers (e.g. `<|im_end|>`) in the
|
||||
decoded output.
|
||||
vLLM `skip_special_tokens` (default on). Off keeps
|
||||
chat-template markers like `<|im_end|>` in output.
|
||||
</InfoHint>
|
||||
</div>
|
||||
<Switch
|
||||
|
|
|
|||
|
|
@ -1,13 +1,11 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// Per-provider sampling capability matrix. Sourced from each
|
||||
// provider's chat-completion docs (2026-05). The panel hides params
|
||||
// the active provider does not accept so users never move a knob that
|
||||
// would be silently dropped or rejected.
|
||||
// When adding a new knob: default it to false on every SaaS bucket;
|
||||
// only local backends + the permissive openrouter bucket should
|
||||
// expose llama.cpp-specific samplers.
|
||||
// Per-provider sampling capability matrix sourced from each provider's
|
||||
// chat-completion docs (2026-05); panel hides params the active
|
||||
// provider would silently drop or reject.
|
||||
// New knobs: default to false on every SaaS bucket; only local +
|
||||
// openrouter expose llama.cpp samplers.
|
||||
export interface ProviderCapabilities {
|
||||
/** OpenAI gpt-5.x / o-series reject via /v1/responses. */
|
||||
temperature: boolean;
|
||||
|
|
@ -76,17 +74,12 @@ export interface ProviderCapabilities {
|
|||
}
|
||||
|
||||
/**
|
||||
* Per-provider stop-sequence max count. Resolved by
|
||||
* `getProviderStopMax(providerType)`. Mirrors the backend's
|
||||
* `provider_info.stop_max` for the same provider type.
|
||||
* - openai: 4 (Chat Completions hard cap; Responses drops stop)
|
||||
* - anthropic: 16 (client-side guard; docs publish no max)
|
||||
* - kimi: 5 (https://platform.kimi.ai/docs/api/chat)
|
||||
* - deepseek: 16 (https://api-docs.deepseek.com/api/create-chat-completion)
|
||||
* - mistral: 16 (no documented max; widen to permissive default)
|
||||
* - gemini: 4 (https://ai.google.dev/gemini-api/docs/openai inherits OAI cap)
|
||||
* - openrouter: 4 (normalises to OpenAI's chat schema)
|
||||
* - default: 16 (covers ollama, vllm, llama.cpp, custom)
|
||||
* Per-provider stop-sequence max count. Mirrors backend `stop_max`.
|
||||
* openai 4 (Chat hard cap; Responses drops stop)
|
||||
* anthropic 16 (client-side guard, docs no max)
|
||||
* kimi 5 (https://platform.kimi.ai/docs/api/chat)
|
||||
* deepseek 16 (https://api-docs.deepseek.com/api/create-chat-completion)
|
||||
* mistral 16, gemini 4, openrouter 4, default 16 (ollama/vllm/llama.cpp/custom)
|
||||
*/
|
||||
const PROVIDER_STOP_MAX: Record<string, number> = {
|
||||
openai: 4,
|
||||
|
|
@ -114,13 +107,9 @@ export type ServiceTierOption =
|
|||
| "standard_only";
|
||||
|
||||
/**
|
||||
* Legal `service_tier` values per provider. Anthropic exposes only
|
||||
* `auto` and `standard_only`. OpenAI in Studio is routed through
|
||||
* `/v1/responses`, which the live docs list as
|
||||
* `auto|default|flex|priority`; `scale` is excluded here even though
|
||||
* the openai-python SDK type happens to include it. Other providers
|
||||
* fall through to a permissive `auto` / `default` pair so the picker
|
||||
* stays usable for OpenAI-compat backends.
|
||||
* Legal `service_tier` per provider. anthropic=auto|standard_only;
|
||||
* openai (/v1/responses)=auto|default|flex|priority (scale excluded
|
||||
* though SDK lists it); others fall through to auto|default.
|
||||
*/
|
||||
export function getServiceTierOptions(
|
||||
providerType: string | null | undefined,
|
||||
|
|
@ -179,13 +168,12 @@ export function clampReasoningEffortToLevels(
|
|||
*/
|
||||
export const EXTERNAL_MAX_OUTPUT_TOKENS = 32768;
|
||||
|
||||
// Per-model max-output caps from each provider's docs (verified May 2026):
|
||||
// Per-model max-output caps (verified May 2026). Longer prefixes first
|
||||
// so .startsWith() picks the specific id over the family root.
|
||||
// OpenAI: developers.openai.com/api/docs/models/<model>
|
||||
// Anthropic: platform.claude.com/docs/en/about-claude/models/overview
|
||||
// Gemini: ai.google.dev/gemini-api/docs/models
|
||||
// DeepSeek: api-docs.deepseek.com/quick_start/pricing
|
||||
// Order matters: list specific chat-class ids before broader gpt-5 /
|
||||
// claude-opus-4 entries so the longer prefix wins via .startsWith().
|
||||
const EXTERNAL_MAX_OUTPUT_TOKENS_BY_MODEL: Array<{
|
||||
providerType: string;
|
||||
prefixes: readonly string[];
|
||||
|
|
@ -262,16 +250,13 @@ function _inferProviderFromOpenrouterId(
|
|||
return null;
|
||||
}
|
||||
|
||||
// Gates the composer's Search button. Backend translates
|
||||
// enable_tools:["web_search"] into each provider's tool schema:
|
||||
// OpenAI: tools:[{type:"web_search"}] on /v1/responses
|
||||
// Anthropic: tools:[{type:"web_search_20250305", max_uses:5}] on /v1/messages
|
||||
// OpenRouter: plugins:[{id:"web"}] (router's universal shape)
|
||||
// Kimi: $web_search builtin (two-call round trip via
|
||||
// _stream_kimi_web_search)
|
||||
// Mistral excluded: their web_search is on the Agents API, not chat
|
||||
// completions, and 400s if injected. Gemini grounded-search needs
|
||||
// matching backend translation first.
|
||||
// Gates the composer's Search button. Backend maps
|
||||
// enable_tools:["web_search"] to each provider's tool schema:
|
||||
// OpenAI: tools:[{type:"web_search"}] on /v1/responses
|
||||
// Anthropic: tools:[{type:"web_search_20250305", max_uses:5}]
|
||||
// OpenRouter: plugins:[{id:"web"}]
|
||||
// Kimi: $web_search builtin (2-call via _stream_kimi_web_search)
|
||||
// Mistral excluded (web_search lives on Agents API, 400s on /v1/chat).
|
||||
export function providerSupportsBuiltinWebSearch(
|
||||
providerType: string | null | undefined,
|
||||
modelId?: string | null | undefined,
|
||||
|
|
@ -418,9 +403,8 @@ export function providerSupportsBuiltinCodeExecution(
|
|||
return false;
|
||||
}
|
||||
|
||||
// OpenAI Responses-API image_generation tool. OpenAI cloud +
|
||||
// Responses-family ids only; backend mirrors via is_openai_cloud so
|
||||
// custom OAI-compat servers reporting provider_type="openai" don't 400.
|
||||
// OpenAI Responses-API image_generation. OpenAI cloud +
|
||||
// Responses-family ids only; backend mirrors via is_openai_cloud.
|
||||
const OPENAI_IMAGE_GENERATION_MODEL_PREFIXES = [
|
||||
"gpt-5.5-pro",
|
||||
"gpt-5.5",
|
||||
|
|
@ -510,10 +494,8 @@ function geminiImageModelAllowsGoogleSearch(modelId: string): boolean {
|
|||
);
|
||||
}
|
||||
|
||||
// Per-provider min on outbound max_tokens. Kimi thinking models need
|
||||
// >=16000 or the response truncates mid-stream. Other providers fall
|
||||
// through to the generic 64. chat-adapter bumps the user's stored
|
||||
// maxTokens up to the floor on send; the slider min mirrors the same.
|
||||
// Per-provider min on outbound max_tokens. Kimi thinking needs >=16000
|
||||
// (truncates mid-stream below); chat-adapter bumps on send.
|
||||
const EXTERNAL_MIN_OUTPUT_TOKENS_BY_PROVIDER: Record<string, number> = {
|
||||
kimi: 16000,
|
||||
};
|
||||
|
|
@ -902,12 +884,9 @@ const PROVIDER_CAPABILITIES: Record<string, ProviderCapabilities> = {
|
|||
postSamplingProbs: false,
|
||||
},
|
||||
mistral: OPENAI_COMPAT_BASE,
|
||||
// Gemini's native generationConfig accepts temperature, topP, topK,
|
||||
// presencePenalty, frequencyPenalty (not surfaced today), seed and
|
||||
// stopSequences. minP and repetitionPenalty are not part of the
|
||||
// contract -- see https://ai.google.dev/api/rest/v1beta/GenerationConfig.
|
||||
// Backend request shaping lives in _stream_gemini in
|
||||
// studio/backend/core/inference/external_provider.py.
|
||||
// Gemini generationConfig: temperature/topP/topK/presencePenalty/
|
||||
// frequencyPenalty/seed/stopSequences. No minP/repetitionPenalty.
|
||||
// https://ai.google.dev/api/rest/v1beta/GenerationConfig
|
||||
gemini: {
|
||||
temperature: true,
|
||||
topP: true,
|
||||
|
|
@ -1053,14 +1032,9 @@ const PROVIDER_CAPABILITIES: Record<string, ProviderCapabilities> = {
|
|||
|
||||
const DEFAULT_EXTERNAL_CAPABILITIES = OPENAI_COMPAT_BASE;
|
||||
|
||||
// Per-model specialisations:
|
||||
// openai + chat-class (gpt-4o, gpt-4-turbo, gpt-4, gpt-3.5):
|
||||
// full sampling surface (OPENAI_CHAT_CAPABILITIES).
|
||||
// openai + reasoning (gpt-5.x, o1, o3, o4): OPENAI_REASONING_CAPABILITIES.
|
||||
// anthropic + claude-opus-4-7: strips temp/top_p/top_k (Opus only
|
||||
// in 4.7; Sonnet/Haiku don't ship).
|
||||
// deepseek reasoner: hides temp/top_p (silently ignored upstream).
|
||||
// Returns null for local models (caller treats as "every knob applies").
|
||||
// Per-model overrides: openai+chat-class -> OPENAI_CHAT_CAPABILITIES;
|
||||
// anthropic claude-opus-4-7 strips temp/top_p/top_k; deepseek reasoner
|
||||
// hides temp/top_p. Local (no providerType) returns null = every knob.
|
||||
export function getProviderCapabilities(
|
||||
providerType: string | null | undefined,
|
||||
modelId?: string | null | undefined,
|
||||
|
|
@ -1114,12 +1088,9 @@ const NO_REASONING_CAPS: ReasoningCaps = {
|
|||
reasoningEffortLevels: DEFAULT_EFFORT_LEVELS,
|
||||
};
|
||||
|
||||
// Order matters: longest prefixes first so find() picks the right
|
||||
// bucket before the bare-family fallback ("claude-opus-4") sweeps.
|
||||
// Levels per platform.claude.com/docs/en/about-claude/models/overview;
|
||||
// 4.5 line uses budget_tokens mapped by the backend. Legacy 4.x
|
||||
// (opus-4-1 / opus-4 / sonnet-4) supports Extended thinking per the
|
||||
// overview table; sonnet-4 / opus-4 retire 2026-06-15.
|
||||
// Longest prefixes first (find() must match before the bare-family
|
||||
// fallback). Levels per platform.claude.com/docs/en/about-claude/models/overview.
|
||||
// 4.5 line maps to budget_tokens; sonnet-4/opus-4 retire 2026-06-15.
|
||||
const ANTHROPIC_REASONING_MODELS = [
|
||||
{
|
||||
prefixes: ["claude-opus-4-7"],
|
||||
|
|
@ -1373,10 +1344,9 @@ function resolveGeminiReasoningCapabilities(
|
|||
}
|
||||
|
||||
function resolveMistralReasoningCapabilities(modelId: string): ExternalReasoningCapabilities {
|
||||
// magistral-* is native always-on (no reasoning_effort param; 422 if
|
||||
// injected). mistral-{small,medium,vibe-cli}-latest is adjustable
|
||||
// none/low/medium/high. See docs.mistral.ai/studio-api/conversations/
|
||||
// reasoning + mistral.ai/news/magistral.
|
||||
// magistral-*: native always-on (422 on reasoning_effort).
|
||||
// mistral-{small,medium,vibe-cli}-latest: none/low/medium/high.
|
||||
// https://docs.mistral.ai/studio-api/conversations/reasoning
|
||||
if (
|
||||
modelId === "magistral-medium-latest" ||
|
||||
modelId === "magistral-small-latest"
|
||||
|
|
|
|||
|
|
@ -320,11 +320,7 @@ export interface OpenAIChatCompletionsRequest {
|
|||
seed?: number;
|
||||
/** OAI Chat caps at 4; Anthropic mapped to `stop_sequences`. */
|
||||
stop?: string[];
|
||||
/**
|
||||
* Per-provider enum (see getServiceTierOptions). Union stays
|
||||
* permissive; external_provider.py drops values the active provider
|
||||
* doesn't accept.
|
||||
*/
|
||||
/** Per-provider enum (see getServiceTierOptions); external_provider.py drops unsupported values. */
|
||||
service_tier?:
|
||||
| "auto"
|
||||
| "default"
|
||||
|
|
|
|||
|
|
@ -9,11 +9,8 @@ export type ServiceTier =
|
|||
| "scale"
|
||||
| "standard_only";
|
||||
|
||||
// All `number | null` / `boolean | null` fields below follow the same
|
||||
// convention: `null` = field omitted from the wire request (provider
|
||||
// uses its own default). Per-provider capability gating lives in
|
||||
// provider-capabilities.ts; the chat-adapter forwards only when the
|
||||
// active provider's bucket has the matching flag set true.
|
||||
// null = field omitted from wire (provider default).
|
||||
// Per-provider gates in provider-capabilities.ts.
|
||||
export interface InferenceParams {
|
||||
temperature: number;
|
||||
topP: number;
|
||||
|
|
@ -47,10 +44,7 @@ export interface InferenceParams {
|
|||
mirostatEta: number | null;
|
||||
/** OpenRouter `top_a`. Range [0, 1]. */
|
||||
topA: number | null;
|
||||
/**
|
||||
* llama.cpp DRY sampler — multiplier is the master switch (0 disables
|
||||
* the 4-field chain). See llama.cpp/tools/server/README.md.
|
||||
*/
|
||||
/** llama.cpp DRY: multiplier is master switch (0 disables 4-field chain). See llama.cpp/tools/server/README.md. */
|
||||
dryMultiplier: number | null;
|
||||
/** Default 1.75. */
|
||||
dryBase: number | null;
|
||||
|
|
|
|||
|
|
@ -45,12 +45,9 @@ const NUMERIC_INFERENCE_FIELDS = [
|
|||
"maxTokens",
|
||||
] as const satisfies readonly (keyof PersistedInferenceParams)[];
|
||||
|
||||
// `seed` is numeric but nullable (null = no seed on the wire) so it
|
||||
// can't go through the NUMERIC_INFERENCE_FIELDS finite-number filter.
|
||||
// Keep this set in sync with `ServiceTier` in ../types/runtime.ts and
|
||||
// `getServiceTierOptions` in ../provider-capabilities.ts. `scale` is
|
||||
// kept in the allowlist for forward-compat with legacy persisted data
|
||||
// even though the resolver no longer surfaces it for OpenAI Responses.
|
||||
// `seed` is nullable so it skips NUMERIC_INFERENCE_FIELDS' finite-number filter.
|
||||
// Keep in sync with ServiceTier (../types/runtime.ts) and getServiceTierOptions.
|
||||
// "scale" stays for forward-compat with legacy persisted data.
|
||||
const VALID_SERVICE_TIERS = new Set([
|
||||
"auto",
|
||||
"default",
|
||||
|
|
@ -162,15 +159,12 @@ function sanitizeInferenceParams(
|
|||
} else if (typeof value.seed === "number" && Number.isInteger(value.seed)) {
|
||||
params.seed = value.seed;
|
||||
}
|
||||
// stop: cap at 16 (Anthropic's widest supported). The backend's
|
||||
// per-provider stream helper re-truncates to the wire cap. An EMPTY
|
||||
// array must persist (not be sanitized away) so clearing the last
|
||||
// chip actually saves; otherwise the old stops come back on reload.
|
||||
// stop: cap at 16 (Anthropic widest); backend re-truncates per provider.
|
||||
// Empty array MUST persist or clearing the last chip is reverted on reload.
|
||||
if (Array.isArray(value.stop)) {
|
||||
const stops = value.stop.filter((s): s is string => typeof s === "string");
|
||||
params.stop = stops.slice(0, 16);
|
||||
}
|
||||
// serviceTier: nullable enum string.
|
||||
if (value.serviceTier === null) {
|
||||
params.serviceTier = null;
|
||||
} else if (
|
||||
|
|
@ -182,8 +176,8 @@ function sanitizeInferenceParams(
|
|||
if (typeof value.parallelToolCalls === "boolean") {
|
||||
params.parallelToolCalls = value.parallelToolCalls;
|
||||
}
|
||||
// typicalP: nullable float (null = no typ_p on the wire, matching
|
||||
// llama-server's default 1.0). Mirror seed's nullable-float handling.
|
||||
// typicalP: nullable float (null = no typ_p on wire, matches
|
||||
// llama-server default 1.0). Mirrors seed handling.
|
||||
if (value.typicalP === null) {
|
||||
params.typicalP = null;
|
||||
} else if (
|
||||
|
|
@ -192,8 +186,7 @@ function sanitizeInferenceParams(
|
|||
) {
|
||||
params.typicalP = value.typicalP;
|
||||
}
|
||||
// New llama.cpp / OpenRouter samplers — all nullable numbers with
|
||||
// the same handling as `typicalP` / `seed`.
|
||||
// Nullable numeric samplers (same handling as typicalP/seed).
|
||||
for (const key of [
|
||||
"topNSigma",
|
||||
"repeatLastN",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue