From facdff9ad7f3591f80bea84b2bce8cb64b53c3a8 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 26 May 2026 14:34:32 +0000 Subject: [PATCH] Add extended llama.cpp samplers + OpenRouter top_a (PR #5711) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cross-checked every supported sampling field against each provider's live docs + LiteLLM's drop_params surface + the llama.cpp server README. Pulled in the most-asked-for samplers that the PR was missing. New ProviderCapabilities flags (default false on every SaaS provider since none accept these): - typicalP (already shipped one commit prior) - topNSigma llama.cpp `top_n_sigma` - repeatLastN llama.cpp `repeat_last_n` (paired w/ repeat_penalty) - dynatempRange llama.cpp `dynatemp_range` - dynatempExponent llama.cpp `dynatemp_exponent` - mirostat llama.cpp `mirostat` mode (0/1/2) - mirostatTau llama.cpp `mirostat_tau` - mirostatEta llama.cpp `mirostat_eta` - topA OpenRouter `top_a` (alternate dynamic-top-P) Capability bucketing split: ALL_SUPPORTED retired in favor of - LOCAL_LLAMA_CAPABILITIES -> custom / vllm / ollama / llama_cpp (full llama.cpp sampler chain, top_a off — not a llama.cpp field) - OPENROUTER_CAPABILITIES -> openrouter (gateway's documented set incl. top_a, llama.cpp-only knobs off because OpenRouter docs don't list them and they'd be silently dropped on most underlying routes) InferenceParams gains 8 nullable-number fields (mirroring `seed`'s "null = unset, finite-number = forwarded" shape). DEFAULT_INFERENCE_PARAMS defaults each to null. Persistence handler in chat-settings-storage mirrors typicalP's nullable-float handling for all 8. Backend: - 8 new ChatCompletionRequest fields with appropriate `ge`/`le` validators (mirostat 0..2, ranges 0.0..1.0 where applicable). - llama_cpp.py: signatures + payload forwarding extended on all three builders (chat-completion, agentic tool-loop, final-pass) so the new fields survive the local tool-loop too. `is not None` gating so defaults (e.g. mirostat=0) reach the wire only when the caller explicitly opted in. - routes/inference.py: _build_passthrough_payload extends to the extended sampler chain; 3 call sites (generate_chat_completion, generate_chat_completion_with_tools, _build_passthrough_payload) forward each field from `payload.*`. Frontend chat-adapter: external branch forwards only when capability allows (so OpenRouter gets top_a but not mirostat, local gets mirostat but not top_a); local branch forwards unconditionally when the value is meaningful (e.g. mirostat != 0, dynatemp_range > 0). Test pinning the new field round-trip through _build_passthrough_payload added; full PR-touched suite now 163 passing (was 161). References: - llama.cpp server params: https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md - OpenRouter params: https://openrouter.ai/docs/api/reference/parameters - LiteLLM provider params: https://docs.litellm.ai/docs/completion/input --- studio/backend/core/inference/llama_cpp.py | 59 +++++++ studio/backend/models/inference.py | 58 +++++++ studio/backend/routes/inference.py | 45 +++++ .../tests/test_sampling_params_routing.py | 55 ++++++ .../src/features/chat/api/chat-adapter.ts | 74 +++++++++ .../features/chat/provider-capabilities.ts | 156 ++++++++++++++++-- .../frontend/src/features/chat/types/api.ts | 19 +++ .../src/features/chat/types/runtime.ts | 27 +++ .../chat/utils/chat-settings-storage.ts | 19 +++ 9 files changed, 501 insertions(+), 11 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 5fb7395c00..6211a2a57c 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -4249,6 +4249,13 @@ class LlamaCppBackend: seed: Optional[int] = None, parallel_tool_calls: Optional[bool] = None, typical_p: Optional[float] = None, + top_n_sigma: Optional[float] = None, + repeat_last_n: Optional[int] = None, + dynatemp_range: Optional[float] = None, + dynatemp_exponent: Optional[float] = None, + mirostat: Optional[int] = None, + mirostat_tau: Optional[float] = None, + mirostat_eta: Optional[float] = None, ) -> Generator[str | dict, None, None]: """ Send a chat completion request to llama-server and stream tokens back. @@ -4312,6 +4319,23 @@ class LlamaCppBackend: # so we only forward it when the caller explicitly sets one. if typical_p is not None: payload["typical_p"] = typical_p + # Extended llama.cpp sampler chain (top_n_sigma, repeat_last_n, + # dynatemp_*, mirostat_*). All llama.cpp-specific; the frontend + # capability map gates them to local backends only. + if top_n_sigma is not None: + payload["top_n_sigma"] = top_n_sigma + if repeat_last_n is not None: + payload["repeat_last_n"] = repeat_last_n + if dynatemp_range is not None: + payload["dynatemp_range"] = dynatemp_range + if dynatemp_exponent is not None: + payload["dynatemp_exponent"] = dynatemp_exponent + if mirostat is not None: + payload["mirostat"] = mirostat + if mirostat_tau is not None: + payload["mirostat_tau"] = mirostat_tau + if mirostat_eta is not None: + payload["mirostat_eta"] = mirostat_eta payload["stream_options"] = {"include_usage": True} url = f"{self.base_url}/v1/chat/completions" @@ -4458,6 +4482,13 @@ class LlamaCppBackend: seed: Optional[int] = None, parallel_tool_calls: Optional[bool] = None, typical_p: Optional[float] = None, + top_n_sigma: Optional[float] = None, + repeat_last_n: Optional[int] = None, + dynatemp_range: Optional[float] = None, + dynatemp_exponent: Optional[float] = None, + mirostat: Optional[int] = None, + mirostat_tau: Optional[float] = None, + mirostat_eta: Optional[float] = None, ) -> Generator[dict, None, None]: """ Agentic loop: let the model call tools, execute them, and continue. @@ -4557,6 +4588,20 @@ class LlamaCppBackend: payload["parallel_tool_calls"] = parallel_tool_calls if typical_p is not None: payload["typical_p"] = typical_p + if top_n_sigma is not None: + payload["top_n_sigma"] = top_n_sigma + if repeat_last_n is not None: + payload["repeat_last_n"] = repeat_last_n + if dynatemp_range is not None: + payload["dynatemp_range"] = dynatemp_range + if dynatemp_exponent is not None: + payload["dynatemp_exponent"] = dynatemp_exponent + if mirostat is not None: + payload["mirostat"] = mirostat + if mirostat_tau is not None: + payload["mirostat_tau"] = mirostat_tau + if mirostat_eta is not None: + payload["mirostat_eta"] = mirostat_eta try: _auth_headers = ( @@ -5251,6 +5296,20 @@ class LlamaCppBackend: stream_payload["parallel_tool_calls"] = parallel_tool_calls if typical_p is not None: stream_payload["typical_p"] = typical_p + if top_n_sigma is not None: + stream_payload["top_n_sigma"] = top_n_sigma + if repeat_last_n is not None: + stream_payload["repeat_last_n"] = repeat_last_n + if dynatemp_range is not None: + stream_payload["dynatemp_range"] = dynatemp_range + if dynatemp_exponent is not None: + stream_payload["dynatemp_exponent"] = dynatemp_exponent + if mirostat is not None: + stream_payload["mirostat"] = mirostat + if mirostat_tau is not None: + stream_payload["mirostat_tau"] = mirostat_tau + if mirostat_eta is not None: + stream_payload["mirostat_eta"] = mirostat_eta stream_payload["stream_options"] = {"include_usage": True} cumulative = "" diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index b2c12b82c4..214e7e99e4 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -840,6 +840,64 @@ class ChatCompletionRequest(BaseModel): "/v1/chat/completions." ), ) + top_n_sigma: Optional[float] = Field( + None, + description = ( + "llama.cpp `top_n_sigma` sampler. -1.0 disables (server " + "default). Local only — no SaaS provider accepts it." + ), + ) + repeat_last_n: Optional[int] = Field( + None, + description = ( + "llama.cpp `repeat_last_n`. 0 disables, -1 = ctx-size. " + "Pairs with repetition_penalty. Local only." + ), + ) + dynatemp_range: Optional[float] = Field( + None, + ge = 0.0, + description = ( + "llama.cpp `dynatemp_range`. 0.0 disables. Local only." + ), + ) + dynatemp_exponent: Optional[float] = Field( + None, + ge = 0.0, + description = ( + "llama.cpp `dynatemp_exponent`. Local only; pairs with " + "dynatemp_range." + ), + ) + mirostat: Optional[int] = Field( + None, + ge = 0, + le = 2, + description = ( + "llama.cpp `mirostat` mode. 0 = disabled, 1 = Mirostat, " + "2 = Mirostat 2.0. Local only." + ), + ) + mirostat_tau: Optional[float] = Field( + None, + ge = 0.0, + description = "llama.cpp `mirostat_tau` target entropy. Local only.", + ) + mirostat_eta: Optional[float] = Field( + None, + ge = 0.0, + description = "llama.cpp `mirostat_eta` learning rate. Local only.", + ) + top_a: Optional[float] = Field( + None, + ge = 0.0, + le = 1.0, + description = ( + "OpenRouter `top_a` alternate dynamic-top-P. Documented at " + "https://openrouter.ai/docs/api/reference/parameters. " + "OpenRouter-only; other gateways silently drop it." + ), + ) fast_mode: Optional[bool] = Field( None, description = ( diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 5d71b446b5..8f079cf462 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -2518,6 +2518,13 @@ async def openai_chat_completions( seed = payload.seed, parallel_tool_calls = payload.parallel_tool_calls, typical_p = payload.typical_p, + top_n_sigma = payload.top_n_sigma, + repeat_last_n = payload.repeat_last_n, + dynatemp_range = payload.dynatemp_range, + dynatemp_exponent = payload.dynatemp_exponent, + mirostat = payload.mirostat, + mirostat_tau = payload.mirostat_tau, + mirostat_eta = payload.mirostat_eta, ) _tool_sentinel = object() @@ -2692,6 +2699,13 @@ async def openai_chat_completions( seed = payload.seed, parallel_tool_calls = payload.parallel_tool_calls, typical_p = payload.typical_p, + top_n_sigma = payload.top_n_sigma, + repeat_last_n = payload.repeat_last_n, + dynatemp_range = payload.dynatemp_range, + dynatemp_exponent = payload.dynatemp_exponent, + mirostat = payload.mirostat, + mirostat_tau = payload.mirostat_tau, + mirostat_eta = payload.mirostat_eta, ) _gguf_sentinel = object() @@ -4966,6 +4980,13 @@ def _build_passthrough_payload( seed = None, parallel_tool_calls = None, typical_p = None, + top_n_sigma = None, + repeat_last_n = None, + dynatemp_range = None, + dynatemp_exponent = None, + mirostat = None, + mirostat_tau = None, + mirostat_eta = None, tool_choice = "auto", response_format = None, chat_template_kwargs = None, @@ -5021,6 +5042,23 @@ def _build_passthrough_payload( # gates it to local only. if typical_p is not None: body["typical_p"] = typical_p + # Extended llama.cpp sampler chain. All llama.cpp-specific; the + # frontend capability map gates them to local backends only. Server + # silently ignores fields it doesn't recognise. + if top_n_sigma is not None: + body["top_n_sigma"] = top_n_sigma + if repeat_last_n is not None: + body["repeat_last_n"] = repeat_last_n + if dynatemp_range is not None: + body["dynatemp_range"] = dynatemp_range + if dynatemp_exponent is not None: + body["dynatemp_exponent"] = dynatemp_exponent + if mirostat is not None: + body["mirostat"] = mirostat + if mirostat_tau is not None: + body["mirostat_tau"] = mirostat_tau + if mirostat_eta is not None: + body["mirostat_eta"] = mirostat_eta if response_format is not None: # llama-server applies a GBNF grammar derived from the JSON schema # when response_format is present. Field is documented flat at the @@ -5456,6 +5494,13 @@ def _build_openai_passthrough_body(payload, backend_ctx = None) -> dict: seed = payload.seed, parallel_tool_calls = payload.parallel_tool_calls, typical_p = payload.typical_p, + top_n_sigma = payload.top_n_sigma, + repeat_last_n = payload.repeat_last_n, + dynatemp_range = payload.dynatemp_range, + dynatemp_exponent = payload.dynatemp_exponent, + mirostat = payload.mirostat, + mirostat_tau = payload.mirostat_tau, + mirostat_eta = payload.mirostat_eta, tool_choice = tool_choice, response_format = _extract_response_format(payload), chat_template_kwargs = tpl_kwargs, diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index 50714fcac4..19fd9daf34 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -1001,6 +1001,61 @@ def test_local_anthropic_passthrough_helpers_accept_parallel_tool_calls(): assert body.get("parallel_tool_calls") is False, body +def test_local_passthrough_forwards_extended_llama_cpp_samplers(): + """The local llama.cpp passthrough payload builder must forward + the extended sampler chain when set: top_n_sigma, repeat_last_n, + dynatemp_range/exponent, mirostat/mirostat_tau/mirostat_eta. Each + is gated `is not None` so a default-off value (e.g. mirostat=0) is + still forwarded explicitly when the caller opted in. + """ + from routes import inference as route_mod + + body = route_mod._build_passthrough_payload( + openai_messages = [{"role": "user", "content": "hi"}], + openai_tools = None, + temperature = 0.6, + top_p = 0.95, + top_k = 20, + max_tokens = 64, + stream = True, + top_n_sigma = 1.5, + repeat_last_n = 128, + dynatemp_range = 0.2, + dynatemp_exponent = 1.5, + mirostat = 2, + mirostat_tau = 5.0, + mirostat_eta = 0.1, + ) + assert body.get("top_n_sigma") == 1.5 + assert body.get("repeat_last_n") == 128 + assert body.get("dynatemp_range") == 0.2 + assert body.get("dynatemp_exponent") == 1.5 + assert body.get("mirostat") == 2 + assert body.get("mirostat_tau") == 5.0 + assert body.get("mirostat_eta") == 0.1 + + # Unset = absent from body so llama-server falls back to defaults. + body2 = route_mod._build_passthrough_payload( + openai_messages = [{"role": "user", "content": "hi"}], + openai_tools = None, + temperature = 0.6, + top_p = 0.95, + top_k = 20, + max_tokens = 64, + stream = True, + ) + for key in ( + "top_n_sigma", + "repeat_last_n", + "dynatemp_range", + "dynatemp_exponent", + "mirostat", + "mirostat_tau", + "mirostat_eta", + ): + assert key not in body2, body2 + + def test_local_passthrough_forwards_typical_p_when_set(): """`typical_p` is a llama.cpp-specific sampler (`typ_p` in the sampler chain). The local-llama-cpp passthrough payload builder must diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index 52e2b87d2d..b288589432 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -1596,6 +1596,55 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { params.typicalP !== 1 ? { typical_p: params.typicalP } : {}), + // llama.cpp `top_n_sigma`. -1 disables (server default); + // only forward meaningful values. + ...(externalCapabilities?.topNSigma && + params.topNSigma !== null && + params.topNSigma !== -1 + ? { top_n_sigma: params.topNSigma } + : {}), + // llama.cpp `repeat_last_n`. Pairs with repetition_penalty. + ...(externalCapabilities?.repeatLastN && + params.repeatLastN !== null + ? { repeat_last_n: params.repeatLastN } + : {}), + // llama.cpp dynamic-temperature. Only forward when the + // user opted in (range > 0). + ...(externalCapabilities?.dynatempRange && + params.dynatempRange !== null && + params.dynatempRange > 0 + ? { + dynatemp_range: params.dynatempRange, + ...(externalCapabilities?.dynatempExponent && + params.dynatempExponent !== null + ? { dynatemp_exponent: params.dynatempExponent } + : {}), + } + : {}), + // llama.cpp Mirostat. Mode 0 disables; only forward the + // sub-params when mode is enabled. + ...(externalCapabilities?.mirostat && + params.mirostat !== null && + params.mirostat !== 0 + ? { + mirostat: params.mirostat, + ...(externalCapabilities?.mirostatTau && + params.mirostatTau !== null + ? { mirostat_tau: params.mirostatTau } + : {}), + ...(externalCapabilities?.mirostatEta && + params.mirostatEta !== null + ? { mirostat_eta: params.mirostatEta } + : {}), + } + : {}), + // OpenRouter `top_a` (alternate dynamic top-P). Documented + // range [0, 1]; 0 disables. + ...(externalCapabilities?.topA && + params.topA !== null && + params.topA > 0 + ? { top_a: params.topA } + : {}), // Built-in tools: Search pill maps to provider-side // web_search (currently OpenAI / Anthropic / OpenRouter / // Kimi); Code pill maps to Anthropic's server-side @@ -1722,6 +1771,31 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { ...(params.typicalP !== null && params.typicalP !== 1 ? { typical_p: params.typicalP } : {}), + ...(params.topNSigma !== null && params.topNSigma !== -1 + ? { top_n_sigma: params.topNSigma } + : {}), + ...(params.repeatLastN !== null + ? { repeat_last_n: params.repeatLastN } + : {}), + ...(params.dynatempRange !== null && params.dynatempRange > 0 + ? { + dynatemp_range: params.dynatempRange, + ...(params.dynatempExponent !== null + ? { dynatemp_exponent: params.dynatempExponent } + : {}), + } + : {}), + ...(params.mirostat !== null && params.mirostat !== 0 + ? { + mirostat: params.mirostat, + ...(params.mirostatTau !== null + ? { mirostat_tau: params.mirostatTau } + : {}), + ...(params.mirostatEta !== null + ? { mirostat_eta: params.mirostatEta } + : {}), + } + : {}), parallel_tool_calls: params.parallelToolCalls, image_base64: imageBase64, audio_base64: audioBase64, diff --git a/studio/frontend/src/features/chat/provider-capabilities.ts b/studio/frontend/src/features/chat/provider-capabilities.ts index 5a48aa8fc6..f3e9d6261b 100644 --- a/studio/frontend/src/features/chat/provider-capabilities.ts +++ b/studio/frontend/src/features/chat/provider-capabilities.ts @@ -69,6 +69,48 @@ export interface ProviderCapabilities { * permissive {custom, vllm, ollama, llama_cpp} buckets. */ typicalP: boolean; + /** + * llama.cpp `top_n_sigma` sampler (newer top-sigma cutoff). Local + * only; -1 disables. + * https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md + */ + topNSigma: boolean; + /** + * llama.cpp repetition window (`repeat_last_n`). Pairs with + * `repeat_penalty`. Local only; 0 disables, -1 = ctx-size. + */ + repeatLastN: boolean; + /** + * llama.cpp dynamic temperature range (`dynatemp_range`). Local + * only; 0.0 disables. + */ + dynatempRange: boolean; + /** + * llama.cpp dynamic temperature exponent (`dynatemp_exponent`). + * Local only. Paired with dynatempRange. + */ + dynatempExponent: boolean; + /** + * llama.cpp Mirostat sampling mode (`mirostat`). Local only. + * 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0. + */ + mirostat: boolean; + /** + * llama.cpp Mirostat target entropy (`mirostat_tau`). Local only. + * Only meaningful when mirostat != 0. + */ + mirostatTau: boolean; + /** + * llama.cpp Mirostat learning rate (`mirostat_eta`). Local only. + * Only meaningful when mirostat != 0. + */ + mirostatEta: boolean; + /** + * OpenRouter `top_a` (alternate dynamic-top-P). Documented at + * https://openrouter.ai/docs/api/reference/parameters. Other + * gateways silently drop it; we surface it only for openrouter. + */ + topA: boolean; } /** @@ -413,9 +455,21 @@ const OPENAI_COMPAT_BASE: ProviderCapabilities = { serviceTier: false, parallelToolCalls: true, typicalP: false, + topNSigma: false, + repeatLastN: false, + dynatempRange: false, + dynatempExponent: false, + mirostat: false, + mirostatTau: false, + mirostatEta: false, + topA: false, }; -const ALL_SUPPORTED: ProviderCapabilities = { +// Local llama.cpp-style backends (own llama-server, vLLM with extended +// sampler support, Ollama). Exposes the full llama.cpp sampler chain +// (typical_p / top_n_sigma / mirostat / dynatemp / repeat_last_n) but +// not OpenRouter's gateway-specific top_a. +const LOCAL_LLAMA_CAPABILITIES: ProviderCapabilities = { temperature: true, topP: true, topK: true, @@ -428,6 +482,44 @@ const ALL_SUPPORTED: ProviderCapabilities = { serviceTier: false, parallelToolCalls: true, typicalP: true, + topNSigma: true, + repeatLastN: true, + dynatempRange: true, + dynatempExponent: true, + mirostat: true, + mirostatTau: true, + mirostatEta: true, + topA: false, +}; + +// OpenRouter is a router-of-routers: the gateway accepts a wider set +// of OpenAI-style sampling fields than any single upstream supports +// and silently drops what the chosen route does not, per +// https://openrouter.ai/docs/api/reference/parameters. Surface the +// router's full documented set (incl. top_a) and leave the +// llama.cpp-only knobs off (the docs don't list them, so we don't +// either even though many openrouter routes terminate at llama.cpp). +const OPENROUTER_CAPABILITIES: ProviderCapabilities = { + temperature: true, + topP: true, + topK: true, + minP: true, + repetitionPenalty: true, + presencePenalty: true, + frequencyPenalty: true, + seed: true, + stop: true, + serviceTier: false, + parallelToolCalls: true, + typicalP: false, + topNSigma: false, + repeatLastN: false, + dynatempRange: false, + dynatempExponent: false, + mirostat: false, + mirostatTau: false, + mirostatEta: false, + topA: true, }; // Reasoning-class OpenAI models served via /v1/responses fix temperature @@ -451,6 +543,14 @@ const OPENAI_REASONING_CAPABILITIES: ProviderCapabilities = { serviceTier: true, parallelToolCalls: true, typicalP: false, + topNSigma: false, + repeatLastN: false, + dynatempRange: false, + dynatempExponent: false, + mirostat: false, + mirostatTau: false, + mirostatEta: false, + topA: false, }; const OPENAI_CHAT_CAPABILITIES: ProviderCapabilities = { temperature: true, @@ -467,6 +567,14 @@ const OPENAI_CHAT_CAPABILITIES: ProviderCapabilities = { serviceTier: true, parallelToolCalls: true, typicalP: false, + topNSigma: false, + repeatLastN: false, + dynatempRange: false, + dynatempExponent: false, + mirostat: false, + mirostatTau: false, + mirostatEta: false, + topA: false, }; // Prefix list for OpenAI reasoning-class model ids. Kept in sync with @@ -553,6 +661,14 @@ const PROVIDER_CAPABILITIES: Record = { serviceTier: true, parallelToolCalls: true, typicalP: false, + topNSigma: false, + repeatLastN: false, + dynatempRange: false, + dynatempExponent: false, + mirostat: false, + mirostatTau: false, + mirostatEta: false, + topA: false, }, mistral: OPENAI_COMPAT_BASE, gemini: OPENAI_COMPAT_BASE, @@ -581,6 +697,14 @@ const PROVIDER_CAPABILITIES: Record = { serviceTier: false, parallelToolCalls: false, typicalP: false, + topNSigma: false, + repeatLastN: false, + dynatempRange: false, + dynatempExponent: false, + mirostat: false, + mirostatTau: false, + mirostatEta: false, + topA: false, }, // DeepSeek deprecated presence/frequency penalty in their current docs. // Chat-class defaults (deepseek-chat / deepseek-v4-flash non-thinking): @@ -604,19 +728,29 @@ const PROVIDER_CAPABILITIES: Record = { serviceTier: false, parallelToolCalls: true, typicalP: false, + topNSigma: false, + repeatLastN: false, + dynatempRange: false, + dynatempExponent: false, + mirostat: false, + mirostatTau: false, + mirostatEta: false, + topA: false, }, qwen: OPENAI_COMPAT_BASE, huggingface: OPENAI_COMPAT_BASE, - // OpenRouter silently drops params the target model does not support, so we - // surface every knob and let the gateway handle the per-model fan-out. - openrouter: ALL_SUPPORTED, - // Local OpenAI-compatible connections are proxied through the OpenAI backend - // path, but vLLM/Ollama/llama.cpp users often want top_k / min_p / - // repetition controls, so be permissive. - custom: ALL_SUPPORTED, - vllm: ALL_SUPPORTED, - ollama: ALL_SUPPORTED, - llama_cpp: ALL_SUPPORTED, + // OpenRouter surfaces the gateway's documented sampling field set + // (incl. top_a). llama.cpp-specific knobs (typical_p, mirostat, + // dynatemp, top_n_sigma, repeat_last_n) are gated off because the + // OpenRouter API docs do not list them; they would be silently + // dropped on most underlying models. + openrouter: OPENROUTER_CAPABILITIES, + // Local OpenAI-compatible connections terminate at llama-server- + // style backends — full llama.cpp sampler chain available. + custom: LOCAL_LLAMA_CAPABILITIES, + vllm: LOCAL_LLAMA_CAPABILITIES, + ollama: LOCAL_LLAMA_CAPABILITIES, + llama_cpp: LOCAL_LLAMA_CAPABILITIES, }; const DEFAULT_EXTERNAL_CAPABILITIES = OPENAI_COMPAT_BASE; diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index 0db72ac5c2..055764d0df 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -312,6 +312,25 @@ export interface OpenAIChatCompletionsRequest { * permissive {custom, vllm, ollama, llama_cpp} buckets. */ typical_p?: number; + /** llama.cpp `top_n_sigma`. -1 disables. Local only. */ + top_n_sigma?: number; + /** llama.cpp `repeat_last_n`. 0 disables, -1 = ctx-size. Local only. */ + repeat_last_n?: number; + /** llama.cpp `dynatemp_range`. 0 disables. Local only. */ + dynatemp_range?: number; + /** llama.cpp `dynatemp_exponent`. Local only, paired with dynatemp_range. */ + dynatemp_exponent?: number; + /** llama.cpp `mirostat` (0/1/2). 0 disables. Local only. */ + mirostat?: number; + /** llama.cpp `mirostat_tau` target entropy. Local only. */ + mirostat_tau?: number; + /** llama.cpp `mirostat_eta` learning rate. Local only. */ + mirostat_eta?: number; + /** + * OpenRouter `top_a` (alternate dynamic-top-P). + * https://openrouter.ai/docs/api/reference/parameters — gateway-only. + */ + top_a?: number; /** * Anthropic fast-mode toggle. Opus 4.6 / 4.7 only; backend drops * silently on every other model + provider. See diff --git a/studio/frontend/src/features/chat/types/runtime.ts b/studio/frontend/src/features/chat/types/runtime.ts index 0793573583..ab6fd9db63 100644 --- a/studio/frontend/src/features/chat/types/runtime.ts +++ b/studio/frontend/src/features/chat/types/runtime.ts @@ -50,6 +50,25 @@ export interface InferenceParams { * is the llama-server default). `null` = unset (not forwarded). */ typicalP: number | null; + /** llama.cpp `top_n_sigma`. -1 disables. `null` = unset. */ + topNSigma: number | null; + /** llama.cpp `repeat_last_n`. 0 disables, -1 = ctx-size. `null` = unset. */ + repeatLastN: number | null; + /** llama.cpp `dynatemp_range`. 0.0 disables. `null` = unset. */ + dynatempRange: number | null; + /** llama.cpp `dynatemp_exponent`. `null` = unset. */ + dynatempExponent: number | null; + /** llama.cpp `mirostat` mode (0/1/2). 0 disables. `null` = unset. */ + mirostat: number | null; + /** llama.cpp `mirostat_tau` target entropy. `null` = unset. */ + mirostatTau: number | null; + /** llama.cpp `mirostat_eta` learning rate. `null` = unset. */ + mirostatEta: number | null; + /** + * OpenRouter `top_a` alternate dynamic-top-P. OpenRouter-only. + * Range [0, 1]. `null` = unset. + */ + topA: number | null; maxSeqLength: number; maxTokens: number; systemPrompt: string; @@ -77,6 +96,14 @@ export const DEFAULT_INFERENCE_PARAMS: InferenceParams = { serviceTier: null, parallelToolCalls: true, typicalP: null, + topNSigma: null, + repeatLastN: null, + dynatempRange: null, + dynatempExponent: null, + mirostat: null, + mirostatTau: null, + mirostatEta: null, + topA: null, maxSeqLength: 4096, maxTokens: 8192, systemPrompt: "", diff --git a/studio/frontend/src/features/chat/utils/chat-settings-storage.ts b/studio/frontend/src/features/chat/utils/chat-settings-storage.ts index ede79c7098..c954ff0510 100644 --- a/studio/frontend/src/features/chat/utils/chat-settings-storage.ts +++ b/studio/frontend/src/features/chat/utils/chat-settings-storage.ts @@ -192,6 +192,25 @@ function sanitizeInferenceParams( ) { params.typicalP = value.typicalP; } + // New llama.cpp / OpenRouter samplers — all nullable numbers with + // the same handling as `typicalP` / `seed`. + for (const key of [ + "topNSigma", + "repeatLastN", + "dynatempRange", + "dynatempExponent", + "mirostat", + "mirostatTau", + "mirostatEta", + "topA", + ] as const) { + const raw = value[key]; + if (raw === null) { + (params as Record)[key] = null; + } else if (typeof raw === "number" && Number.isFinite(raw)) { + (params as Record)[key] = raw; + } + } // Mirror trustRemoteCode handling so the toggle survives reload // and the /api/chat/settings round-trip. if (typeof value.fastMode === "boolean") {