diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index c8791c7438..249505b8b3 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -444,6 +444,44 @@ export function ChatSettingsPanel({ const showParallelToolCalls = isExternalModel ? Boolean(providerCapabilities?.parallelToolCalls) : localSamplerSupportsExtras; + // Extended llama.cpp / vLLM / OpenRouter samplers. Same gate as the + // core knobs above: external → providerCapabilities flag; local → + // GGUF-only (safetensors transformers ignores these). + const capAdv = (k: keyof ProviderCapabilities): boolean => + isExternalModel + ? Boolean(providerCapabilities?.[k]) + : localSamplerSupportsExtras; + const advCaps = { + typicalP: capAdv("typicalP"), + topNSigma: capAdv("topNSigma"), + repeatLastN: capAdv("repeatLastN"), + dynatempRange: capAdv("dynatempRange"), + dynatempExponent: capAdv("dynatempExponent"), + mirostat: capAdv("mirostat"), + mirostatTau: capAdv("mirostatTau"), + mirostatEta: capAdv("mirostatEta"), + topA: capAdv("topA"), + dryMultiplier: capAdv("dryMultiplier"), + dryBase: capAdv("dryBase"), + dryAllowedLength: capAdv("dryAllowedLength"), + dryPenaltyLastN: capAdv("dryPenaltyLastN"), + xtcProbability: capAdv("xtcProbability"), + xtcThreshold: capAdv("xtcThreshold"), + minKeep: capAdv("minKeep"), + ignoreEos: capAdv("ignoreEos"), + minTokens: capAdv("minTokens"), + skipSpecialTokens: capAdv("skipSpecialTokens"), + spacesBetweenSpecialTokens: capAdv("spacesBetweenSpecialTokens"), + includeStopStrInOutput: capAdv("includeStopStrInOutput"), + truncatePromptTokens: capAdv("truncatePromptTokens"), + nKeep: capAdv("nKeep"), + nProbs: capAdv("nProbs"), + cachePrompt: capAdv("cachePrompt"), + returnTokens: capAdv("returnTokens"), + timingsPerToken: capAdv("timingsPerToken"), + postSamplingProbs: capAdv("postSamplingProbs"), + }; + const showAdvancedSamplingSection = Object.values(advCaps).some(Boolean); // Per-provider stop cap from provider-capabilities.ts; backend // re-trims on the wire if a stale UI sends more than the upstream // accepts. @@ -1498,6 +1536,504 @@ export function ChatSettingsPanel({ + {showAdvancedSamplingSection ? ( + +
+ {advCaps.typicalP ? ( + + set("typicalP")(v >= 1 ? null : v) + } + displayValue={ + params.typicalP == null || params.typicalP >= 1 + ? "Off" + : undefined + } + info="llama.cpp `typ_p`. Locally typical sampling. 1.0 = off." + /> + ) : null} + {advCaps.topNSigma ? ( + + set("topNSigma")(v <= -1 ? null : v) + } + displayValue={ + params.topNSigma == null || params.topNSigma <= -1 + ? "Off" + : undefined + } + info="llama.cpp `top_n_sigma`. Sigma-based truncation. -1 = off." + /> + ) : null} + {advCaps.repeatLastN ? ( + + set("repeatLastN")(v === 0 ? null : v) + } + displayValue={ + params.repeatLastN == null + ? "Off" + : params.repeatLastN === -1 + ? "Ctx" + : undefined + } + info="llama.cpp `repeat_last_n`. Token window the repetition penalty considers. 0 = off, -1 = full context." + /> + ) : null} + {advCaps.dynatempRange ? ( + + set("dynatempRange")(v === 0 ? null : v) + } + displayValue={ + params.dynatempRange == null || params.dynatempRange === 0 + ? "Off" + : undefined + } + info="llama.cpp `dynatemp_range`. Dynamic temperature swing around base temperature. 0 = off." + /> + ) : null} + {advCaps.dynatempExponent ? ( + set("dynatempExponent")(v)} + info="llama.cpp `dynatemp_exponent`. Curve exponent for dynamic temperature. Paired with Dynatemp Range." + /> + ) : null} + {advCaps.mirostat ? ( + + set("mirostat")(v === 0 ? null : v) + } + displayValue={ + params.mirostat == null || params.mirostat === 0 + ? "Off" + : params.mirostat === 1 + ? "v1" + : "v2" + } + info="llama.cpp `mirostat`. Target-entropy sampler. 0 = off, 1 = Mirostat v1, 2 = Mirostat v2." + /> + ) : null} + {advCaps.mirostatTau ? ( + set("mirostatTau")(v)} + info="llama.cpp `mirostat_tau`. Target entropy. Higher = more diverse." + /> + ) : null} + {advCaps.mirostatEta ? ( + set("mirostatEta")(v)} + info="llama.cpp `mirostat_eta`. Learning rate for the entropy controller." + /> + ) : null} + {advCaps.topA ? ( + + set("topA")(v === 0 ? null : v) + } + displayValue={ + params.topA == null || params.topA === 0 ? "Off" : undefined + } + info="OpenRouter `top_a`. Tail-cut sampler scaled by the top token's probability. 0 = off." + /> + ) : null} + {advCaps.dryMultiplier ? ( + + set("dryMultiplier")(v === 0 ? null : v) + } + displayValue={ + params.dryMultiplier == null || params.dryMultiplier === 0 + ? "Off" + : undefined + } + info="llama.cpp DRY sampler. Master switch for the 4-field DRY chain (base / allowed length / penalty last N). 0 = off." + /> + ) : null} + {advCaps.dryBase && (params.dryMultiplier ?? 0) > 0 ? ( + set("dryBase")(v)} + info="llama.cpp `dry_base`. Exponential base for the DRY penalty. Default 1.75." + /> + ) : null} + {advCaps.dryAllowedLength && (params.dryMultiplier ?? 0) > 0 ? ( + set("dryAllowedLength")(v)} + info="llama.cpp `dry_allowed_length`. Repeats up to this length are not penalised. Default 2." + /> + ) : null} + {advCaps.dryPenaltyLastN && (params.dryMultiplier ?? 0) > 0 ? ( + + set("dryPenaltyLastN")(v === 0 ? null : v) + } + displayValue={ + params.dryPenaltyLastN == null + ? "Off" + : params.dryPenaltyLastN === -1 + ? "Ctx" + : undefined + } + info="llama.cpp `dry_penalty_last_n`. Token window the DRY penalty considers. 0 = off, -1 = full context." + /> + ) : null} + {advCaps.xtcProbability ? ( + + set("xtcProbability")(v === 0 ? null : v) + } + displayValue={ + params.xtcProbability == null || params.xtcProbability === 0 + ? "Off" + : undefined + } + info="llama.cpp XTC (eXclude Top Choices). Master switch. 0 = off." + /> + ) : null} + {advCaps.xtcThreshold && (params.xtcProbability ?? 0) > 0 ? ( + set("xtcThreshold")(v)} + info="llama.cpp `xtc_threshold`. Minimum probability for a token to be removable by XTC. Default 0.1." + /> + ) : null} + {advCaps.minKeep ? ( + + set("minKeep")(v === 0 ? null : v) + } + displayValue={ + params.minKeep == null || params.minKeep === 0 + ? "Off" + : undefined + } + info="llama.cpp `min_keep`. Minimum tokens retained past all sampler filters." + /> + ) : null} + {advCaps.minTokens ? ( + + set("minTokens")(v === 0 ? null : v) + } + displayValue={ + params.minTokens == null || params.minTokens === 0 + ? "Off" + : undefined + } + info="llama.cpp + vLLM. Minimum tokens before stop / EOS can fire." + /> + ) : null} + {advCaps.truncatePromptTokens ? ( + + set("truncatePromptTokens")(v === 0 ? null : v) + } + displayValue={ + params.truncatePromptTokens == null || + params.truncatePromptTokens === 0 + ? "Off" + : undefined + } + info="vLLM `truncate_prompt_tokens`. Left-truncate the prompt to this many tokens. 0 = off." + /> + ) : null} + {advCaps.nKeep ? ( + + set("nKeep")(v === 0 ? null : v) + } + displayValue={ + params.nKeep == null + ? "Off" + : params.nKeep === -1 + ? "All" + : undefined + } + info="llama.cpp `n_keep`. Tokens to retain when the context is shifted. 0 = off, -1 = keep all." + /> + ) : null} + {advCaps.nProbs ? ( + + set("nProbs")(v === 0 ? null : v) + } + displayValue={ + params.nProbs == null || params.nProbs === 0 + ? "Off" + : undefined + } + info="llama.cpp `n_probs`. Return the top-N token probabilities per token (diagnostic)." + /> + ) : null} + {advCaps.ignoreEos ? ( +
+
+ + Ignore EOS + + + llama.cpp + vLLM. Keep generating past the model's + end-of-sequence token. Useful for forcing long replies. + +
+ set("ignoreEos")(v ? true : null)} + aria-label="Ignore EOS" + /> +
+ ) : null} + {advCaps.skipSpecialTokens ? ( +
+
+ + Skip Special Tokens + + + vLLM `skip_special_tokens`. Default on. Turn off to keep + chat-template markers (e.g. `<|im_end|>`) in the + decoded output. + +
+ + set("skipSpecialTokens")(v ? null : false) + } + aria-label="Skip special tokens" + /> +
+ ) : null} + {advCaps.spacesBetweenSpecialTokens ? ( +
+
+ + Spaces Between Special Tokens + + + vLLM `spaces_between_special_tokens`. Default on. + +
+ + set("spacesBetweenSpecialTokens")(v ? null : false) + } + aria-label="Spaces between special tokens" + /> +
+ ) : null} + {advCaps.includeStopStrInOutput ? ( +
+
+ + Include Stop String + + + vLLM `include_stop_str_in_output`. Echo the matched stop + string back in the response (useful for agentic tools). + +
+ + set("includeStopStrInOutput")(v ? true : null) + } + aria-label="Include stop string in output" + /> +
+ ) : null} + {advCaps.cachePrompt ? ( +
+
+ + Cache Prompt + + + llama.cpp `cache_prompt`. Default on. Reuses the KV cache + across requests with shared prefixes. + +
+ + set("cachePrompt")(v ? null : false) + } + aria-label="Cache prompt" + /> +
+ ) : null} + {advCaps.returnTokens ? ( +
+
+ + Return Tokens + + + llama.cpp `return_tokens`. Include the raw token ids in + the response (debug). + +
+ + set("returnTokens")(v ? true : null) + } + aria-label="Return tokens" + /> +
+ ) : null} + {advCaps.timingsPerToken ? ( +
+
+ + Timings Per Token + + + llama.cpp `timings_per_token`. Per-token wall-clock + timings in the response (perf debug). + +
+ + set("timingsPerToken")(v ? true : null) + } + aria-label="Timings per token" + /> +
+ ) : null} + {advCaps.postSamplingProbs ? ( +
+
+ + Post-Sampling Probs + + + llama.cpp `post_sampling_probs`. Report the + post-sampling distribution (sampler debug). + +
+ + set("postSamplingProbs")(v ? true : null) + } + aria-label="Post-sampling probs" + /> +
+ ) : null} +
+ + ) : null} + {!isExternalModel ? (