diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 5d0644ef52..d952bb8660 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -4267,6 +4267,16 @@ class LlamaCppBackend: min_keep: Optional[int] = None, ignore_eos: Optional[bool] = None, min_tokens: Optional[int] = None, + skip_special_tokens: Optional[bool] = None, + spaces_between_special_tokens: Optional[bool] = None, + include_stop_str_in_output: Optional[bool] = None, + truncate_prompt_tokens: Optional[int] = None, + n_keep: Optional[int] = None, + n_probs: Optional[int] = None, + cache_prompt: Optional[bool] = None, + return_tokens: Optional[bool] = None, + timings_per_token: Optional[bool] = None, + post_sampling_probs: Optional[bool] = None, ) -> Generator[str | dict, None, None]: """ Send a chat completion request to llama-server and stream tokens back. @@ -4368,6 +4378,29 @@ class LlamaCppBackend: payload["ignore_eos"] = ignore_eos if min_tokens is not None: payload["min_tokens"] = min_tokens + # vLLM output-shape knobs — forwarded `is not None` so user + # opt-outs (skip_special_tokens=False etc) still reach the wire. + if skip_special_tokens is not None: + payload["skip_special_tokens"] = skip_special_tokens + if spaces_between_special_tokens is not None: + payload["spaces_between_special_tokens"] = spaces_between_special_tokens + if include_stop_str_in_output is not None: + payload["include_stop_str_in_output"] = include_stop_str_in_output + if truncate_prompt_tokens is not None: + payload["truncate_prompt_tokens"] = truncate_prompt_tokens + # llama.cpp context / KV-cache / instrumentation knobs. + if n_keep is not None: + payload["n_keep"] = n_keep + if n_probs is not None: + payload["n_probs"] = n_probs + if cache_prompt is not None: + payload["cache_prompt"] = cache_prompt + if return_tokens is not None: + payload["return_tokens"] = return_tokens + if timings_per_token is not None: + payload["timings_per_token"] = timings_per_token + if post_sampling_probs is not None: + payload["post_sampling_probs"] = post_sampling_probs payload["stream_options"] = {"include_usage": True} url = f"{self.base_url}/v1/chat/completions" @@ -4530,6 +4563,16 @@ class LlamaCppBackend: min_keep: Optional[int] = None, ignore_eos: Optional[bool] = None, min_tokens: Optional[int] = None, + skip_special_tokens: Optional[bool] = None, + spaces_between_special_tokens: Optional[bool] = None, + include_stop_str_in_output: Optional[bool] = None, + truncate_prompt_tokens: Optional[int] = None, + n_keep: Optional[int] = None, + n_probs: Optional[int] = None, + cache_prompt: Optional[bool] = None, + return_tokens: Optional[bool] = None, + timings_per_token: Optional[bool] = None, + post_sampling_probs: Optional[bool] = None, ) -> Generator[dict, None, None]: """ Agentic loop: let the model call tools, execute them, and continue. @@ -4661,6 +4704,26 @@ class LlamaCppBackend: payload["ignore_eos"] = ignore_eos if min_tokens is not None: payload["min_tokens"] = min_tokens + if skip_special_tokens is not None: + payload["skip_special_tokens"] = skip_special_tokens + if spaces_between_special_tokens is not None: + payload["spaces_between_special_tokens"] = spaces_between_special_tokens + if include_stop_str_in_output is not None: + payload["include_stop_str_in_output"] = include_stop_str_in_output + if truncate_prompt_tokens is not None: + payload["truncate_prompt_tokens"] = truncate_prompt_tokens + if n_keep is not None: + payload["n_keep"] = n_keep + if n_probs is not None: + payload["n_probs"] = n_probs + if cache_prompt is not None: + payload["cache_prompt"] = cache_prompt + if return_tokens is not None: + payload["return_tokens"] = return_tokens + if timings_per_token is not None: + payload["timings_per_token"] = timings_per_token + if post_sampling_probs is not None: + payload["post_sampling_probs"] = post_sampling_probs try: _auth_headers = ( @@ -5387,6 +5450,26 @@ class LlamaCppBackend: stream_payload["ignore_eos"] = ignore_eos if min_tokens is not None: stream_payload["min_tokens"] = min_tokens + if skip_special_tokens is not None: + stream_payload["skip_special_tokens"] = skip_special_tokens + if spaces_between_special_tokens is not None: + stream_payload["spaces_between_special_tokens"] = spaces_between_special_tokens + if include_stop_str_in_output is not None: + stream_payload["include_stop_str_in_output"] = include_stop_str_in_output + if truncate_prompt_tokens is not None: + stream_payload["truncate_prompt_tokens"] = truncate_prompt_tokens + if n_keep is not None: + stream_payload["n_keep"] = n_keep + if n_probs is not None: + stream_payload["n_probs"] = n_probs + if cache_prompt is not None: + stream_payload["cache_prompt"] = cache_prompt + if return_tokens is not None: + stream_payload["return_tokens"] = return_tokens + if timings_per_token is not None: + stream_payload["timings_per_token"] = timings_per_token + if post_sampling_probs is not None: + stream_payload["post_sampling_probs"] = post_sampling_probs stream_payload["stream_options"] = {"include_usage": True} cumulative = "" diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index 8dea9c71c9..f712e8dd67 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -1011,6 +1011,85 @@ class ChatCompletionRequest(BaseModel): "it. 0 disables (server default)." ), ) + skip_special_tokens: Optional[bool] = Field( + None, + description = ( + "vLLM `skip_special_tokens` (default true). Forward only when " + "false — i.e. user wants raw special tokens in the output. " + "vLLM only; llama-server / Ollama do not document this field." + ), + ) + spaces_between_special_tokens: Optional[bool] = Field( + None, + description = ( + "vLLM `spaces_between_special_tokens` (default true). Forward " + "only when false. vLLM only." + ), + ) + include_stop_str_in_output: Optional[bool] = Field( + None, + description = ( + "vLLM `include_stop_str_in_output` (default false). Forward " + "only when true — useful for agentic tools that need the " + "matched stop string echoed back. vLLM only." + ), + ) + truncate_prompt_tokens: Optional[int] = Field( + None, + ge = 1, + description = ( + "vLLM `truncate_prompt_tokens` — left-truncate the prompt to " + "this many tokens. Useful for long-context overflow. vLLM " + "only; llama-server / Ollama drop this on the OAI path." + ), + ) + n_keep: Optional[int] = Field( + None, + description = ( + "llama.cpp `n_keep` — tokens to retain when context overflows. " + "0 disables (server default), -1 keeps the whole prompt. " + "Local llama-server only." + ), + ) + n_probs: Optional[int] = Field( + None, + ge = 0, + description = ( + "llama.cpp `n_probs` — return top-N token probabilities per " + "generated token. 0 disables (server default). Local only." + ), + ) + cache_prompt: Optional[bool] = Field( + None, + description = ( + "llama.cpp `cache_prompt` — reuse KV cache across requests " + "with a shared prefix. Default true upstream; forward only " + "when explicitly false (e.g. deterministic benchmarks). " + "Local llama-server only." + ), + ) + return_tokens: Optional[bool] = Field( + None, + description = ( + "llama.cpp `return_tokens` — include raw token IDs in the " + "response. Debug. Local only." + ), + ) + timings_per_token: Optional[bool] = Field( + None, + description = ( + "llama.cpp `timings_per_token` — include per-token speed " + "metrics in the streaming response. Local only." + ), + ) + post_sampling_probs: Optional[bool] = Field( + None, + description = ( + "llama.cpp `post_sampling_probs` — return token probabilities " + "AFTER the sampler chain runs (useful for sampler-tuning). " + "Local only." + ), + ) fast_mode: Optional[bool] = Field( None, description = ( diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 91b5845825..7e38747fc3 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -2588,6 +2588,16 @@ async def openai_chat_completions( min_keep = payload.min_keep, ignore_eos = payload.ignore_eos, min_tokens = payload.min_tokens, + skip_special_tokens = payload.skip_special_tokens, + spaces_between_special_tokens = payload.spaces_between_special_tokens, + include_stop_str_in_output = payload.include_stop_str_in_output, + truncate_prompt_tokens = payload.truncate_prompt_tokens, + n_keep = payload.n_keep, + n_probs = payload.n_probs, + cache_prompt = payload.cache_prompt, + return_tokens = payload.return_tokens, + timings_per_token = payload.timings_per_token, + post_sampling_probs = payload.post_sampling_probs, top_n_sigma = payload.top_n_sigma, repeat_last_n = payload.repeat_last_n, dynatemp_range = payload.dynatemp_range, @@ -2778,6 +2788,16 @@ async def openai_chat_completions( min_keep = payload.min_keep, ignore_eos = payload.ignore_eos, min_tokens = payload.min_tokens, + skip_special_tokens = payload.skip_special_tokens, + spaces_between_special_tokens = payload.spaces_between_special_tokens, + include_stop_str_in_output = payload.include_stop_str_in_output, + truncate_prompt_tokens = payload.truncate_prompt_tokens, + n_keep = payload.n_keep, + n_probs = payload.n_probs, + cache_prompt = payload.cache_prompt, + return_tokens = payload.return_tokens, + timings_per_token = payload.timings_per_token, + post_sampling_probs = payload.post_sampling_probs, top_n_sigma = payload.top_n_sigma, repeat_last_n = payload.repeat_last_n, dynatemp_range = payload.dynatemp_range, @@ -5075,6 +5095,16 @@ def _build_passthrough_payload( min_keep = None, ignore_eos = None, min_tokens = None, + skip_special_tokens = None, + spaces_between_special_tokens = None, + include_stop_str_in_output = None, + truncate_prompt_tokens = None, + n_keep = None, + n_probs = None, + cache_prompt = None, + return_tokens = None, + timings_per_token = None, + post_sampling_probs = None, tool_choice = "auto", response_format = None, chat_template_kwargs = None, @@ -5169,6 +5199,30 @@ def _build_passthrough_payload( body["ignore_eos"] = ignore_eos if min_tokens is not None: body["min_tokens"] = min_tokens + # vLLM output-shape knobs + llama.cpp context / KV / instrumentation + # knobs. Per-backend capability gating on the frontend prevents these + # from being forwarded to wires that don't recognise them; here we + # only enforce the `is not None` rule so explicit defaults still pass. + if skip_special_tokens is not None: + body["skip_special_tokens"] = skip_special_tokens + if spaces_between_special_tokens is not None: + body["spaces_between_special_tokens"] = spaces_between_special_tokens + if include_stop_str_in_output is not None: + body["include_stop_str_in_output"] = include_stop_str_in_output + if truncate_prompt_tokens is not None: + body["truncate_prompt_tokens"] = truncate_prompt_tokens + if n_keep is not None: + body["n_keep"] = n_keep + if n_probs is not None: + body["n_probs"] = n_probs + if cache_prompt is not None: + body["cache_prompt"] = cache_prompt + if return_tokens is not None: + body["return_tokens"] = return_tokens + if timings_per_token is not None: + body["timings_per_token"] = timings_per_token + if post_sampling_probs is not None: + body["post_sampling_probs"] = post_sampling_probs if response_format is not None: # llama-server applies a GBNF grammar derived from the JSON schema # when response_format is present. Field is documented flat at the diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index 32937ed1d7..50ccff3191 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -1118,6 +1118,73 @@ def test_local_passthrough_forwards_dry_xtc_min_keep_eos_min_tokens(): assert key not in body2, body2 +def test_local_passthrough_forwards_vllm_output_and_llama_cpp_instrumentation(): + """Round-trip the 10 extra knobs added in round 4: + skip_special_tokens / spaces_between_special_tokens / + include_stop_str_in_output / truncate_prompt_tokens (vLLM + SamplingParams) + n_keep / n_probs / cache_prompt / return_tokens / + timings_per_token / post_sampling_probs (llama-server README). + Each is gated `is not None` so explicit upstream-default values + (skip_special_tokens=True, cache_prompt=True, etc) still reach the + wire when the caller opted in. + """ + from routes import inference as route_mod + + body = route_mod._build_passthrough_payload( + openai_messages = [{"role": "user", "content": "hi"}], + openai_tools = None, + temperature = 0.6, + top_p = 0.95, + top_k = 20, + max_tokens = 64, + stream = True, + skip_special_tokens = False, + spaces_between_special_tokens = False, + include_stop_str_in_output = True, + truncate_prompt_tokens = 4096, + n_keep = -1, + n_probs = 5, + cache_prompt = False, + return_tokens = True, + timings_per_token = True, + post_sampling_probs = True, + ) + assert body.get("skip_special_tokens") is False + assert body.get("spaces_between_special_tokens") is False + assert body.get("include_stop_str_in_output") is True + assert body.get("truncate_prompt_tokens") == 4096 + assert body.get("n_keep") == -1 + assert body.get("n_probs") == 5 + assert body.get("cache_prompt") is False + assert body.get("return_tokens") is True + assert body.get("timings_per_token") is True + assert body.get("post_sampling_probs") is True + + # Unset = absent from body so each backend applies its own default. + body2 = route_mod._build_passthrough_payload( + openai_messages = [{"role": "user", "content": "hi"}], + openai_tools = None, + temperature = 0.6, + top_p = 0.95, + top_k = 20, + max_tokens = 64, + stream = True, + ) + for key in ( + "skip_special_tokens", + "spaces_between_special_tokens", + "include_stop_str_in_output", + "truncate_prompt_tokens", + "n_keep", + "n_probs", + "cache_prompt", + "return_tokens", + "timings_per_token", + "post_sampling_probs", + ): + assert key not in body2, body2 + + def test_local_passthrough_forwards_typical_p_when_set(): """`typical_p` is a llama.cpp-specific sampler (`typ_p` in the sampler chain). The local-llama-cpp passthrough payload builder must diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index 5351673a51..75dcea59be 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -1859,6 +1859,55 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { params.minTokens > 0 ? { min_tokens: params.minTokens } : {}), + // vLLM output-shape knobs. Upstream defaults: + // skip_special_tokens=true, spaces_between_special_tokens=true, + // include_stop_str_in_output=false. Forward only when user + // opted away from the default to avoid no-op wire bloat. + ...(externalCapabilities?.skipSpecialTokens && + params.skipSpecialTokens === false + ? { skip_special_tokens: false } + : {}), + ...(externalCapabilities?.spacesBetweenSpecialTokens && + params.spacesBetweenSpecialTokens === false + ? { spaces_between_special_tokens: false } + : {}), + ...(externalCapabilities?.includeStopStrInOutput && + params.includeStopStrInOutput === true + ? { include_stop_str_in_output: true } + : {}), + ...(externalCapabilities?.truncatePromptTokens && + params.truncatePromptTokens !== null && + params.truncatePromptTokens > 0 + ? { truncate_prompt_tokens: params.truncatePromptTokens } + : {}), + // llama.cpp-only context / KV-cache / instrumentation knobs. + // n_keep accepts -1 (= keep all) so the gate is != 0. + ...(externalCapabilities?.nKeep && + params.nKeep !== null && + params.nKeep !== 0 + ? { n_keep: params.nKeep } + : {}), + ...(externalCapabilities?.nProbs && + params.nProbs !== null && + params.nProbs > 0 + ? { n_probs: params.nProbs } + : {}), + ...(externalCapabilities?.cachePrompt && + params.cachePrompt === false + ? { cache_prompt: false } + : {}), + ...(externalCapabilities?.returnTokens && + params.returnTokens === true + ? { return_tokens: true } + : {}), + ...(externalCapabilities?.timingsPerToken && + params.timingsPerToken === true + ? { timings_per_token: true } + : {}), + ...(externalCapabilities?.postSamplingProbs && + params.postSamplingProbs === true + ? { post_sampling_probs: true } + : {}), // Built-in tools: Search pill maps to provider-side // web_search (currently OpenAI / Anthropic / OpenRouter / // Kimi); Code pill maps to Anthropic's server-side @@ -2045,6 +2094,36 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { ...(params.minTokens !== null && params.minTokens > 0 ? { min_tokens: params.minTokens } : {}), + // Local llama-server / vLLM / Ollama route. Per-backend + // capability gating handles the silent-drop story; here we + // forward only when the value diverges from upstream default. + ...(params.skipSpecialTokens === false + ? { skip_special_tokens: false } + : {}), + ...(params.spacesBetweenSpecialTokens === false + ? { spaces_between_special_tokens: false } + : {}), + ...(params.includeStopStrInOutput === true + ? { include_stop_str_in_output: true } + : {}), + ...(params.truncatePromptTokens !== null && + params.truncatePromptTokens > 0 + ? { truncate_prompt_tokens: params.truncatePromptTokens } + : {}), + ...(params.nKeep !== null && params.nKeep !== 0 + ? { n_keep: params.nKeep } + : {}), + ...(params.nProbs !== null && params.nProbs > 0 + ? { n_probs: params.nProbs } + : {}), + ...(params.cachePrompt === false ? { cache_prompt: false } : {}), + ...(params.returnTokens === true ? { return_tokens: true } : {}), + ...(params.timingsPerToken === true + ? { timings_per_token: true } + : {}), + ...(params.postSamplingProbs === true + ? { post_sampling_probs: true } + : {}), parallel_tool_calls: params.parallelToolCalls, image_base64: imageBase64, audio_base64: audioBase64, diff --git a/studio/frontend/src/features/chat/provider-capabilities.ts b/studio/frontend/src/features/chat/provider-capabilities.ts index 750f3c938e..0d297550af 100644 --- a/studio/frontend/src/features/chat/provider-capabilities.ts +++ b/studio/frontend/src/features/chat/provider-capabilities.ts @@ -139,6 +139,26 @@ export interface ProviderCapabilities { * llama.cpp + vLLM accept this; Ollama's OAI translator drops it. */ minTokens: boolean; + /** vLLM `skip_special_tokens` (vLLM SamplingParams). vLLM only. */ + skipSpecialTokens: boolean; + /** vLLM `spaces_between_special_tokens`. vLLM only. */ + spacesBetweenSpecialTokens: boolean; + /** vLLM `include_stop_str_in_output`. vLLM only — useful for agentic tools. */ + includeStopStrInOutput: boolean; + /** vLLM `truncate_prompt_tokens` — left-truncate the prompt. vLLM only. */ + truncatePromptTokens: boolean; + /** llama.cpp `n_keep` — tokens to retain on context overflow. llama.cpp only. */ + nKeep: boolean; + /** llama.cpp `n_probs` — return top-N token probabilities. llama.cpp only. */ + nProbs: boolean; + /** llama.cpp `cache_prompt` — KV-cache reuse. llama.cpp only. */ + cachePrompt: boolean; + /** llama.cpp `return_tokens` — debug. llama.cpp only. */ + returnTokens: boolean; + /** llama.cpp `timings_per_token` — performance debug. llama.cpp only. */ + timingsPerToken: boolean; + /** llama.cpp `post_sampling_probs` — sampling-chain debug. llama.cpp only. */ + postSamplingProbs: boolean; } /** @@ -595,6 +615,16 @@ const OPENAI_COMPAT_BASE: ProviderCapabilities = { minKeep: false, ignoreEos: false, minTokens: false, + skipSpecialTokens: false, + spacesBetweenSpecialTokens: false, + includeStopStrInOutput: false, + truncatePromptTokens: false, + nKeep: false, + nProbs: false, + cachePrompt: false, + returnTokens: false, + timingsPerToken: false, + postSamplingProbs: false, }; // Unsloth's first-party llama-server runtime (provider type `llama_cpp`) @@ -632,6 +662,18 @@ const LLAMA_CPP_CAPABILITIES: ProviderCapabilities = { minKeep: true, ignoreEos: true, minTokens: true, + // vLLM-only output-shape knobs — llama-server does not document them. + skipSpecialTokens: false, + spacesBetweenSpecialTokens: false, + includeStopStrInOutput: false, + truncatePromptTokens: false, + // llama.cpp-only context / KV-cache / instrumentation knobs. + nKeep: true, + nProbs: true, + cachePrompt: true, + returnTokens: true, + timingsPerToken: true, + postSamplingProbs: true, }; // vLLM's OpenAI-compat endpoint accepts the OpenAI subset plus top_k / @@ -658,6 +700,18 @@ const VLLM_CAPABILITIES: ProviderCapabilities = { xtcProbability: false, xtcThreshold: false, minKeep: false, + // vLLM-only output-shape knobs — flip the LLAMA_CPP defaults. + skipSpecialTokens: true, + spacesBetweenSpecialTokens: true, + includeStopStrInOutput: true, + truncatePromptTokens: true, + // llama.cpp-only instrumentation knobs — vLLM has no analog. + nKeep: false, + nProbs: false, + cachePrompt: false, + returnTokens: false, + timingsPerToken: false, + postSamplingProbs: false, }; // Ollama is stricter than vLLM. Studio reaches Ollama via the OpenAI- @@ -677,6 +731,12 @@ const OLLAMA_CAPABILITIES: ProviderCapabilities = { // on the /v1/chat/completions path Studio uses. ignoreEos: false, minTokens: false, + // The vLLM-specific output-shape knobs are not recognised by the + // Ollama OAI translator; flip them back to false. + skipSpecialTokens: false, + spacesBetweenSpecialTokens: false, + includeStopStrInOutput: false, + truncatePromptTokens: false, }; // OpenRouter is a router-of-routers: the gateway accepts a wider set @@ -716,6 +776,16 @@ const OPENROUTER_CAPABILITIES: ProviderCapabilities = { minKeep: false, ignoreEos: false, minTokens: false, + skipSpecialTokens: false, + spacesBetweenSpecialTokens: false, + includeStopStrInOutput: false, + truncatePromptTokens: false, + nKeep: false, + nProbs: false, + cachePrompt: false, + returnTokens: false, + timingsPerToken: false, + postSamplingProbs: false, }; // Reasoning-class OpenAI models served via /v1/responses fix temperature @@ -756,6 +826,16 @@ const OPENAI_REASONING_CAPABILITIES: ProviderCapabilities = { minKeep: false, ignoreEos: false, minTokens: false, + skipSpecialTokens: false, + spacesBetweenSpecialTokens: false, + includeStopStrInOutput: false, + truncatePromptTokens: false, + nKeep: false, + nProbs: false, + cachePrompt: false, + returnTokens: false, + timingsPerToken: false, + postSamplingProbs: false, }; const OPENAI_CHAT_CAPABILITIES: ProviderCapabilities = { temperature: true, @@ -789,6 +869,16 @@ const OPENAI_CHAT_CAPABILITIES: ProviderCapabilities = { minKeep: false, ignoreEos: false, minTokens: false, + skipSpecialTokens: false, + spacesBetweenSpecialTokens: false, + includeStopStrInOutput: false, + truncatePromptTokens: false, + nKeep: false, + nProbs: false, + cachePrompt: false, + returnTokens: false, + timingsPerToken: false, + postSamplingProbs: false, }; // Prefix list for OpenAI reasoning-class model ids. Kept in sync with @@ -894,6 +984,16 @@ const PROVIDER_CAPABILITIES: Record = { minKeep: false, ignoreEos: false, minTokens: false, + skipSpecialTokens: false, + spacesBetweenSpecialTokens: false, + includeStopStrInOutput: false, + truncatePromptTokens: false, + nKeep: false, + nProbs: false, + cachePrompt: false, + returnTokens: false, + timingsPerToken: false, + postSamplingProbs: false, }, mistral: OPENAI_COMPAT_BASE, gemini: OPENAI_COMPAT_BASE, @@ -939,6 +1039,16 @@ const PROVIDER_CAPABILITIES: Record = { minKeep: false, ignoreEos: false, minTokens: false, + skipSpecialTokens: false, + spacesBetweenSpecialTokens: false, + includeStopStrInOutput: false, + truncatePromptTokens: false, + nKeep: false, + nProbs: false, + cachePrompt: false, + returnTokens: false, + timingsPerToken: false, + postSamplingProbs: false, }, // DeepSeek deprecated presence/frequency penalty and never published // `seed` or `parallel_tool_calls` in the current chat-completion @@ -982,6 +1092,16 @@ const PROVIDER_CAPABILITIES: Record = { minKeep: false, ignoreEos: false, minTokens: false, + skipSpecialTokens: false, + spacesBetweenSpecialTokens: false, + includeStopStrInOutput: false, + truncatePromptTokens: false, + nKeep: false, + nProbs: false, + cachePrompt: false, + returnTokens: false, + timingsPerToken: false, + postSamplingProbs: false, }, qwen: OPENAI_COMPAT_BASE, huggingface: OPENAI_COMPAT_BASE, diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index e8be7d52a1..a013460637 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -386,6 +386,26 @@ export interface OpenAIChatCompletionsRequest { * 0 disables. */ min_tokens?: number; + /** vLLM `skip_special_tokens` — default true; forward only when false. */ + skip_special_tokens?: boolean; + /** vLLM `spaces_between_special_tokens` — default true; forward only when false. */ + spaces_between_special_tokens?: boolean; + /** vLLM `include_stop_str_in_output` — default false; forward only when true. */ + include_stop_str_in_output?: boolean; + /** vLLM `truncate_prompt_tokens` — left-truncate the prompt. > 0 only. */ + truncate_prompt_tokens?: number; + /** llama.cpp `n_keep` — tokens to retain on context overflow. -1 = all. */ + n_keep?: number; + /** llama.cpp `n_probs` — return top-N token probabilities. > 0 only. */ + n_probs?: number; + /** llama.cpp `cache_prompt` — KV-cache reuse. Default true upstream; forward only when false. */ + cache_prompt?: boolean; + /** llama.cpp `return_tokens` — include raw token IDs in response. Default false. */ + return_tokens?: boolean; + /** llama.cpp `timings_per_token` — include per-token speed metrics. Default false. */ + timings_per_token?: boolean; + /** llama.cpp `post_sampling_probs` — token probs after the sampler chain. Default false. */ + post_sampling_probs?: boolean; } export interface OpenAIChatDelta { diff --git a/studio/frontend/src/features/chat/types/runtime.ts b/studio/frontend/src/features/chat/types/runtime.ts index 99beb9ccc4..f0d7b5a59e 100644 --- a/studio/frontend/src/features/chat/types/runtime.ts +++ b/studio/frontend/src/features/chat/types/runtime.ts @@ -97,6 +97,58 @@ export interface InferenceParams { * vLLM + llama.cpp accept this. 0 disables. `null` = unset. */ minTokens: number | null; + /** + * vLLM `skip_special_tokens`. Default true. Forward only when false + * (i.e. user wants to see raw special tokens in the output). + * https://docs.vllm.ai/en/latest/api/vllm/sampling_params/ + */ + skipSpecialTokens: boolean | null; + /** + * vLLM `spaces_between_special_tokens`. Default true. Forward only + * when false. + */ + spacesBetweenSpecialTokens: boolean | null; + /** + * vLLM `include_stop_str_in_output`. Default false. Useful for + * agentic tools that need the matched stop string echoed back. + */ + includeStopStrInOutput: boolean | null; + /** + * vLLM `truncate_prompt_tokens` — left-truncate the prompt to this + * many tokens. Useful for long-context overflow. `null` = unset. + */ + truncatePromptTokens: number | null; + /** + * llama.cpp `n_keep` — tokens to retain when context overflows. + * 0 disables, -1 keeps all. `null` = unset. + */ + nKeep: number | null; + /** + * llama.cpp `n_probs` — return top-N token probabilities per + * generated token. 0 disables. `null` = unset. + */ + nProbs: number | null; + /** + * llama.cpp `cache_prompt` — reuse KV cache from previous prompts + * with a shared prefix. Default true upstream. Forward only when + * explicitly false (e.g. for deterministic benchmarks). + */ + cachePrompt: boolean | null; + /** + * llama.cpp `return_tokens` — include raw token IDs in the response. + * Debug. Default false. + */ + returnTokens: boolean | null; + /** + * llama.cpp `timings_per_token` — include per-token speed metrics. + * Default false. + */ + timingsPerToken: boolean | null; + /** + * llama.cpp `post_sampling_probs` — return token probabilities AFTER + * the sampler chain runs. Debug. Default false. + */ + postSamplingProbs: boolean | null; maxSeqLength: number; maxTokens: number; systemPrompt: string; @@ -141,6 +193,16 @@ export const DEFAULT_INFERENCE_PARAMS: InferenceParams = { minKeep: null, ignoreEos: null, minTokens: null, + skipSpecialTokens: null, + spacesBetweenSpecialTokens: null, + includeStopStrInOutput: null, + truncatePromptTokens: null, + nKeep: null, + nProbs: null, + cachePrompt: null, + returnTokens: null, + timingsPerToken: null, + postSamplingProbs: null, maxSeqLength: 4096, maxTokens: 8192, systemPrompt: "", diff --git a/studio/frontend/src/features/chat/utils/chat-settings-storage.ts b/studio/frontend/src/features/chat/utils/chat-settings-storage.ts index 3b9a5a1f21..7c31d2caa7 100644 --- a/studio/frontend/src/features/chat/utils/chat-settings-storage.ts +++ b/studio/frontend/src/features/chat/utils/chat-settings-storage.ts @@ -211,6 +211,9 @@ function sanitizeInferenceParams( "xtcThreshold", "minKeep", "minTokens", + "truncatePromptTokens", + "nKeep", + "nProbs", ] as const) { const raw = value[key]; if (raw === null) { @@ -219,11 +222,24 @@ function sanitizeInferenceParams( (params as Record)[key] = raw; } } - // ignoreEos is the only nullable BOOLEAN in the new batch. - if (value.ignoreEos === null) { - params.ignoreEos = null; - } else if (typeof value.ignoreEos === "boolean") { - params.ignoreEos = value.ignoreEos; + // Nullable booleans (ignoreEos, skip/spaces special-tokens, include-stop, + // cache_prompt, return_tokens, timings_per_token, post_sampling_probs). + for (const key of [ + "ignoreEos", + "skipSpecialTokens", + "spacesBetweenSpecialTokens", + "includeStopStrInOutput", + "cachePrompt", + "returnTokens", + "timingsPerToken", + "postSamplingProbs", + ] as const) { + const raw = value[key]; + if (raw === null) { + (params as Record)[key] = null; + } else if (typeof raw === "boolean") { + (params as Record)[key] = raw; + } } // Mirror trustRemoteCode handling so the toggle survives reload // and the /api/chat/settings round-trip.