diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index bdb56c33ab..c71ef9ee1c 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -904,6 +904,34 @@ class ExternalProviderClient: tools: Optional[list[dict[str, Any]]] = None, tool_choice: Optional[Any] = None, fast_mode: Optional[bool] = None, + typical_p: Optional[float] = None, + top_n_sigma: Optional[float] = None, + repeat_last_n: Optional[int] = None, + dynatemp_range: Optional[float] = None, + dynatemp_exponent: Optional[float] = None, + mirostat: Optional[int] = None, + mirostat_tau: Optional[float] = None, + mirostat_eta: Optional[float] = None, + top_a: Optional[float] = None, + dry_multiplier: Optional[float] = None, + dry_base: Optional[float] = None, + dry_allowed_length: Optional[int] = None, + dry_penalty_last_n: Optional[int] = None, + xtc_probability: Optional[float] = None, + xtc_threshold: Optional[float] = None, + min_keep: Optional[int] = None, + ignore_eos: Optional[bool] = None, + min_tokens: Optional[int] = None, + skip_special_tokens: Optional[bool] = None, + spaces_between_special_tokens: Optional[bool] = None, + include_stop_str_in_output: Optional[bool] = None, + truncate_prompt_tokens: Optional[int] = None, + n_keep: Optional[int] = None, + n_probs: Optional[int] = None, + cache_prompt: Optional[bool] = None, + return_tokens: Optional[bool] = None, + timings_per_token: Optional[bool] = None, + post_sampling_probs: Optional[bool] = None, stream: bool = True, ) -> AsyncGenerator[str, None]: """ @@ -1067,10 +1095,76 @@ class ExternalProviderClient: if parallel_tool_calls is not None: body["parallel_tool_calls"] = parallel_tool_calls + # Extended OAI-compat samplers (OpenRouter `top_a`, vLLM output + # knobs, llama.cpp samplers on custom proxies). Each is gated `is + # not None` so explicit 0/False reach the wire; `body_omit` below + # strips fields the upstream rejects. + if typical_p is not None: + body["typical_p"] = typical_p + if top_n_sigma is not None: + body["top_n_sigma"] = top_n_sigma + if repeat_last_n is not None: + body["repeat_last_n"] = repeat_last_n + if dynatemp_range is not None: + body["dynatemp_range"] = dynatemp_range + if dynatemp_exponent is not None: + body["dynatemp_exponent"] = dynatemp_exponent + if mirostat is not None: + body["mirostat"] = mirostat + if mirostat_tau is not None: + body["mirostat_tau"] = mirostat_tau + if mirostat_eta is not None: + body["mirostat_eta"] = mirostat_eta + if top_a is not None: + body["top_a"] = top_a + if dry_multiplier is not None: + body["dry_multiplier"] = dry_multiplier + if dry_base is not None: + body["dry_base"] = dry_base + if dry_allowed_length is not None: + body["dry_allowed_length"] = dry_allowed_length + if dry_penalty_last_n is not None: + body["dry_penalty_last_n"] = dry_penalty_last_n + if xtc_probability is not None: + body["xtc_probability"] = xtc_probability + if xtc_threshold is not None: + body["xtc_threshold"] = xtc_threshold + if min_keep is not None: + body["min_keep"] = min_keep + if ignore_eos is not None: + body["ignore_eos"] = ignore_eos + if min_tokens is not None: + body["min_tokens"] = min_tokens + if skip_special_tokens is not None: + body["skip_special_tokens"] = skip_special_tokens + if spaces_between_special_tokens is not None: + body["spaces_between_special_tokens"] = spaces_between_special_tokens + if include_stop_str_in_output is not None: + body["include_stop_str_in_output"] = include_stop_str_in_output + if truncate_prompt_tokens is not None: + body["truncate_prompt_tokens"] = truncate_prompt_tokens + if n_keep is not None: + body["n_keep"] = n_keep + if n_probs is not None: + body["n_probs"] = n_probs + if cache_prompt is not None: + body["cache_prompt"] = cache_prompt + if return_tokens is not None: + body["return_tokens"] = return_tokens + if timings_per_token is not None: + body["timings_per_token"] = timings_per_token + if post_sampling_probs is not None: + body["post_sampling_probs"] = post_sampling_probs + # Drop body fields the provider's registry entry locks down # (e.g. Kimi k2.5/k2.6 only accept temperature=1, top_p=1). + # Also pop the renamed seed field so `body_omit=("seed",)` on a + # provider with `seed_field` rename still strips correctly. + _seed_field = provider_info.get("seed_field", "seed") for field in provider_info.get("body_omit", ()): body.pop(field, None) + if field == "seed" and _seed_field != "seed": + body.pop(_seed_field, None) # Kimi thinking is a top-level body field. kimi-k2-thinking is # always on (ignore the toggle); kimi-k2.6 defaults on, can be diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index ed0047d8da..46ad4cee4d 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -192,6 +192,9 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { # Mistral renames OpenAI's `seed` to `random_seed` on # /v1/chat/completions. https://docs.mistral.ai/api/endpoint/chat "seed_field": "random_seed", + # Mistral's docs publish no max but third-party shims cap at 4; + # match OpenAI Chat's cap to avoid silent upstream truncation. + "stop_max": 4, }, "kimi": { "display_name": "Kimi", @@ -216,12 +219,13 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { "notes": "Moonshot API key. China: use base URL https://api.moonshot.cn/v1", "model_id_allowlist": re.compile(r"^kimi-k2\.[56]$"), # k2.5/k2.6 are reasoning-class: API locks temperature=1, top_p=1, - # frequency_penalty; seed and parallel_tool_calls are undocumented - # and 400. + # frequency_penalty; presence_penalty / seed / parallel_tool_calls + # are undocumented in the Kimi chat schema. "body_omit": ( "temperature", "top_p", "frequency_penalty", + "presence_penalty", "seed", "parallel_tool_calls", ), diff --git a/studio/backend/routes/chat_history.py b/studio/backend/routes/chat_history.py index 800873afba..4e4faeb1ba 100644 --- a/studio/backend/routes/chat_history.py +++ b/studio/backend/routes/chat_history.py @@ -99,6 +99,9 @@ class ChatExportResponse(BaseModel): class ChatInferenceSettings(BaseModel): + # extra="forbid" requires every persisted key to be listed. Keep + # aligned with PERSISTED_INFERENCE_PARAM_KEYS in + # studio/frontend/src/features/chat/stores/chat-runtime-store.ts. model_config = ConfigDict(extra = "forbid") temperature: Optional[float] = None @@ -107,10 +110,6 @@ class ChatInferenceSettings(BaseModel): minP: Optional[float] = None repetitionPenalty: Optional[float] = None presencePenalty: Optional[float] = None - # New per-provider sampling knobs. extra="forbid" requires these - # to be listed; otherwise every save from the new frontend 422s. - # Keep aligned with InferenceParams in - # studio/frontend/src/features/chat/types/runtime.ts. frequencyPenalty: Optional[float] = Field(default = None, ge = -2.0, le = 2.0) seed: Optional[int] = None stop: Optional[list[str]] = None @@ -122,6 +121,36 @@ class ChatInferenceSettings(BaseModel): maxTokens: Optional[float] = None systemPrompt: Optional[str] = None trustRemoteCode: Optional[bool] = None + fastMode: Optional[bool] = None + # Extended llama.cpp / vLLM / OpenRouter samplers exposed by PR #5711. + typicalP: Optional[float] = None + topNSigma: Optional[float] = None + repeatLastN: Optional[int] = None + dynatempRange: Optional[float] = None + dynatempExponent: Optional[float] = None + mirostat: Optional[int] = None + mirostatTau: Optional[float] = None + mirostatEta: Optional[float] = None + topA: Optional[float] = None + dryMultiplier: Optional[float] = None + dryBase: Optional[float] = None + dryAllowedLength: Optional[int] = None + dryPenaltyLastN: Optional[int] = None + xtcProbability: Optional[float] = None + xtcThreshold: Optional[float] = None + minKeep: Optional[int] = None + ignoreEos: Optional[bool] = None + minTokens: Optional[int] = None + skipSpecialTokens: Optional[bool] = None + spacesBetweenSpecialTokens: Optional[bool] = None + includeStopStrInOutput: Optional[bool] = None + truncatePromptTokens: Optional[int] = None + nKeep: Optional[int] = None + nProbs: Optional[int] = None + cachePrompt: Optional[bool] = None + returnTokens: Optional[bool] = None + timingsPerToken: Optional[bool] = None + postSamplingProbs: Optional[bool] = None class ChatPreset(BaseModel): diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index cc0e236b49..481c9239d7 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -2168,6 +2168,34 @@ async def _proxy_to_external_provider( tools = payload.tools, tool_choice = payload.tool_choice, fast_mode = payload.fast_mode, + typical_p = payload.typical_p, + top_n_sigma = payload.top_n_sigma, + repeat_last_n = payload.repeat_last_n, + dynatemp_range = payload.dynatemp_range, + dynatemp_exponent = payload.dynatemp_exponent, + mirostat = payload.mirostat, + mirostat_tau = payload.mirostat_tau, + mirostat_eta = payload.mirostat_eta, + top_a = payload.top_a, + dry_multiplier = payload.dry_multiplier, + dry_base = payload.dry_base, + dry_allowed_length = payload.dry_allowed_length, + dry_penalty_last_n = payload.dry_penalty_last_n, + xtc_probability = payload.xtc_probability, + xtc_threshold = payload.xtc_threshold, + min_keep = payload.min_keep, + ignore_eos = payload.ignore_eos, + min_tokens = payload.min_tokens, + skip_special_tokens = payload.skip_special_tokens, + spaces_between_special_tokens = payload.spaces_between_special_tokens, + include_stop_str_in_output = payload.include_stop_str_in_output, + truncate_prompt_tokens = payload.truncate_prompt_tokens, + n_keep = payload.n_keep, + n_probs = payload.n_probs, + cache_prompt = payload.cache_prompt, + return_tokens = payload.return_tokens, + timings_per_token = payload.timings_per_token, + post_sampling_probs = payload.post_sampling_probs, stream = payload.stream, ) try: @@ -6009,6 +6037,25 @@ def _build_openai_passthrough_body(payload, backend_ctx = None) -> dict: mirostat = payload.mirostat, mirostat_tau = payload.mirostat_tau, mirostat_eta = payload.mirostat_eta, + dry_multiplier = payload.dry_multiplier, + dry_base = payload.dry_base, + dry_allowed_length = payload.dry_allowed_length, + dry_penalty_last_n = payload.dry_penalty_last_n, + xtc_probability = payload.xtc_probability, + xtc_threshold = payload.xtc_threshold, + min_keep = payload.min_keep, + ignore_eos = payload.ignore_eos, + min_tokens = payload.min_tokens, + skip_special_tokens = payload.skip_special_tokens, + spaces_between_special_tokens = payload.spaces_between_special_tokens, + include_stop_str_in_output = payload.include_stop_str_in_output, + truncate_prompt_tokens = payload.truncate_prompt_tokens, + n_keep = payload.n_keep, + n_probs = payload.n_probs, + cache_prompt = payload.cache_prompt, + return_tokens = payload.return_tokens, + timings_per_token = payload.timings_per_token, + post_sampling_probs = payload.post_sampling_probs, tool_choice = tool_choice, response_format = _extract_response_format(payload), chat_template_kwargs = tpl_kwargs, diff --git a/studio/backend/tests/test_sampling_params_routing.py b/studio/backend/tests/test_sampling_params_routing.py index 72e7eab412..c3115bf065 100644 --- a/studio/backend/tests/test_sampling_params_routing.py +++ b/studio/backend/tests/test_sampling_params_routing.py @@ -239,22 +239,20 @@ def test_anthropic_rejects_openai_only_knobs(monkeypatch): def _drive_openai_compat(captured, **kwargs) -> dict: """Send through the default OAI-compat branch (NOT /v1/responses). - Use a non-OpenAI provider_type so the dispatcher takes the default - branch at the bottom of stream_chat_completion rather than the - Responses translator path that routes provider_type=="openai". + Use qwen so the dispatcher takes the default branch and the provider + inherits the default 16-stop cap (Mistral now caps at 4 per its + third-party shims). """ async def run(): client = ExternalProviderClient( - provider_type = "mistral", - base_url = "https://api.mistral.ai/v1", + provider_type = "qwen", + base_url = "https://dashscope-intl.aliyuncs.com/compatible-mode/v1", api_key = "test-key", ) - # mistral's OpenAI-compat /v1/chat/completions returns OpenAI - # SSE; a single DONE frame is enough to drain the stream. async for _ in client.stream_chat_completion( messages = [{"role": "user", "content": "hi"}], - model = "mistral-small-latest", + model = "qwen-plus", temperature = 0.5, top_p = 0.9, max_tokens = 64, @@ -278,10 +276,36 @@ def test_openai_compat_forwards_frequency_penalty(monkeypatch): def test_openai_compat_forwards_seed(monkeypatch): + """qwen has no seed_field override so seed forwards as `seed`.""" captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) body = _drive_openai_compat(captured, seed = 12345) - # Default OAI-compat provider (mistral here) renames seed to - # random_seed via provider registry's seed_field. + assert body.get("seed") == 12345, body + + +def test_mistral_renames_seed_to_random_seed(monkeypatch): + """Mistral's registry sets seed_field="random_seed" so the OAI seed + is renamed on the wire. https://docs.mistral.ai/api/endpoint/chat""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "mistral", + base_url = "https://api.mistral.ai/v1", + api_key = "mistral-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "mistral-small-latest", + temperature = 0.5, + top_p = 0.9, + max_tokens = 64, + seed = 12345, + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] assert body.get("random_seed") == 12345, body assert "seed" not in body, body @@ -349,8 +373,8 @@ def test_openai_compat_forwards_stop_array(monkeypatch): def test_openai_compat_truncates_stop_to_default_cap(monkeypatch): - """Default OAI-compat cap is 16 (DeepSeek and Mistral both accept - that many); only OpenAI Chat has a tighter 4-entry hard limit.""" + """Default OAI-compat cap is 16 (Qwen, DeepSeek, HuggingFace, custom); + OpenAI Chat / OpenRouter / Gemini / Mistral have tighter 4-entry caps.""" captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) body = _drive_openai_compat(captured, stop = [f"s{i}" for i in range(20)]) assert len(body.get("stop", [])) == 16, body @@ -358,6 +382,33 @@ def test_openai_compat_truncates_stop_to_default_cap(monkeypatch): assert body["stop"][-1] == "s15" +def test_mistral_stop_cap_is_4(monkeypatch): + """Mistral's docs publish no max but third-party shims cap at 4; + match OpenAI Chat's cap to avoid silent upstream truncation.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "mistral", + base_url = "https://api.mistral.ai/v1", + api_key = "mistral-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "mistral-small-latest", + temperature = 0.5, + top_p = 0.9, + max_tokens = 64, + stop = [f"S{i}" for i in range(8)], + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert body.get("stop") == ["S0", "S1", "S2", "S3"], body + + def test_openai_compat_stop_dedup_and_drop_empties(monkeypatch): """Duplicates and empties shouldn't eat into the cap.""" captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) @@ -590,9 +641,9 @@ def test_kimi_web_search_bypass_forwards_new_sampling_fields(monkeypatch): assert "top_p" not in body, body assert "seed" not in body, body assert "parallel_tool_calls" not in body, body + assert "presence_penalty" not in body, body # Knobs not on Kimi's drop-list forward through the bypass. assert body.get("stop") == ["END"], body - assert body.get("presence_penalty") == 0.5, body def test_kimi_web_search_uses_kimi_stop_cap_5(monkeypatch): @@ -681,6 +732,72 @@ def test_gemini_stop_sequences_capped_to_5(monkeypatch): assert gen_config.get("stopSequences") == ["S0", "S1", "S2", "S3", "S4"], body +def test_openrouter_forwards_top_a(monkeypatch): + """OpenRouter exposes `top_a` (the tail-cut sampler). The frontend + capability map lets the value through; verify the OAI-compat body + builder actually carries it to the wire.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "openrouter", + base_url = "https://openrouter.ai/api/v1", + api_key = "or-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "openai/gpt-4o", + temperature = 0.5, + top_p = 0.9, + max_tokens = 64, + top_a = 0.25, + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert body.get("top_a") == 0.25, body + + +def test_vllm_forwards_output_shape_knobs(monkeypatch): + """vLLM accepts skip_special_tokens / spaces_between_special_tokens / + include_stop_str_in_output / truncate_prompt_tokens. Verify the + OAI-compat external proxy forwards them to the wire body.""" + captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload()) + + async def run(): + client = ExternalProviderClient( + provider_type = "vllm", + base_url = "https://vllm.example.com/v1", + api_key = "vllm-test", + ) + async for _ in client.stream_chat_completion( + messages = [{"role": "user", "content": "hi"}], + model = "Qwen/Qwen3-4B", + temperature = 0.5, + top_p = 0.9, + max_tokens = 64, + skip_special_tokens = False, + spaces_between_special_tokens = False, + include_stop_str_in_output = True, + truncate_prompt_tokens = 2048, + min_tokens = 8, + ignore_eos = True, + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert body.get("skip_special_tokens") is False, body + assert body.get("spaces_between_special_tokens") is False, body + assert body.get("include_stop_str_in_output") is True, body + assert body.get("truncate_prompt_tokens") == 2048, body + assert body.get("min_tokens") == 8, body + assert body.get("ignore_eos") is True, body + + def test_kimi_drops_stop_strings_over_32_bytes(monkeypatch): """Kimi limits each stop string to <= 32 bytes per https://platform.kimi.ai/docs/api/chat. Drop overlong entries @@ -837,27 +954,52 @@ def test_responses_to_chat_bridge_omits_unset_parallel_tool_calls(): def test_chat_settings_payload_accepts_new_sampling_keys(): """ChatSettingsPayload has extra="forbid" so the new keys must be listed explicitly; otherwise every settings save with any of them - 422s. Pin the round-trip.""" + 422s. Pin the round-trip for every persisted sampler the frontend + can emit (see PERSISTED_INFERENCE_PARAM_KEYS in chat-runtime-store.ts).""" from routes.chat_history import ChatSettingsPayload - parsed = ChatSettingsPayload.model_validate( - { - "inferenceParams": { - "frequencyPenalty": 0.7, - "seed": 42, - "stop": ["END"], - "serviceTier": "standard_only", - "parallelToolCalls": False, - } - } - ) + inference = { + "frequencyPenalty": 0.7, + "seed": 42, + "stop": ["END"], + "serviceTier": "standard_only", + "parallelToolCalls": False, + "fastMode": True, + # Extended llama.cpp / vLLM / OpenRouter samplers. + "typicalP": 0.85, + "topNSigma": 2.5, + "repeatLastN": 64, + "dynatempRange": 0.3, + "dynatempExponent": 1.2, + "mirostat": 2, + "mirostatTau": 4.0, + "mirostatEta": 0.15, + "topA": 0.2, + "dryMultiplier": 0.8, + "dryBase": 1.75, + "dryAllowedLength": 2, + "dryPenaltyLastN": -1, + "xtcProbability": 0.5, + "xtcThreshold": 0.1, + "minKeep": 5, + "ignoreEos": True, + "minTokens": 10, + "skipSpecialTokens": False, + "spacesBetweenSpecialTokens": False, + "includeStopStrInOutput": True, + "truncatePromptTokens": 1024, + "nKeep": -1, + "nProbs": 5, + "cachePrompt": False, + "returnTokens": True, + "timingsPerToken": True, + "postSamplingProbs": True, + } + parsed = ChatSettingsPayload.model_validate({"inferenceParams": inference}) ip = parsed.inferenceParams assert ip is not None - assert ip.frequencyPenalty == 0.7 - assert ip.seed == 42 - assert ip.stop == ["END"] - assert ip.serviceTier == "standard_only" - assert ip.parallelToolCalls is False + for key, expected in inference.items(): + assert getattr(ip, key) == expected, f"{key} did not round-trip" # ── Local /v1/messages: disable_parallel_tool_use translation ────────── @@ -1093,7 +1235,7 @@ def test_local_passthrough_forwards_dry_xtc_min_keep_eos_min_tokens(): assert body.get("min_tokens") == 16 # Unset = absent from body. Matches the upstream "use default" - # contract — llama-server / vLLM apply their own defaults instead. + # contract: llama-server / vLLM apply their own defaults instead. body2 = route_mod._build_passthrough_payload( openai_messages = [{"role": "user", "content": "hi"}], openai_tools = None, diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index 5ec6cf40a9..11ef3b4700 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -2335,7 +2335,12 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { ...(params.postSamplingProbs === true ? { post_sampling_probs: true } : {}), - parallel_tool_calls: params.parallelToolCalls, + // Forward only on explicit opt-out (default true on every + // backend; default omit keeps wire-shape stable for users + // who never opened the new settings panel). + ...(params.parallelToolCalls === false + ? { parallel_tool_calls: false } + : {}), image_base64: imageBase64, audio_base64: audioBase64, cancel_id: cancelId, diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index 2abc423ae5..ab6d68e3b2 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -133,21 +133,11 @@ export function InfoHint({ children }: { children: ReactNode }) { ); } -/** - * Editable numeric value display. - * - * Renders as a single that *looks* like text by default — - * transparent background, no border, no ring — and only shows a faint - * surface tint on hover/focus to signal editability. When unfocused, - * the input shows the formatted display string (`displayValue ?? value`, - * so labels like "Off" / "Max" still render); on focus, it switches to - * the raw numeric value, selects it, and accepts free text input. - * Commit happens on blur or Enter; Escape reverts. The clamp-to-range - * happens on commit so users can type intermediate values without the - * input fighting them mid-keystroke. Single component shared by every - * slider value and the Context Length input so the click-to-edit - * affordance is consistent across the panel. - */ +/** Editable numeric value display: transparent text-like input that + * shows formatted display on blur (so "Off"/"Max" labels render) and + * switches to the raw number on focus. Commits on blur/Enter, reverts + * on Escape, clamps on commit. Shared by every slider value + the + * Context Length input. */ function snapToStep( value: number, step: number, @@ -1431,7 +1421,19 @@ export function ChatSettingsPanel({