Fix consensus review findings on PR 5711
Round 3 of 3-Opus parallel review (2 reviewers HIGH on the persistence
chain, 2 HIGH on the routing chain, 1 HIGH on the test coverage gap).
HIGH fixes:
1. chat-runtime-store.ts: PERSISTED_INFERENCE_PARAM_KEYS extended from
16 to 44 keys (28 extended samplers added). Before this, any value
set on the Advanced Sampling sliders was lost on page reload because
getChangedInferenceParams / getHydratedSettingsState iterate this
list.
2. routes/chat_history.py: ChatInferenceSettings (extra="forbid")
extended to mirror InferenceParams including fastMode + 28 new
samplers. Without this every settings PUT containing any of those
fields would 422.
3. routes/inference.py _build_openai_passthrough_body: was forwarding
typical_p / mirostat / dynatemp but silently dropping dry_*, xtc_*,
min_keep, ignore_eos, min_tokens, vLLM output knobs (skip/spaces
special-tokens, include_stop_str_in_output, truncate_prompt_tokens),
and llama.cpp instrumentation flags (n_keep, n_probs, cache_prompt,
return_tokens, timings_per_token, post_sampling_probs). Now forwards
all 18 to _build_passthrough_payload.
4. routes/inference.py _proxy_to_external_provider + external_provider.py
stream_chat_completion: 20 extended kwargs are now plumbed through
the route -> client -> OAI-compat body builder. Before this fix the
chat-adapter computed top_a / vLLM output knobs / llama.cpp samplers
on the frontend, sent them on the wire, and the route layer dropped
them on the floor.
5. test_sampling_params_routing.py: extended
test_chat_settings_payload_accepts_new_sampling_keys to round-trip
every persisted field (was only 5). Added
test_openrouter_forwards_top_a and test_vllm_forwards_output_shape_knobs
to lock in the new wire forwarding.
MEDIUM fixes:
- providers.py: Mistral stop_max=4 (matches third-party shims; OAI docs
publish no max but every consumer caps at 4).
- providers.py: Kimi body_omit now includes "presence_penalty" (Kimi
k2.5/k2.6 chat schema lists temperature/top_p/max_tokens/stream/tools/
tool_choice/thinking but not presence_penalty).
- external_provider.py: body_omit loop also pops the seed_field
rename so a future provider with both `seed_field="random_seed"` and
`body_omit=("seed",)` strips correctly. No current provider has both;
defensive only.
- chat-adapter.ts: local-path parallel_tool_calls forwards only on
explicit opt-out (matches the external-path stanza). Before this the
field was sent on every chat from every existing local user.
- chat-settings-sheet.tsx: service tier Select now clamps the displayed
value to a legal option for the active provider (e.g. "priority"
saved on OpenAI, then user switches to Anthropic which only allows
auto/standard_only -> Radix Select was showing a blank trigger).
LOW fixes:
- Em-dash cleanup: 7 em-dashes removed from provider-capabilities.ts /
runtime.ts / chat-settings-sheet.tsx / test_sampling_params_routing.py
per project rules.
Tests: 397/397 backend pass (sampling routing 69 plus anthropic /
openai / gemini / llama-server suites). Frontend tsc + vite build clean.
This commit is contained in:
Daniel Han
parent
1add4dbd0e
commit
5ec1208206
10 changed files with 418 additions and 66 deletions
|
|
@ -904,6 +904,34 @@ class ExternalProviderClient:
|
|||
tools: Optional[list[dict[str, Any]]] = None,
|
||||
tool_choice: Optional[Any] = None,
|
||||
fast_mode: Optional[bool] = None,
|
||||
typical_p: Optional[float] = None,
|
||||
top_n_sigma: Optional[float] = None,
|
||||
repeat_last_n: Optional[int] = None,
|
||||
dynatemp_range: Optional[float] = None,
|
||||
dynatemp_exponent: Optional[float] = None,
|
||||
mirostat: Optional[int] = None,
|
||||
mirostat_tau: Optional[float] = None,
|
||||
mirostat_eta: Optional[float] = None,
|
||||
top_a: Optional[float] = None,
|
||||
dry_multiplier: Optional[float] = None,
|
||||
dry_base: Optional[float] = None,
|
||||
dry_allowed_length: Optional[int] = None,
|
||||
dry_penalty_last_n: Optional[int] = None,
|
||||
xtc_probability: Optional[float] = None,
|
||||
xtc_threshold: Optional[float] = None,
|
||||
min_keep: Optional[int] = None,
|
||||
ignore_eos: Optional[bool] = None,
|
||||
min_tokens: Optional[int] = None,
|
||||
skip_special_tokens: Optional[bool] = None,
|
||||
spaces_between_special_tokens: Optional[bool] = None,
|
||||
include_stop_str_in_output: Optional[bool] = None,
|
||||
truncate_prompt_tokens: Optional[int] = None,
|
||||
n_keep: Optional[int] = None,
|
||||
n_probs: Optional[int] = None,
|
||||
cache_prompt: Optional[bool] = None,
|
||||
return_tokens: Optional[bool] = None,
|
||||
timings_per_token: Optional[bool] = None,
|
||||
post_sampling_probs: Optional[bool] = None,
|
||||
stream: bool = True,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""
|
||||
|
|
@ -1067,10 +1095,76 @@ class ExternalProviderClient:
|
|||
if parallel_tool_calls is not None:
|
||||
body["parallel_tool_calls"] = parallel_tool_calls
|
||||
|
||||
# Extended OAI-compat samplers (OpenRouter `top_a`, vLLM output
|
||||
# knobs, llama.cpp samplers on custom proxies). Each is gated `is
|
||||
# not None` so explicit 0/False reach the wire; `body_omit` below
|
||||
# strips fields the upstream rejects.
|
||||
if typical_p is not None:
|
||||
body["typical_p"] = typical_p
|
||||
if top_n_sigma is not None:
|
||||
body["top_n_sigma"] = top_n_sigma
|
||||
if repeat_last_n is not None:
|
||||
body["repeat_last_n"] = repeat_last_n
|
||||
if dynatemp_range is not None:
|
||||
body["dynatemp_range"] = dynatemp_range
|
||||
if dynatemp_exponent is not None:
|
||||
body["dynatemp_exponent"] = dynatemp_exponent
|
||||
if mirostat is not None:
|
||||
body["mirostat"] = mirostat
|
||||
if mirostat_tau is not None:
|
||||
body["mirostat_tau"] = mirostat_tau
|
||||
if mirostat_eta is not None:
|
||||
body["mirostat_eta"] = mirostat_eta
|
||||
if top_a is not None:
|
||||
body["top_a"] = top_a
|
||||
if dry_multiplier is not None:
|
||||
body["dry_multiplier"] = dry_multiplier
|
||||
if dry_base is not None:
|
||||
body["dry_base"] = dry_base
|
||||
if dry_allowed_length is not None:
|
||||
body["dry_allowed_length"] = dry_allowed_length
|
||||
if dry_penalty_last_n is not None:
|
||||
body["dry_penalty_last_n"] = dry_penalty_last_n
|
||||
if xtc_probability is not None:
|
||||
body["xtc_probability"] = xtc_probability
|
||||
if xtc_threshold is not None:
|
||||
body["xtc_threshold"] = xtc_threshold
|
||||
if min_keep is not None:
|
||||
body["min_keep"] = min_keep
|
||||
if ignore_eos is not None:
|
||||
body["ignore_eos"] = ignore_eos
|
||||
if min_tokens is not None:
|
||||
body["min_tokens"] = min_tokens
|
||||
if skip_special_tokens is not None:
|
||||
body["skip_special_tokens"] = skip_special_tokens
|
||||
if spaces_between_special_tokens is not None:
|
||||
body["spaces_between_special_tokens"] = spaces_between_special_tokens
|
||||
if include_stop_str_in_output is not None:
|
||||
body["include_stop_str_in_output"] = include_stop_str_in_output
|
||||
if truncate_prompt_tokens is not None:
|
||||
body["truncate_prompt_tokens"] = truncate_prompt_tokens
|
||||
if n_keep is not None:
|
||||
body["n_keep"] = n_keep
|
||||
if n_probs is not None:
|
||||
body["n_probs"] = n_probs
|
||||
if cache_prompt is not None:
|
||||
body["cache_prompt"] = cache_prompt
|
||||
if return_tokens is not None:
|
||||
body["return_tokens"] = return_tokens
|
||||
if timings_per_token is not None:
|
||||
body["timings_per_token"] = timings_per_token
|
||||
if post_sampling_probs is not None:
|
||||
body["post_sampling_probs"] = post_sampling_probs
|
||||
|
||||
# Drop body fields the provider's registry entry locks down
|
||||
# (e.g. Kimi k2.5/k2.6 only accept temperature=1, top_p=1).
|
||||
# Also pop the renamed seed field so `body_omit=("seed",)` on a
|
||||
# provider with `seed_field` rename still strips correctly.
|
||||
_seed_field = provider_info.get("seed_field", "seed")
|
||||
for field in provider_info.get("body_omit", ()):
|
||||
body.pop(field, None)
|
||||
if field == "seed" and _seed_field != "seed":
|
||||
body.pop(_seed_field, None)
|
||||
|
||||
# Kimi thinking is a top-level body field. kimi-k2-thinking is
|
||||
# always on (ignore the toggle); kimi-k2.6 defaults on, can be
|
||||
|
|
|
|||
|
|
@ -192,6 +192,9 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = {
|
|||
# Mistral renames OpenAI's `seed` to `random_seed` on
|
||||
# /v1/chat/completions. https://docs.mistral.ai/api/endpoint/chat
|
||||
"seed_field": "random_seed",
|
||||
# Mistral's docs publish no max but third-party shims cap at 4;
|
||||
# match OpenAI Chat's cap to avoid silent upstream truncation.
|
||||
"stop_max": 4,
|
||||
},
|
||||
"kimi": {
|
||||
"display_name": "Kimi",
|
||||
|
|
@ -216,12 +219,13 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = {
|
|||
"notes": "Moonshot API key. China: use base URL https://api.moonshot.cn/v1",
|
||||
"model_id_allowlist": re.compile(r"^kimi-k2\.[56]$"),
|
||||
# k2.5/k2.6 are reasoning-class: API locks temperature=1, top_p=1,
|
||||
# frequency_penalty; seed and parallel_tool_calls are undocumented
|
||||
# and 400.
|
||||
# frequency_penalty; presence_penalty / seed / parallel_tool_calls
|
||||
# are undocumented in the Kimi chat schema.
|
||||
"body_omit": (
|
||||
"temperature",
|
||||
"top_p",
|
||||
"frequency_penalty",
|
||||
"presence_penalty",
|
||||
"seed",
|
||||
"parallel_tool_calls",
|
||||
),
|
||||
|
|
|
|||
|
|
@ -99,6 +99,9 @@ class ChatExportResponse(BaseModel):
|
|||
|
||||
|
||||
class ChatInferenceSettings(BaseModel):
|
||||
# extra="forbid" requires every persisted key to be listed. Keep
|
||||
# aligned with PERSISTED_INFERENCE_PARAM_KEYS in
|
||||
# studio/frontend/src/features/chat/stores/chat-runtime-store.ts.
|
||||
model_config = ConfigDict(extra = "forbid")
|
||||
|
||||
temperature: Optional[float] = None
|
||||
|
|
@ -107,10 +110,6 @@ class ChatInferenceSettings(BaseModel):
|
|||
minP: Optional[float] = None
|
||||
repetitionPenalty: Optional[float] = None
|
||||
presencePenalty: Optional[float] = None
|
||||
# New per-provider sampling knobs. extra="forbid" requires these
|
||||
# to be listed; otherwise every save from the new frontend 422s.
|
||||
# Keep aligned with InferenceParams in
|
||||
# studio/frontend/src/features/chat/types/runtime.ts.
|
||||
frequencyPenalty: Optional[float] = Field(default = None, ge = -2.0, le = 2.0)
|
||||
seed: Optional[int] = None
|
||||
stop: Optional[list[str]] = None
|
||||
|
|
@ -122,6 +121,36 @@ class ChatInferenceSettings(BaseModel):
|
|||
maxTokens: Optional[float] = None
|
||||
systemPrompt: Optional[str] = None
|
||||
trustRemoteCode: Optional[bool] = None
|
||||
fastMode: Optional[bool] = None
|
||||
# Extended llama.cpp / vLLM / OpenRouter samplers exposed by PR #5711.
|
||||
typicalP: Optional[float] = None
|
||||
topNSigma: Optional[float] = None
|
||||
repeatLastN: Optional[int] = None
|
||||
dynatempRange: Optional[float] = None
|
||||
dynatempExponent: Optional[float] = None
|
||||
mirostat: Optional[int] = None
|
||||
mirostatTau: Optional[float] = None
|
||||
mirostatEta: Optional[float] = None
|
||||
topA: Optional[float] = None
|
||||
dryMultiplier: Optional[float] = None
|
||||
dryBase: Optional[float] = None
|
||||
dryAllowedLength: Optional[int] = None
|
||||
dryPenaltyLastN: Optional[int] = None
|
||||
xtcProbability: Optional[float] = None
|
||||
xtcThreshold: Optional[float] = None
|
||||
minKeep: Optional[int] = None
|
||||
ignoreEos: Optional[bool] = None
|
||||
minTokens: Optional[int] = None
|
||||
skipSpecialTokens: Optional[bool] = None
|
||||
spacesBetweenSpecialTokens: Optional[bool] = None
|
||||
includeStopStrInOutput: Optional[bool] = None
|
||||
truncatePromptTokens: Optional[int] = None
|
||||
nKeep: Optional[int] = None
|
||||
nProbs: Optional[int] = None
|
||||
cachePrompt: Optional[bool] = None
|
||||
returnTokens: Optional[bool] = None
|
||||
timingsPerToken: Optional[bool] = None
|
||||
postSamplingProbs: Optional[bool] = None
|
||||
|
||||
|
||||
class ChatPreset(BaseModel):
|
||||
|
|
|
|||
|
|
@ -2168,6 +2168,34 @@ async def _proxy_to_external_provider(
|
|||
tools = payload.tools,
|
||||
tool_choice = payload.tool_choice,
|
||||
fast_mode = payload.fast_mode,
|
||||
typical_p = payload.typical_p,
|
||||
top_n_sigma = payload.top_n_sigma,
|
||||
repeat_last_n = payload.repeat_last_n,
|
||||
dynatemp_range = payload.dynatemp_range,
|
||||
dynatemp_exponent = payload.dynatemp_exponent,
|
||||
mirostat = payload.mirostat,
|
||||
mirostat_tau = payload.mirostat_tau,
|
||||
mirostat_eta = payload.mirostat_eta,
|
||||
top_a = payload.top_a,
|
||||
dry_multiplier = payload.dry_multiplier,
|
||||
dry_base = payload.dry_base,
|
||||
dry_allowed_length = payload.dry_allowed_length,
|
||||
dry_penalty_last_n = payload.dry_penalty_last_n,
|
||||
xtc_probability = payload.xtc_probability,
|
||||
xtc_threshold = payload.xtc_threshold,
|
||||
min_keep = payload.min_keep,
|
||||
ignore_eos = payload.ignore_eos,
|
||||
min_tokens = payload.min_tokens,
|
||||
skip_special_tokens = payload.skip_special_tokens,
|
||||
spaces_between_special_tokens = payload.spaces_between_special_tokens,
|
||||
include_stop_str_in_output = payload.include_stop_str_in_output,
|
||||
truncate_prompt_tokens = payload.truncate_prompt_tokens,
|
||||
n_keep = payload.n_keep,
|
||||
n_probs = payload.n_probs,
|
||||
cache_prompt = payload.cache_prompt,
|
||||
return_tokens = payload.return_tokens,
|
||||
timings_per_token = payload.timings_per_token,
|
||||
post_sampling_probs = payload.post_sampling_probs,
|
||||
stream = payload.stream,
|
||||
)
|
||||
try:
|
||||
|
|
@ -6009,6 +6037,25 @@ def _build_openai_passthrough_body(payload, backend_ctx = None) -> dict:
|
|||
mirostat = payload.mirostat,
|
||||
mirostat_tau = payload.mirostat_tau,
|
||||
mirostat_eta = payload.mirostat_eta,
|
||||
dry_multiplier = payload.dry_multiplier,
|
||||
dry_base = payload.dry_base,
|
||||
dry_allowed_length = payload.dry_allowed_length,
|
||||
dry_penalty_last_n = payload.dry_penalty_last_n,
|
||||
xtc_probability = payload.xtc_probability,
|
||||
xtc_threshold = payload.xtc_threshold,
|
||||
min_keep = payload.min_keep,
|
||||
ignore_eos = payload.ignore_eos,
|
||||
min_tokens = payload.min_tokens,
|
||||
skip_special_tokens = payload.skip_special_tokens,
|
||||
spaces_between_special_tokens = payload.spaces_between_special_tokens,
|
||||
include_stop_str_in_output = payload.include_stop_str_in_output,
|
||||
truncate_prompt_tokens = payload.truncate_prompt_tokens,
|
||||
n_keep = payload.n_keep,
|
||||
n_probs = payload.n_probs,
|
||||
cache_prompt = payload.cache_prompt,
|
||||
return_tokens = payload.return_tokens,
|
||||
timings_per_token = payload.timings_per_token,
|
||||
post_sampling_probs = payload.post_sampling_probs,
|
||||
tool_choice = tool_choice,
|
||||
response_format = _extract_response_format(payload),
|
||||
chat_template_kwargs = tpl_kwargs,
|
||||
|
|
|
|||
|
|
@ -239,22 +239,20 @@ def test_anthropic_rejects_openai_only_knobs(monkeypatch):
|
|||
def _drive_openai_compat(captured, **kwargs) -> dict:
|
||||
"""Send through the default OAI-compat branch (NOT /v1/responses).
|
||||
|
||||
Use a non-OpenAI provider_type so the dispatcher takes the default
|
||||
branch at the bottom of stream_chat_completion rather than the
|
||||
Responses translator path that routes provider_type=="openai".
|
||||
Use qwen so the dispatcher takes the default branch and the provider
|
||||
inherits the default 16-stop cap (Mistral now caps at 4 per its
|
||||
third-party shims).
|
||||
"""
|
||||
|
||||
async def run():
|
||||
client = ExternalProviderClient(
|
||||
provider_type = "mistral",
|
||||
base_url = "https://api.mistral.ai/v1",
|
||||
provider_type = "qwen",
|
||||
base_url = "https://dashscope-intl.aliyuncs.com/compatible-mode/v1",
|
||||
api_key = "test-key",
|
||||
)
|
||||
# mistral's OpenAI-compat /v1/chat/completions returns OpenAI
|
||||
# SSE; a single DONE frame is enough to drain the stream.
|
||||
async for _ in client.stream_chat_completion(
|
||||
messages = [{"role": "user", "content": "hi"}],
|
||||
model = "mistral-small-latest",
|
||||
model = "qwen-plus",
|
||||
temperature = 0.5,
|
||||
top_p = 0.9,
|
||||
max_tokens = 64,
|
||||
|
|
@ -278,10 +276,36 @@ def test_openai_compat_forwards_frequency_penalty(monkeypatch):
|
|||
|
||||
|
||||
def test_openai_compat_forwards_seed(monkeypatch):
|
||||
"""qwen has no seed_field override so seed forwards as `seed`."""
|
||||
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
||||
body = _drive_openai_compat(captured, seed = 12345)
|
||||
# Default OAI-compat provider (mistral here) renames seed to
|
||||
# random_seed via provider registry's seed_field.
|
||||
assert body.get("seed") == 12345, body
|
||||
|
||||
|
||||
def test_mistral_renames_seed_to_random_seed(monkeypatch):
|
||||
"""Mistral's registry sets seed_field="random_seed" so the OAI seed
|
||||
is renamed on the wire. https://docs.mistral.ai/api/endpoint/chat"""
|
||||
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
||||
|
||||
async def run():
|
||||
client = ExternalProviderClient(
|
||||
provider_type = "mistral",
|
||||
base_url = "https://api.mistral.ai/v1",
|
||||
api_key = "mistral-test",
|
||||
)
|
||||
async for _ in client.stream_chat_completion(
|
||||
messages = [{"role": "user", "content": "hi"}],
|
||||
model = "mistral-small-latest",
|
||||
temperature = 0.5,
|
||||
top_p = 0.9,
|
||||
max_tokens = 64,
|
||||
seed = 12345,
|
||||
):
|
||||
pass
|
||||
await client.close()
|
||||
|
||||
_drive(run())
|
||||
body = captured["body"]
|
||||
assert body.get("random_seed") == 12345, body
|
||||
assert "seed" not in body, body
|
||||
|
||||
|
|
@ -349,8 +373,8 @@ def test_openai_compat_forwards_stop_array(monkeypatch):
|
|||
|
||||
|
||||
def test_openai_compat_truncates_stop_to_default_cap(monkeypatch):
|
||||
"""Default OAI-compat cap is 16 (DeepSeek and Mistral both accept
|
||||
that many); only OpenAI Chat has a tighter 4-entry hard limit."""
|
||||
"""Default OAI-compat cap is 16 (Qwen, DeepSeek, HuggingFace, custom);
|
||||
OpenAI Chat / OpenRouter / Gemini / Mistral have tighter 4-entry caps."""
|
||||
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
||||
body = _drive_openai_compat(captured, stop = [f"s{i}" for i in range(20)])
|
||||
assert len(body.get("stop", [])) == 16, body
|
||||
|
|
@ -358,6 +382,33 @@ def test_openai_compat_truncates_stop_to_default_cap(monkeypatch):
|
|||
assert body["stop"][-1] == "s15"
|
||||
|
||||
|
||||
def test_mistral_stop_cap_is_4(monkeypatch):
|
||||
"""Mistral's docs publish no max but third-party shims cap at 4;
|
||||
match OpenAI Chat's cap to avoid silent upstream truncation."""
|
||||
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
||||
|
||||
async def run():
|
||||
client = ExternalProviderClient(
|
||||
provider_type = "mistral",
|
||||
base_url = "https://api.mistral.ai/v1",
|
||||
api_key = "mistral-test",
|
||||
)
|
||||
async for _ in client.stream_chat_completion(
|
||||
messages = [{"role": "user", "content": "hi"}],
|
||||
model = "mistral-small-latest",
|
||||
temperature = 0.5,
|
||||
top_p = 0.9,
|
||||
max_tokens = 64,
|
||||
stop = [f"S{i}" for i in range(8)],
|
||||
):
|
||||
pass
|
||||
await client.close()
|
||||
|
||||
_drive(run())
|
||||
body = captured["body"]
|
||||
assert body.get("stop") == ["S0", "S1", "S2", "S3"], body
|
||||
|
||||
|
||||
def test_openai_compat_stop_dedup_and_drop_empties(monkeypatch):
|
||||
"""Duplicates and empties shouldn't eat into the cap."""
|
||||
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
||||
|
|
@ -590,9 +641,9 @@ def test_kimi_web_search_bypass_forwards_new_sampling_fields(monkeypatch):
|
|||
assert "top_p" not in body, body
|
||||
assert "seed" not in body, body
|
||||
assert "parallel_tool_calls" not in body, body
|
||||
assert "presence_penalty" not in body, body
|
||||
# Knobs not on Kimi's drop-list forward through the bypass.
|
||||
assert body.get("stop") == ["END"], body
|
||||
assert body.get("presence_penalty") == 0.5, body
|
||||
|
||||
|
||||
def test_kimi_web_search_uses_kimi_stop_cap_5(monkeypatch):
|
||||
|
|
@ -681,6 +732,72 @@ def test_gemini_stop_sequences_capped_to_5(monkeypatch):
|
|||
assert gen_config.get("stopSequences") == ["S0", "S1", "S2", "S3", "S4"], body
|
||||
|
||||
|
||||
def test_openrouter_forwards_top_a(monkeypatch):
|
||||
"""OpenRouter exposes `top_a` (the tail-cut sampler). The frontend
|
||||
capability map lets the value through; verify the OAI-compat body
|
||||
builder actually carries it to the wire."""
|
||||
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
||||
|
||||
async def run():
|
||||
client = ExternalProviderClient(
|
||||
provider_type = "openrouter",
|
||||
base_url = "https://openrouter.ai/api/v1",
|
||||
api_key = "or-test",
|
||||
)
|
||||
async for _ in client.stream_chat_completion(
|
||||
messages = [{"role": "user", "content": "hi"}],
|
||||
model = "openai/gpt-4o",
|
||||
temperature = 0.5,
|
||||
top_p = 0.9,
|
||||
max_tokens = 64,
|
||||
top_a = 0.25,
|
||||
):
|
||||
pass
|
||||
await client.close()
|
||||
|
||||
_drive(run())
|
||||
body = captured["body"]
|
||||
assert body.get("top_a") == 0.25, body
|
||||
|
||||
|
||||
def test_vllm_forwards_output_shape_knobs(monkeypatch):
|
||||
"""vLLM accepts skip_special_tokens / spaces_between_special_tokens /
|
||||
include_stop_str_in_output / truncate_prompt_tokens. Verify the
|
||||
OAI-compat external proxy forwards them to the wire body."""
|
||||
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
||||
|
||||
async def run():
|
||||
client = ExternalProviderClient(
|
||||
provider_type = "vllm",
|
||||
base_url = "https://vllm.example.com/v1",
|
||||
api_key = "vllm-test",
|
||||
)
|
||||
async for _ in client.stream_chat_completion(
|
||||
messages = [{"role": "user", "content": "hi"}],
|
||||
model = "Qwen/Qwen3-4B",
|
||||
temperature = 0.5,
|
||||
top_p = 0.9,
|
||||
max_tokens = 64,
|
||||
skip_special_tokens = False,
|
||||
spaces_between_special_tokens = False,
|
||||
include_stop_str_in_output = True,
|
||||
truncate_prompt_tokens = 2048,
|
||||
min_tokens = 8,
|
||||
ignore_eos = True,
|
||||
):
|
||||
pass
|
||||
await client.close()
|
||||
|
||||
_drive(run())
|
||||
body = captured["body"]
|
||||
assert body.get("skip_special_tokens") is False, body
|
||||
assert body.get("spaces_between_special_tokens") is False, body
|
||||
assert body.get("include_stop_str_in_output") is True, body
|
||||
assert body.get("truncate_prompt_tokens") == 2048, body
|
||||
assert body.get("min_tokens") == 8, body
|
||||
assert body.get("ignore_eos") is True, body
|
||||
|
||||
|
||||
def test_kimi_drops_stop_strings_over_32_bytes(monkeypatch):
|
||||
"""Kimi limits each stop string to <= 32 bytes per
|
||||
https://platform.kimi.ai/docs/api/chat. Drop overlong entries
|
||||
|
|
@ -837,27 +954,52 @@ def test_responses_to_chat_bridge_omits_unset_parallel_tool_calls():
|
|||
def test_chat_settings_payload_accepts_new_sampling_keys():
|
||||
"""ChatSettingsPayload has extra="forbid" so the new keys must be
|
||||
listed explicitly; otherwise every settings save with any of them
|
||||
422s. Pin the round-trip."""
|
||||
422s. Pin the round-trip for every persisted sampler the frontend
|
||||
can emit (see PERSISTED_INFERENCE_PARAM_KEYS in chat-runtime-store.ts)."""
|
||||
from routes.chat_history import ChatSettingsPayload
|
||||
|
||||
parsed = ChatSettingsPayload.model_validate(
|
||||
{
|
||||
"inferenceParams": {
|
||||
"frequencyPenalty": 0.7,
|
||||
"seed": 42,
|
||||
"stop": ["END"],
|
||||
"serviceTier": "standard_only",
|
||||
"parallelToolCalls": False,
|
||||
}
|
||||
}
|
||||
)
|
||||
inference = {
|
||||
"frequencyPenalty": 0.7,
|
||||
"seed": 42,
|
||||
"stop": ["END"],
|
||||
"serviceTier": "standard_only",
|
||||
"parallelToolCalls": False,
|
||||
"fastMode": True,
|
||||
# Extended llama.cpp / vLLM / OpenRouter samplers.
|
||||
"typicalP": 0.85,
|
||||
"topNSigma": 2.5,
|
||||
"repeatLastN": 64,
|
||||
"dynatempRange": 0.3,
|
||||
"dynatempExponent": 1.2,
|
||||
"mirostat": 2,
|
||||
"mirostatTau": 4.0,
|
||||
"mirostatEta": 0.15,
|
||||
"topA": 0.2,
|
||||
"dryMultiplier": 0.8,
|
||||
"dryBase": 1.75,
|
||||
"dryAllowedLength": 2,
|
||||
"dryPenaltyLastN": -1,
|
||||
"xtcProbability": 0.5,
|
||||
"xtcThreshold": 0.1,
|
||||
"minKeep": 5,
|
||||
"ignoreEos": True,
|
||||
"minTokens": 10,
|
||||
"skipSpecialTokens": False,
|
||||
"spacesBetweenSpecialTokens": False,
|
||||
"includeStopStrInOutput": True,
|
||||
"truncatePromptTokens": 1024,
|
||||
"nKeep": -1,
|
||||
"nProbs": 5,
|
||||
"cachePrompt": False,
|
||||
"returnTokens": True,
|
||||
"timingsPerToken": True,
|
||||
"postSamplingProbs": True,
|
||||
}
|
||||
parsed = ChatSettingsPayload.model_validate({"inferenceParams": inference})
|
||||
ip = parsed.inferenceParams
|
||||
assert ip is not None
|
||||
assert ip.frequencyPenalty == 0.7
|
||||
assert ip.seed == 42
|
||||
assert ip.stop == ["END"]
|
||||
assert ip.serviceTier == "standard_only"
|
||||
assert ip.parallelToolCalls is False
|
||||
for key, expected in inference.items():
|
||||
assert getattr(ip, key) == expected, f"{key} did not round-trip"
|
||||
|
||||
|
||||
# ── Local /v1/messages: disable_parallel_tool_use translation ──────────
|
||||
|
|
@ -1093,7 +1235,7 @@ def test_local_passthrough_forwards_dry_xtc_min_keep_eos_min_tokens():
|
|||
assert body.get("min_tokens") == 16
|
||||
|
||||
# Unset = absent from body. Matches the upstream "use default"
|
||||
# contract — llama-server / vLLM apply their own defaults instead.
|
||||
# contract: llama-server / vLLM apply their own defaults instead.
|
||||
body2 = route_mod._build_passthrough_payload(
|
||||
openai_messages = [{"role": "user", "content": "hi"}],
|
||||
openai_tools = None,
|
||||
|
|
|
|||
|
|
@ -2335,7 +2335,12 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
...(params.postSamplingProbs === true
|
||||
? { post_sampling_probs: true }
|
||||
: {}),
|
||||
parallel_tool_calls: params.parallelToolCalls,
|
||||
// Forward only on explicit opt-out (default true on every
|
||||
// backend; default omit keeps wire-shape stable for users
|
||||
// who never opened the new settings panel).
|
||||
...(params.parallelToolCalls === false
|
||||
? { parallel_tool_calls: false }
|
||||
: {}),
|
||||
image_base64: imageBase64,
|
||||
audio_base64: audioBase64,
|
||||
cancel_id: cancelId,
|
||||
|
|
|
|||
|
|
@ -133,21 +133,11 @@ export function InfoHint({ children }: { children: ReactNode }) {
|
|||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Editable numeric value display.
|
||||
*
|
||||
* Renders as a single <input> that *looks* like text by default —
|
||||
* transparent background, no border, no ring — and only shows a faint
|
||||
* surface tint on hover/focus to signal editability. When unfocused,
|
||||
* the input shows the formatted display string (`displayValue ?? value`,
|
||||
* so labels like "Off" / "Max" still render); on focus, it switches to
|
||||
* the raw numeric value, selects it, and accepts free text input.
|
||||
* Commit happens on blur or Enter; Escape reverts. The clamp-to-range
|
||||
* happens on commit so users can type intermediate values without the
|
||||
* input fighting them mid-keystroke. Single component shared by every
|
||||
* slider value and the Context Length input so the click-to-edit
|
||||
* affordance is consistent across the panel.
|
||||
*/
|
||||
/** Editable numeric value display: transparent text-like input that
|
||||
* shows formatted display on blur (so "Off"/"Max" labels render) and
|
||||
* switches to the raw number on focus. Commits on blur/Enter, reverts
|
||||
* on Escape, clamps on commit. Shared by every slider value + the
|
||||
* Context Length input. */
|
||||
function snapToStep(
|
||||
value: number,
|
||||
step: number,
|
||||
|
|
@ -1431,7 +1421,19 @@ export function ChatSettingsPanel({
|
|||
</InfoHint>
|
||||
</div>
|
||||
<Select
|
||||
value={params.serviceTier ?? "auto"}
|
||||
value={
|
||||
// Fall back to "auto" when the persisted tier is not
|
||||
// legal for the active provider (e.g. "priority" saved
|
||||
// on OpenAI, then user switched to Anthropic which only
|
||||
// accepts auto|standard_only). Without this Radix Select
|
||||
// shows a blank trigger.
|
||||
params.serviceTier &&
|
||||
(serviceTierOptions as readonly ServiceTier[]).includes(
|
||||
params.serviceTier,
|
||||
)
|
||||
? params.serviceTier
|
||||
: "auto"
|
||||
}
|
||||
onValueChange={(value) => {
|
||||
// Store "auto" verbatim: Anthropic distinguishes
|
||||
// omitted (provider default) from auto (Priority Tier opt-in).
|
||||
|
|
@ -1484,7 +1486,7 @@ export function ChatSettingsPanel({
|
|||
max={32768}
|
||||
step={128}
|
||||
onChange={set("maxSeqLength")}
|
||||
info="Maximum context window size in tokens — input prompt plus generated output combined. Capped by the model's trained limit."
|
||||
info="Maximum context window in tokens (prompt plus generated output). Capped by the model's trained limit."
|
||||
/>
|
||||
)}
|
||||
<ParamSlider
|
||||
|
|
|
|||
|
|
@ -553,7 +553,7 @@ const OPENAI_COMPAT_BASE: ProviderCapabilities = {
|
|||
// plus the permissive `custom` preset. Exposes the full llama.cpp
|
||||
// sampler chain (typical_p / top_n_sigma / mirostat / dynatemp /
|
||||
// repeat_last_n) per the upstream server README. Not used for vLLM or
|
||||
// Ollama — see VLLM_OLLAMA_CAPABILITIES below.
|
||||
// Ollama; see VLLM_OLLAMA_CAPABILITIES below.
|
||||
const LLAMA_CPP_CAPABILITIES: ProviderCapabilities = {
|
||||
temperature: true,
|
||||
topP: true,
|
||||
|
|
@ -630,8 +630,8 @@ const VLLM_CAPABILITIES: ProviderCapabilities = {
|
|||
};
|
||||
|
||||
// Ollama OAI translator (openai/openai.go FromChatRequest) only copies
|
||||
// the documented OpenAI subset on /v1/chat/completions — top_k / min_p
|
||||
// / repetition_penalty / ignore_eos / min_tokens / the 4 vLLM output
|
||||
// the documented OpenAI subset on /v1/chat/completions: top_k, min_p,
|
||||
// repetition_penalty, ignore_eos, min_tokens, and the 4 vLLM output
|
||||
// knobs all silently drop on this path. (Native /api/chat would forward
|
||||
// them via `options`, but Studio uses /v1.)
|
||||
const OLLAMA_CAPABILITIES: ProviderCapabilities = {
|
||||
|
|
@ -974,7 +974,7 @@ const PROVIDER_CAPABILITIES: Record<string, ProviderCapabilities> = {
|
|||
postSamplingProbs: false,
|
||||
},
|
||||
// DeepSeek schema (api-docs.deepseek.com/api/create-chat-completion)
|
||||
// lists temperature/top_p/stop only — no seed or parallel_tool_calls.
|
||||
// lists temperature/top_p/stop only; no seed or parallel_tool_calls.
|
||||
// Presence/frequency are deprecated. Reasoner ids additionally ignore
|
||||
// temperature/top_p; getProviderCapabilities downshifts them.
|
||||
deepseek: {
|
||||
|
|
@ -1023,7 +1023,7 @@ const PROVIDER_CAPABILITIES: Record<string, ProviderCapabilities> = {
|
|||
openrouter: OPENROUTER_CAPABILITIES,
|
||||
// llama_cpp + custom: first-party llama-server, full chain.
|
||||
// vllm: OAI subset + top_k/min_p/repetition_penalty/seed.
|
||||
// ollama: stricter — OAI translator drops top_k/min_p/rep_pen too.
|
||||
// ollama: stricter (OAI translator drops top_k/min_p/rep_pen too).
|
||||
custom: LLAMA_CPP_CAPABILITIES,
|
||||
llama_cpp: LLAMA_CPP_CAPABILITIES,
|
||||
vllm: VLLM_CAPABILITIES,
|
||||
|
|
@ -1055,8 +1055,8 @@ export function getProviderCapabilities(
|
|||
|
||||
const DEFAULT_EFFORT_LEVELS = ["low", "medium", "high"] as const;
|
||||
// OpenRouter ids with no non-reasoning mode. (google/gemini-pro-latest
|
||||
// was dropped — gateway 404s; don't re-pin to a versioned id that
|
||||
// may rotate again.)
|
||||
// was dropped: gateway 404s; don't re-pin to a versioned id that may
|
||||
// rotate again.)
|
||||
const OPENROUTER_MANDATORY_REASONING_MODELS = new Set([
|
||||
"baidu/cobuddy:free",
|
||||
"inclusionai/ring-2.6-1t:free",
|
||||
|
|
@ -1377,7 +1377,7 @@ export interface ExternalReasoningResolveOptions {
|
|||
baseUrl?: string | null;
|
||||
}
|
||||
|
||||
// vLLM has no per-model reasoning signal on OpenAI-compat — pin via user toggle.
|
||||
// vLLM has no per-model reasoning signal on OpenAI-compat; pin via user toggle.
|
||||
function resolveConnectionLevelReasoning(
|
||||
normalizedProvider: string,
|
||||
options: ExternalReasoningResolveOptions | undefined,
|
||||
|
|
|
|||
|
|
@ -416,6 +416,35 @@ const PERSISTED_INFERENCE_PARAM_KEYS = [
|
|||
"systemPrompt",
|
||||
"trustRemoteCode",
|
||||
"fastMode",
|
||||
// Extended llama.cpp / vLLM / OpenRouter samplers exposed by PR #5711.
|
||||
"typicalP",
|
||||
"topNSigma",
|
||||
"repeatLastN",
|
||||
"dynatempRange",
|
||||
"dynatempExponent",
|
||||
"mirostat",
|
||||
"mirostatTau",
|
||||
"mirostatEta",
|
||||
"topA",
|
||||
"dryMultiplier",
|
||||
"dryBase",
|
||||
"dryAllowedLength",
|
||||
"dryPenaltyLastN",
|
||||
"xtcProbability",
|
||||
"xtcThreshold",
|
||||
"minKeep",
|
||||
"ignoreEos",
|
||||
"minTokens",
|
||||
"skipSpecialTokens",
|
||||
"spacesBetweenSpecialTokens",
|
||||
"includeStopStrInOutput",
|
||||
"truncatePromptTokens",
|
||||
"nKeep",
|
||||
"nProbs",
|
||||
"cachePrompt",
|
||||
"returnTokens",
|
||||
"timingsPerToken",
|
||||
"postSamplingProbs",
|
||||
] as const satisfies readonly PersistedInferenceParamKey[];
|
||||
|
||||
const SCALAR_SETTING_KEYS = [
|
||||
|
|
|
|||
|
|
@ -52,11 +52,11 @@ export interface InferenceParams {
|
|||
dryAllowedLength: number | null;
|
||||
/** 0 disables, -1 = ctx-size. */
|
||||
dryPenaltyLastN: number | null;
|
||||
/** llama.cpp XTC — probability is the master switch (0 disables). */
|
||||
/** llama.cpp XTC: probability is the master switch (0 disables). */
|
||||
xtcProbability: number | null;
|
||||
/** Default 0.1. */
|
||||
xtcThreshold: number | null;
|
||||
/** llama.cpp `min_keep` — min tokens past all filters. 0 disables. */
|
||||
/** llama.cpp `min_keep`: min tokens past all filters. 0 disables. */
|
||||
minKeep: number | null;
|
||||
/** Continue past EOS. llama.cpp + vLLM. */
|
||||
ignoreEos: boolean | null;
|
||||
|
|
@ -72,7 +72,7 @@ export interface InferenceParams {
|
|||
truncatePromptTokens: number | null;
|
||||
/** llama.cpp `n_keep`. 0 disables, -1 = keep all. */
|
||||
nKeep: number | null;
|
||||
/** llama.cpp `n_probs` — top-N token probabilities per token. */
|
||||
/** llama.cpp `n_probs`: top-N token probabilities per token. */
|
||||
nProbs: number | null;
|
||||
/** llama.cpp `cache_prompt`. Default true; forward only when false. */
|
||||
cachePrompt: boolean | null;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue