d6765fddce
Round-2 round of review-feedback fixes for the sampling-knobs PR: - studio/backend/routes/chat_history.py: ChatInferenceSettings still had the pre-PR field list with extra="forbid", so every settings save the new frontend issued would 422 on the new keys (frequencyPenalty, seed, stop, serviceTier, parallelToolCalls). Add the fields with the same range / enum constraints the chat-completions schema uses, so the settings-persistence path round-trips cleanly. - studio/backend/routes/inference.py: _build_passthrough_payload and _build_openai_passthrough_body now thread frequency_penalty, seed, and parallel_tool_calls through to llama-server. The frontend exposes these knobs for local backends; without the forwarding the UI was a decoration. Each field is gated on `is not None` so 0 / False / "0" still reach the body. - studio/backend/core/inference/external_provider.py: the Kimi $web_search bypass takes an early return into _stream_kimi_web_search before the default OAI-compat body builder runs, so the new sampling fields never landed on Kimi-with-search. Forward them through the helper, with the same dedupe / truncate behavior the main path applies to `stop`. Also extend the OpenAI Responses service_tier allowlist to include `scale` per the live openai-python SDK (response_create_params.py declares Literal["auto","default","flex","scale","priority"]). - studio/frontend/src/features/chat/provider-capabilities.ts + types/runtime.ts: add `scale` to ServiceTier / ServiceTierOption and surface it on the OpenAI Responses options so the UI matches the upstream enum. - studio/backend/tests/test_sampling_params_routing.py: add tests for every gap above: Kimi web-search bypass forwarding, local OpenAI passthrough forwarding, ChatSettingsPayload round-trip, and the full Responses service_tier enum (parametrized over the five accepted values plus a drop check for the Anthropic-only standard_only).
578 lines
22 KiB
Python
578 lines
22 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-only
|
|
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
|
|
|
"""End-to-end routing tests for the new sampling parameters.
|
|
|
|
Pins the per-provider gating contract added by the
|
|
expose-sampling-params PR: each of `frequency_penalty`, `seed`, `stop`
|
|
/ `stop_sequences`, `service_tier`, `parallel_tool_calls` only appears
|
|
on the outbound body when the upstream provider actually accepts it.
|
|
|
|
The provider matrix is captured per docs:
|
|
- Anthropic Messages: accepts stop_sequences, service_tier
|
|
(auto|standard_only), disable_parallel_tool_use (inverted). REJECTS
|
|
frequency_penalty, seed, logprobs (silently dropped client-side).
|
|
- OpenAI Chat Completions (default OAI-compat branch): accepts every
|
|
field; OpenAI cloud uses `max_completion_tokens` rather than
|
|
`max_tokens`.
|
|
- OpenAI Responses (gpt-5.x / o3): rejects temperature, top_p,
|
|
frequency_penalty, seed, stop, logprobs. Accepts service_tier
|
|
(auto|default|flex|priority) and parallel_tool_calls.
|
|
"""
|
|
|
|
import asyncio
|
|
import json
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from core.inference import external_provider as ep_mod
|
|
from core.inference.external_provider import ExternalProviderClient
|
|
|
|
|
|
def _drive(coro):
|
|
return asyncio.new_event_loop().run_until_complete(coro)
|
|
|
|
|
|
def _install_mock(monkeypatch, *, sse_payload: bytes | None = None) -> dict:
|
|
captured: dict = {}
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
try:
|
|
captured["body"] = json.loads(request.content.decode("utf-8"))
|
|
except json.JSONDecodeError:
|
|
captured["body"] = None
|
|
captured["url"] = str(request.url)
|
|
captured["headers"] = dict(request.headers)
|
|
return httpx.Response(
|
|
200,
|
|
content = sse_payload
|
|
or (b'event: message_stop\ndata: {"type":"message_stop"}\n\n'),
|
|
headers = {"content-type": "text/event-stream"},
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
ep_mod,
|
|
"_http_client",
|
|
httpx.AsyncClient(transport = httpx.MockTransport(handler)),
|
|
)
|
|
return captured
|
|
|
|
|
|
# ── Anthropic ──────────────────────────────────────────────────────────
|
|
|
|
|
|
def _drive_anthropic(captured, **kwargs) -> dict:
|
|
async def run():
|
|
client = ExternalProviderClient(
|
|
provider_type = "anthropic",
|
|
base_url = "https://api.anthropic.com/v1",
|
|
api_key = "sk-ant-test",
|
|
)
|
|
async for _ in client.stream_chat_completion(
|
|
messages = [{"role": "user", "content": "hi"}],
|
|
model = "claude-opus-4-7",
|
|
temperature = 0.7,
|
|
top_p = 0.95,
|
|
max_tokens = 64,
|
|
**kwargs,
|
|
):
|
|
pass
|
|
await client.close()
|
|
|
|
_drive(run())
|
|
return captured["body"]
|
|
|
|
|
|
def test_anthropic_stop_sequences_forwarded_as_renamed_field(monkeypatch):
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic(captured, stop = ["END", "DONE"])
|
|
assert body.get("stop_sequences") == ["END", "DONE"], body
|
|
# Anthropic does not have a `stop` field; the unrenamed key must not appear.
|
|
assert "stop" not in body, body
|
|
|
|
|
|
def test_anthropic_single_string_stop_is_wrapped(monkeypatch):
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic(captured, stop = "STOPHERE")
|
|
assert body.get("stop_sequences") == ["STOPHERE"], body
|
|
|
|
|
|
def test_anthropic_empty_stop_omitted(monkeypatch):
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic(captured, stop = [])
|
|
assert "stop_sequences" not in body, body
|
|
assert "stop" not in body, body
|
|
|
|
|
|
def test_anthropic_stop_sequences_dedup_and_drop_whitespace(monkeypatch):
|
|
"""Anthropic 400s on any stop sequence that contains no non-
|
|
whitespace character (`stop_sequences: each stop sequence must
|
|
contain non-whitespace`). Empty strings, " ", "\\n", "\\n\\n", and
|
|
other whitespace-only chips are filtered out client-side so the
|
|
request reaches the wire. Duplicates are deduped to avoid wasting
|
|
slots against the cap.
|
|
"""
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic(
|
|
captured,
|
|
stop = ["END", "", "END", "DONE", " ", "END", "\n\n", "\t"],
|
|
)
|
|
# Order preserved on first sight, duplicates + every whitespace-only
|
|
# entry dropped.
|
|
assert body.get("stop_sequences") == ["END", "DONE"], body
|
|
|
|
|
|
def test_anthropic_single_whitespace_stop_string_dropped(monkeypatch):
|
|
"""Single-string stop="\\n\\n" must not reach the wire either."""
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic(captured, stop = "\n\n")
|
|
assert "stop_sequences" not in body, body
|
|
|
|
|
|
def test_anthropic_stop_sequences_truncated_to_16(monkeypatch):
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic(captured, stop = [f"S{i}" for i in range(20)])
|
|
assert len(body.get("stop_sequences", [])) == 16, body
|
|
assert body["stop_sequences"][0] == "S0"
|
|
assert body["stop_sequences"][-1] == "S15"
|
|
|
|
|
|
def test_anthropic_service_tier_forwarded_when_valid(monkeypatch):
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic(captured, service_tier = "standard_only")
|
|
assert body.get("service_tier") == "standard_only", body
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"bogus", ["flex", "priority", "scale", "default", "", "auto-foo"]
|
|
)
|
|
def test_anthropic_service_tier_unsupported_values_dropped(monkeypatch, bogus):
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic(captured, service_tier = bogus)
|
|
assert "service_tier" not in body, body
|
|
|
|
|
|
def _drive_anthropic_with_tools(captured, **kwargs) -> dict:
|
|
"""Same as `_drive_anthropic` but enables a server-side tool
|
|
(`web_search`) so the request body carries `tools`. Needed to
|
|
exercise the `disable_parallel_tool_use` nesting path, which only
|
|
fires when there is at least one tool defined.
|
|
"""
|
|
enabled_tools = kwargs.pop("enabled_tools", None) or ["web_search"]
|
|
return _drive_anthropic(captured, enabled_tools = enabled_tools, **kwargs)
|
|
|
|
|
|
def test_anthropic_disable_parallel_tool_use_nested_under_tool_choice(monkeypatch):
|
|
"""`disable_parallel_tool_use` must be a property of `tool_choice`,
|
|
NOT a top-level body field. Top-level placement is rejected with
|
|
`extraneous key [disable_parallel_tool_use] is not permitted`. See
|
|
https://platform.claude.com/docs/en/agents-and-tools/tool-use/implement-tool-use.
|
|
"""
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic_with_tools(captured, parallel_tool_calls = False)
|
|
# Top-level fields must not carry the flag — Anthropic 400s otherwise.
|
|
assert "disable_parallel_tool_use" not in body, body
|
|
assert "parallel_tool_calls" not in body, body
|
|
# The flag is set on tool_choice. Default type is "auto" when the
|
|
# user didn't pick one explicitly.
|
|
tc = body.get("tool_choice")
|
|
assert isinstance(tc, dict), body
|
|
assert tc.get("disable_parallel_tool_use") is True, body
|
|
assert tc.get("type") == "auto", body
|
|
|
|
|
|
def test_anthropic_disable_parallel_tool_use_skipped_without_tools(monkeypatch):
|
|
"""Without any tools defined, `disable_parallel_tool_use` is a
|
|
no-op upstream — skip it so the request body stays minimal and the
|
|
flag never lands at top level either.
|
|
"""
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic(captured, parallel_tool_calls = False)
|
|
assert "disable_parallel_tool_use" not in body, body
|
|
assert "parallel_tool_calls" not in body, body
|
|
assert "tool_choice" not in body, body
|
|
|
|
|
|
def test_anthropic_parallel_tool_calls_default_not_sent(monkeypatch):
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic_with_tools(captured, parallel_tool_calls = True)
|
|
# True is the upstream default; do not surface a tool_choice we
|
|
# would otherwise not have set, and definitely no top-level
|
|
# `disable_parallel_tool_use`.
|
|
assert "disable_parallel_tool_use" not in body, body
|
|
assert "parallel_tool_calls" not in body, body
|
|
tc = body.get("tool_choice")
|
|
if isinstance(tc, dict):
|
|
assert "disable_parallel_tool_use" not in tc, body
|
|
|
|
|
|
def test_anthropic_rejects_openai_only_knobs(monkeypatch):
|
|
"""frequency_penalty / seed are dropped at the dispatch layer.
|
|
|
|
Anthropic has no equivalent; the keyword args are not even forwarded
|
|
from stream_chat_completion to _stream_anthropic. This test pins
|
|
that no such field reaches the Messages body.
|
|
"""
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic(
|
|
captured,
|
|
frequency_penalty = 1.5,
|
|
seed = 42,
|
|
)
|
|
assert "frequency_penalty" not in body, body
|
|
assert "seed" not in body, body
|
|
|
|
|
|
# ── OpenAI Chat Completions (default OAI-compat) ─────────────────────────
|
|
|
|
|
|
def _drive_openai_compat(captured, **kwargs) -> dict:
|
|
"""Send through the default OAI-compat branch (NOT /v1/responses).
|
|
|
|
Use a non-OpenAI provider_type so the dispatcher takes the default
|
|
branch at the bottom of stream_chat_completion rather than the
|
|
Responses translator path that routes provider_type=="openai".
|
|
"""
|
|
|
|
async def run():
|
|
client = ExternalProviderClient(
|
|
provider_type = "mistral",
|
|
base_url = "https://api.mistral.ai/v1",
|
|
api_key = "test-key",
|
|
)
|
|
# mistral's OpenAI-compat /v1/chat/completions returns OpenAI
|
|
# SSE; a single DONE frame is enough to drain the stream.
|
|
async for _ in client.stream_chat_completion(
|
|
messages = [{"role": "user", "content": "hi"}],
|
|
model = "mistral-small-latest",
|
|
temperature = 0.5,
|
|
top_p = 0.9,
|
|
max_tokens = 64,
|
|
**kwargs,
|
|
):
|
|
pass
|
|
await client.close()
|
|
|
|
_drive(run())
|
|
return captured["body"]
|
|
|
|
|
|
def _oai_done_payload() -> bytes:
|
|
return b"data: [DONE]\n\n"
|
|
|
|
|
|
def test_openai_compat_forwards_frequency_penalty(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
body = _drive_openai_compat(captured, frequency_penalty = 1.25)
|
|
assert body.get("frequency_penalty") == 1.25, body
|
|
|
|
|
|
def test_openai_compat_forwards_seed(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
body = _drive_openai_compat(captured, seed = 12345)
|
|
assert body.get("seed") == 12345, body
|
|
|
|
|
|
def test_openai_compat_forwards_stop_array(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
body = _drive_openai_compat(captured, stop = ["END", "DONE"])
|
|
assert body.get("stop") == ["END", "DONE"], body
|
|
# The default OAI-compat branch does not rename to stop_sequences.
|
|
assert "stop_sequences" not in body, body
|
|
|
|
|
|
def test_openai_compat_truncates_stop_to_four(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
body = _drive_openai_compat(captured, stop = ["a", "b", "c", "d", "e", "f"])
|
|
assert body.get("stop") == ["a", "b", "c", "d"], body
|
|
|
|
|
|
def test_openai_compat_stop_dedup_and_drop_empties(monkeypatch):
|
|
"""Duplicates and empties shouldn't eat into the 4-entry cap."""
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
body = _drive_openai_compat(captured, stop = ["END", "", "END", "DONE", "FIN", "END"])
|
|
assert body.get("stop") == ["END", "DONE", "FIN"], body
|
|
|
|
|
|
def test_openai_compat_empty_stop_omitted(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
body = _drive_openai_compat(captured, stop = [])
|
|
assert "stop" not in body, body
|
|
|
|
|
|
def test_openai_compat_forwards_service_tier(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
body = _drive_openai_compat(captured, service_tier = "flex")
|
|
assert body.get("service_tier") == "flex", body
|
|
|
|
|
|
def test_openai_compat_forwards_parallel_tool_calls(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
body = _drive_openai_compat(captured, parallel_tool_calls = False)
|
|
assert body.get("parallel_tool_calls") is False, body
|
|
|
|
|
|
def test_openai_compat_omits_unset_optionals(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
body = _drive_openai_compat(captured)
|
|
# Optional knobs default to None / unset -> never appear.
|
|
assert "frequency_penalty" not in body, body
|
|
assert "seed" not in body, body
|
|
assert "stop" not in body, body
|
|
assert "service_tier" not in body, body
|
|
assert "parallel_tool_calls" not in body, body
|
|
|
|
|
|
# ── OpenAI Responses (gpt-5.x via /v1/responses) ─────────────────────────
|
|
|
|
|
|
def _responses_done_payload() -> bytes:
|
|
return (
|
|
b"event: response.completed\n"
|
|
b'data: {"type":"response.completed","response":{"usage":{}}}\n\n'
|
|
)
|
|
|
|
|
|
def _drive_openai_responses(captured, **kwargs) -> dict:
|
|
async def run():
|
|
client = ExternalProviderClient(
|
|
provider_type = "openai",
|
|
base_url = "https://api.openai.com/v1",
|
|
api_key = "sk-test",
|
|
)
|
|
async for _ in client.stream_chat_completion(
|
|
messages = [{"role": "user", "content": "hi"}],
|
|
model = "gpt-5.5",
|
|
temperature = 1.0,
|
|
top_p = 1.0,
|
|
max_tokens = 64,
|
|
**kwargs,
|
|
):
|
|
pass
|
|
await client.close()
|
|
|
|
_drive(run())
|
|
return captured["body"]
|
|
|
|
|
|
def test_openai_responses_drops_temperature_top_p(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
|
body = _drive_openai_responses(captured)
|
|
assert "temperature" not in body, body
|
|
assert "top_p" not in body, body
|
|
|
|
|
|
def test_openai_responses_drops_frequency_penalty_seed_stop(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
|
body = _drive_openai_responses(
|
|
captured,
|
|
frequency_penalty = 1.5,
|
|
seed = 99,
|
|
stop = ["END"],
|
|
)
|
|
# Responses 400s on any of these; the dispatch must drop them
|
|
# before they hit the wire.
|
|
assert "frequency_penalty" not in body, body
|
|
assert "seed" not in body, body
|
|
assert "stop" not in body, body
|
|
assert "stop_sequences" not in body, body
|
|
|
|
|
|
def test_openai_responses_forwards_service_tier(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
|
body = _drive_openai_responses(captured, service_tier = "priority")
|
|
assert body.get("service_tier") == "priority", body
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"value", ["auto", "default", "flex", "scale", "priority"]
|
|
)
|
|
def test_openai_responses_accepts_full_service_tier_enum(monkeypatch, value):
|
|
"""`openai-python`'s ResponseCreateParams declares
|
|
`Optional[Literal["auto", "default", "flex", "scale", "priority"]]`
|
|
so every value in that set forwards untouched."""
|
|
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
|
body = _drive_openai_responses(captured, service_tier = value)
|
|
assert body.get("service_tier") == value, body
|
|
|
|
|
|
def test_openai_responses_drops_anthropic_only_service_tier(monkeypatch):
|
|
"""`standard_only` is Anthropic-only and Responses has never accepted
|
|
it. Drop it client-side so a stale frontend cannot 400 the request.
|
|
"""
|
|
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
|
body = _drive_openai_responses(captured, service_tier = "standard_only")
|
|
assert "service_tier" not in body, body
|
|
|
|
|
|
def test_openai_responses_forwards_parallel_tool_calls(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
|
body = _drive_openai_responses(captured, parallel_tool_calls = False)
|
|
assert body.get("parallel_tool_calls") is False, body
|
|
|
|
|
|
def test_openai_responses_omits_unset_optionals(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
|
body = _drive_openai_responses(captured)
|
|
assert "service_tier" not in body, body
|
|
assert "parallel_tool_calls" not in body, body
|
|
|
|
|
|
# ── Schema-level smoke tests ─────────────────────────────────────────────
|
|
|
|
|
|
def test_chat_completion_request_accepts_new_sampling_fields():
|
|
from models.inference import ChatCompletionRequest
|
|
|
|
payload = ChatCompletionRequest.model_validate(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"frequency_penalty": -1.0,
|
|
"seed": 0,
|
|
"stop": ["END"],
|
|
"service_tier": "auto",
|
|
"parallel_tool_calls": True,
|
|
}
|
|
)
|
|
assert payload.frequency_penalty == -1.0
|
|
assert payload.seed == 0
|
|
assert payload.stop == ["END"]
|
|
assert payload.service_tier == "auto"
|
|
assert payload.parallel_tool_calls is True
|
|
|
|
|
|
def test_chat_completion_request_rejects_bad_service_tier():
|
|
import pydantic
|
|
from models.inference import ChatCompletionRequest
|
|
|
|
with pytest.raises(pydantic.ValidationError):
|
|
ChatCompletionRequest.model_validate(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"service_tier": "bogus",
|
|
}
|
|
)
|
|
|
|
|
|
def test_chat_completion_request_clamps_frequency_penalty_range():
|
|
import pydantic
|
|
from models.inference import ChatCompletionRequest
|
|
|
|
with pytest.raises(pydantic.ValidationError):
|
|
ChatCompletionRequest.model_validate(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"frequency_penalty": 3.0,
|
|
}
|
|
)
|
|
with pytest.raises(pydantic.ValidationError):
|
|
ChatCompletionRequest.model_validate(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"frequency_penalty": -3.0,
|
|
}
|
|
)
|
|
|
|
|
|
# ── Kimi web-search bypass forwards new sampling fields ────────────────
|
|
|
|
|
|
def test_kimi_web_search_bypass_forwards_new_sampling_fields(monkeypatch):
|
|
"""The Kimi `enabled_tools=["web_search"]` path takes an early
|
|
return into `_stream_kimi_web_search` BEFORE the default OAI-compat
|
|
body builder runs. PR #5711 added new sampling fields to the
|
|
default builder; this test pins that the web-search bypass also
|
|
forwards them so Kimi-with-search and Kimi-without-search behave
|
|
consistently."""
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
|
|
async def run():
|
|
client = ExternalProviderClient(
|
|
provider_type = "kimi",
|
|
base_url = "https://api.moonshot.ai/v1",
|
|
api_key = "kimi-test",
|
|
)
|
|
async for _ in client.stream_chat_completion(
|
|
messages = [{"role": "user", "content": "hi"}],
|
|
model = "kimi-k2.6",
|
|
temperature = 1.0,
|
|
top_p = 1.0,
|
|
max_tokens = 256,
|
|
enabled_tools = ["web_search"],
|
|
presence_penalty = 0.5,
|
|
frequency_penalty = 1.25,
|
|
seed = 7,
|
|
stop = ["END"],
|
|
parallel_tool_calls = False,
|
|
):
|
|
pass
|
|
await client.close()
|
|
|
|
_drive(run())
|
|
body = captured["body"]
|
|
assert body.get("frequency_penalty") == 1.25, body
|
|
assert body.get("seed") == 7, body
|
|
assert body.get("stop") == ["END"], body
|
|
assert body.get("parallel_tool_calls") is False, body
|
|
assert body.get("presence_penalty") == 0.5, body
|
|
# body_omit still strips temperature / top_p for Kimi.
|
|
assert "temperature" not in body, body
|
|
assert "top_p" not in body, body
|
|
|
|
|
|
# ── Local OpenAI passthrough forwards new sampling fields ──────────────
|
|
|
|
|
|
def test_local_openai_passthrough_forwards_new_sampling_fields():
|
|
"""Round 1 reviewers (10/20) flagged that
|
|
`_build_openai_passthrough_body` dropped frequency_penalty / seed /
|
|
parallel_tool_calls when forwarding to llama-server. Pin the
|
|
extended contract."""
|
|
from models.inference import ChatCompletionRequest
|
|
from routes.inference import _build_openai_passthrough_body
|
|
|
|
payload = ChatCompletionRequest.model_validate(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"stream": True,
|
|
"frequency_penalty": 1.25,
|
|
"seed": 123,
|
|
"stop": ["END"],
|
|
"parallel_tool_calls": False,
|
|
}
|
|
)
|
|
body = _build_openai_passthrough_body(payload, backend_ctx = 4096)
|
|
assert body["frequency_penalty"] == 1.25, body
|
|
assert body["seed"] == 123, body
|
|
assert body["stop"] == ["END"], body
|
|
assert body["parallel_tool_calls"] is False, body
|
|
|
|
|
|
# ── Backend ChatInferenceSettings schema accepts new fields ────────────
|
|
|
|
|
|
def test_chat_settings_payload_accepts_new_sampling_keys():
|
|
"""Round 1 reviewers flagged that `ChatSettingsPayload.extra="forbid"`
|
|
with the old field list 422'd every settings save that contained
|
|
any of the new keys. Pin that the new keys round-trip."""
|
|
from routes.chat_history import ChatSettingsPayload
|
|
|
|
parsed = ChatSettingsPayload.model_validate(
|
|
{
|
|
"inferenceParams": {
|
|
"frequencyPenalty": 0.7,
|
|
"seed": 42,
|
|
"stop": ["END"],
|
|
"serviceTier": "standard_only",
|
|
"parallelToolCalls": False,
|
|
}
|
|
}
|
|
)
|
|
ip = parsed.inferenceParams
|
|
assert ip is not None
|
|
assert ip.frequencyPenalty == 0.7
|
|
assert ip.seed == 42
|
|
assert ip.stop == ["END"]
|
|
assert ip.serviceTier == "standard_only"
|
|
assert ip.parallelToolCalls is False
|