b48d68f8bf
Mistral chat completions uses random_seed not seed; map the field via a new seed_field on the provider registry so the new seed control actually works on Mistral. Default for other providers stays seed. DeepSeek and Mistral both accept up to 16 stop sequences but the default OAI-compat branch was hard-capping at 4 (the OpenAI Chat limit). Studio routes the openai provider through /v1/responses not /v1/chat/completions so the 4-cap only applies if we explicitly added an openai entry. Raise the default to 16 and let per-provider stop_max overrides tighten if needed. The local GGUF direct chat path (gguf_generate / gguf_generate_with_tools) bypassed _build_openai_passthrough_body and therefore dropped frequency_penalty, seed, stop, and parallel_tool_calls on the floor for users on the default no-tools and with-tools paths. Thread the new fields through LlamaCppBackend.generate_chat_completion and generate_chat_completion_with_tools and the two callsites that invoke them. Also tighten comments to drop review-process narration that crept in and to remove the em dashes I had introduced in this PR's earlier commits. Tests pin the Mistral random_seed rename, the DeepSeek 16-cap, and confirm the openai-compat default cap is 16.
677 lines
25 KiB
Python
677 lines
25 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-only
|
|
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
|
|
|
"""End-to-end routing tests for the new sampling parameters.
|
|
|
|
Pins the per-provider gating contract added by the
|
|
expose-sampling-params PR: each of `frequency_penalty`, `seed`, `stop`
|
|
/ `stop_sequences`, `service_tier`, `parallel_tool_calls` only appears
|
|
on the outbound body when the upstream provider actually accepts it.
|
|
|
|
The provider matrix is captured per docs:
|
|
- Anthropic Messages: accepts stop_sequences, service_tier
|
|
(auto|standard_only), disable_parallel_tool_use (inverted). REJECTS
|
|
frequency_penalty, seed, logprobs (silently dropped client-side).
|
|
- OpenAI Chat Completions (default OAI-compat branch): accepts every
|
|
field; OpenAI cloud uses `max_completion_tokens` rather than
|
|
`max_tokens`.
|
|
- OpenAI Responses (gpt-5.x / o3): rejects temperature, top_p,
|
|
frequency_penalty, seed, stop, logprobs. Accepts service_tier
|
|
(auto|default|flex|priority) and parallel_tool_calls.
|
|
"""
|
|
|
|
import asyncio
|
|
import json
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from core.inference import external_provider as ep_mod
|
|
from core.inference.external_provider import ExternalProviderClient
|
|
|
|
|
|
def _drive(coro):
|
|
return asyncio.new_event_loop().run_until_complete(coro)
|
|
|
|
|
|
def _install_mock(monkeypatch, *, sse_payload: bytes | None = None) -> dict:
|
|
captured: dict = {}
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
try:
|
|
captured["body"] = json.loads(request.content.decode("utf-8"))
|
|
except json.JSONDecodeError:
|
|
captured["body"] = None
|
|
captured["url"] = str(request.url)
|
|
captured["headers"] = dict(request.headers)
|
|
return httpx.Response(
|
|
200,
|
|
content = sse_payload
|
|
or (b'event: message_stop\ndata: {"type":"message_stop"}\n\n'),
|
|
headers = {"content-type": "text/event-stream"},
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
ep_mod,
|
|
"_http_client",
|
|
httpx.AsyncClient(transport = httpx.MockTransport(handler)),
|
|
)
|
|
return captured
|
|
|
|
|
|
# ── Anthropic ──────────────────────────────────────────────────────────
|
|
|
|
|
|
def _drive_anthropic(captured, **kwargs) -> dict:
|
|
async def run():
|
|
client = ExternalProviderClient(
|
|
provider_type = "anthropic",
|
|
base_url = "https://api.anthropic.com/v1",
|
|
api_key = "sk-ant-test",
|
|
)
|
|
async for _ in client.stream_chat_completion(
|
|
messages = [{"role": "user", "content": "hi"}],
|
|
model = "claude-opus-4-7",
|
|
temperature = 0.7,
|
|
top_p = 0.95,
|
|
max_tokens = 64,
|
|
**kwargs,
|
|
):
|
|
pass
|
|
await client.close()
|
|
|
|
_drive(run())
|
|
return captured["body"]
|
|
|
|
|
|
def test_anthropic_stop_sequences_forwarded_as_renamed_field(monkeypatch):
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic(captured, stop = ["END", "DONE"])
|
|
assert body.get("stop_sequences") == ["END", "DONE"], body
|
|
# Anthropic does not have a `stop` field; the unrenamed key must not appear.
|
|
assert "stop" not in body, body
|
|
|
|
|
|
def test_anthropic_single_string_stop_is_wrapped(monkeypatch):
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic(captured, stop = "STOPHERE")
|
|
assert body.get("stop_sequences") == ["STOPHERE"], body
|
|
|
|
|
|
def test_anthropic_empty_stop_omitted(monkeypatch):
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic(captured, stop = [])
|
|
assert "stop_sequences" not in body, body
|
|
assert "stop" not in body, body
|
|
|
|
|
|
def test_anthropic_stop_sequences_dedup_and_drop_whitespace(monkeypatch):
|
|
"""Anthropic 400s on any stop sequence that contains no non-
|
|
whitespace character (`stop_sequences: each stop sequence must
|
|
contain non-whitespace`). Empty strings, " ", "\\n", "\\n\\n", and
|
|
other whitespace-only chips are filtered out client-side so the
|
|
request reaches the wire. Duplicates are deduped to avoid wasting
|
|
slots against the cap.
|
|
"""
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic(
|
|
captured,
|
|
stop = ["END", "", "END", "DONE", " ", "END", "\n\n", "\t"],
|
|
)
|
|
# Order preserved on first sight, duplicates + every whitespace-only
|
|
# entry dropped.
|
|
assert body.get("stop_sequences") == ["END", "DONE"], body
|
|
|
|
|
|
def test_anthropic_single_whitespace_stop_string_dropped(monkeypatch):
|
|
"""Single-string stop="\\n\\n" must not reach the wire either."""
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic(captured, stop = "\n\n")
|
|
assert "stop_sequences" not in body, body
|
|
|
|
|
|
def test_anthropic_stop_sequences_truncated_to_16(monkeypatch):
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic(captured, stop = [f"S{i}" for i in range(20)])
|
|
assert len(body.get("stop_sequences", [])) == 16, body
|
|
assert body["stop_sequences"][0] == "S0"
|
|
assert body["stop_sequences"][-1] == "S15"
|
|
|
|
|
|
def test_anthropic_service_tier_forwarded_when_valid(monkeypatch):
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic(captured, service_tier = "standard_only")
|
|
assert body.get("service_tier") == "standard_only", body
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"bogus", ["flex", "priority", "scale", "default", "", "auto-foo"]
|
|
)
|
|
def test_anthropic_service_tier_unsupported_values_dropped(monkeypatch, bogus):
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic(captured, service_tier = bogus)
|
|
assert "service_tier" not in body, body
|
|
|
|
|
|
def _drive_anthropic_with_tools(captured, **kwargs) -> dict:
|
|
"""Same as `_drive_anthropic` but enables a server-side tool
|
|
(`web_search`) so the request body carries `tools`. Needed to
|
|
exercise the `disable_parallel_tool_use` nesting path, which only
|
|
fires when there is at least one tool defined.
|
|
"""
|
|
enabled_tools = kwargs.pop("enabled_tools", None) or ["web_search"]
|
|
return _drive_anthropic(captured, enabled_tools = enabled_tools, **kwargs)
|
|
|
|
|
|
def test_anthropic_disable_parallel_tool_use_nested_under_tool_choice(monkeypatch):
|
|
"""`disable_parallel_tool_use` must be a property of `tool_choice`,
|
|
NOT a top-level body field. Top-level placement is rejected with
|
|
`extraneous key [disable_parallel_tool_use] is not permitted`. See
|
|
https://platform.claude.com/docs/en/agents-and-tools/tool-use/implement-tool-use.
|
|
"""
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic_with_tools(captured, parallel_tool_calls = False)
|
|
# Top-level placement is rejected with 400.
|
|
assert "disable_parallel_tool_use" not in body, body
|
|
assert "parallel_tool_calls" not in body, body
|
|
# Flag lives on tool_choice; default type is "auto".
|
|
tc = body.get("tool_choice")
|
|
assert isinstance(tc, dict), body
|
|
assert tc.get("disable_parallel_tool_use") is True, body
|
|
assert tc.get("type") == "auto", body
|
|
|
|
|
|
def test_anthropic_disable_parallel_tool_use_skipped_without_tools(monkeypatch):
|
|
"""Without tools the flag is a no-op upstream; keep the body
|
|
minimal and never emit it at top level either.
|
|
"""
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic(captured, parallel_tool_calls = False)
|
|
assert "disable_parallel_tool_use" not in body, body
|
|
assert "parallel_tool_calls" not in body, body
|
|
assert "tool_choice" not in body, body
|
|
|
|
|
|
def test_anthropic_parallel_tool_calls_default_not_sent(monkeypatch):
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic_with_tools(captured, parallel_tool_calls = True)
|
|
# True is the upstream default; do not surface a tool_choice we
|
|
# would otherwise not have set, and definitely no top-level
|
|
# `disable_parallel_tool_use`.
|
|
assert "disable_parallel_tool_use" not in body, body
|
|
assert "parallel_tool_calls" not in body, body
|
|
tc = body.get("tool_choice")
|
|
if isinstance(tc, dict):
|
|
assert "disable_parallel_tool_use" not in tc, body
|
|
|
|
|
|
def test_anthropic_rejects_openai_only_knobs(monkeypatch):
|
|
"""frequency_penalty / seed are dropped at the dispatch layer.
|
|
|
|
Anthropic has no equivalent; the keyword args are not even forwarded
|
|
from stream_chat_completion to _stream_anthropic. This test pins
|
|
that no such field reaches the Messages body.
|
|
"""
|
|
captured = _install_mock(monkeypatch)
|
|
body = _drive_anthropic(
|
|
captured,
|
|
frequency_penalty = 1.5,
|
|
seed = 42,
|
|
)
|
|
assert "frequency_penalty" not in body, body
|
|
assert "seed" not in body, body
|
|
|
|
|
|
# ── OpenAI Chat Completions (default OAI-compat) ─────────────────────────
|
|
|
|
|
|
def _drive_openai_compat(captured, **kwargs) -> dict:
|
|
"""Send through the default OAI-compat branch (NOT /v1/responses).
|
|
|
|
Use a non-OpenAI provider_type so the dispatcher takes the default
|
|
branch at the bottom of stream_chat_completion rather than the
|
|
Responses translator path that routes provider_type=="openai".
|
|
"""
|
|
|
|
async def run():
|
|
client = ExternalProviderClient(
|
|
provider_type = "mistral",
|
|
base_url = "https://api.mistral.ai/v1",
|
|
api_key = "test-key",
|
|
)
|
|
# mistral's OpenAI-compat /v1/chat/completions returns OpenAI
|
|
# SSE; a single DONE frame is enough to drain the stream.
|
|
async for _ in client.stream_chat_completion(
|
|
messages = [{"role": "user", "content": "hi"}],
|
|
model = "mistral-small-latest",
|
|
temperature = 0.5,
|
|
top_p = 0.9,
|
|
max_tokens = 64,
|
|
**kwargs,
|
|
):
|
|
pass
|
|
await client.close()
|
|
|
|
_drive(run())
|
|
return captured["body"]
|
|
|
|
|
|
def _oai_done_payload() -> bytes:
|
|
return b"data: [DONE]\n\n"
|
|
|
|
|
|
def test_openai_compat_forwards_frequency_penalty(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
body = _drive_openai_compat(captured, frequency_penalty = 1.25)
|
|
assert body.get("frequency_penalty") == 1.25, body
|
|
|
|
|
|
def test_openai_compat_forwards_seed(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
body = _drive_openai_compat(captured, seed = 12345)
|
|
# Default OAI-compat provider (mistral here) renames seed to
|
|
# random_seed via provider registry's seed_field.
|
|
assert body.get("random_seed") == 12345, body
|
|
assert "seed" not in body, body
|
|
|
|
|
|
def test_openai_compat_seed_field_default_is_seed(monkeypatch):
|
|
"""Providers without a seed_field override get the OpenAI default."""
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
|
|
async def run():
|
|
client = ExternalProviderClient(
|
|
provider_type = "deepseek",
|
|
base_url = "https://api.deepseek.com/v1",
|
|
api_key = "ds-test",
|
|
)
|
|
async for _ in client.stream_chat_completion(
|
|
messages = [{"role": "user", "content": "hi"}],
|
|
model = "deepseek-chat",
|
|
temperature = 0.5,
|
|
top_p = 0.9,
|
|
max_tokens = 64,
|
|
seed = 7,
|
|
):
|
|
pass
|
|
await client.close()
|
|
|
|
_drive(run())
|
|
body = captured["body"]
|
|
assert body.get("seed") == 7, body
|
|
assert "random_seed" not in body, body
|
|
|
|
|
|
def test_openai_compat_deepseek_stop_cap_is_16(monkeypatch):
|
|
"""DeepSeek docs allow up to 16 stop sequences; the previous
|
|
4-cap silently truncated valid configs."""
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
|
|
async def run():
|
|
client = ExternalProviderClient(
|
|
provider_type = "deepseek",
|
|
base_url = "https://api.deepseek.com/v1",
|
|
api_key = "ds-test",
|
|
)
|
|
async for _ in client.stream_chat_completion(
|
|
messages = [{"role": "user", "content": "hi"}],
|
|
model = "deepseek-chat",
|
|
temperature = 0.5,
|
|
top_p = 0.9,
|
|
max_tokens = 64,
|
|
stop = [f"S{i}" for i in range(20)],
|
|
):
|
|
pass
|
|
await client.close()
|
|
|
|
_drive(run())
|
|
body = captured["body"]
|
|
assert len(body.get("stop", [])) == 16, body
|
|
|
|
|
|
def test_openai_compat_forwards_stop_array(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
body = _drive_openai_compat(captured, stop = ["END", "DONE"])
|
|
assert body.get("stop") == ["END", "DONE"], body
|
|
# The default OAI-compat branch does not rename to stop_sequences.
|
|
assert "stop_sequences" not in body, body
|
|
|
|
|
|
def test_openai_compat_truncates_stop_to_default_cap(monkeypatch):
|
|
"""Default OAI-compat cap is 16 (DeepSeek and Mistral both accept
|
|
that many); only OpenAI Chat has a tighter 4-entry hard limit."""
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
body = _drive_openai_compat(captured, stop = [f"s{i}" for i in range(20)])
|
|
assert len(body.get("stop", [])) == 16, body
|
|
assert body["stop"][0] == "s0"
|
|
assert body["stop"][-1] == "s15"
|
|
|
|
|
|
def test_openai_compat_stop_dedup_and_drop_empties(monkeypatch):
|
|
"""Duplicates and empties shouldn't eat into the cap."""
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
body = _drive_openai_compat(captured, stop = ["END", "", "END", "DONE", "FIN", "END"])
|
|
assert body.get("stop") == ["END", "DONE", "FIN"], body
|
|
|
|
|
|
def test_openai_compat_empty_stop_omitted(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
body = _drive_openai_compat(captured, stop = [])
|
|
assert "stop" not in body, body
|
|
|
|
|
|
def test_openai_compat_forwards_service_tier(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
body = _drive_openai_compat(captured, service_tier = "flex")
|
|
assert body.get("service_tier") == "flex", body
|
|
|
|
|
|
def test_openai_compat_forwards_parallel_tool_calls(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
body = _drive_openai_compat(captured, parallel_tool_calls = False)
|
|
assert body.get("parallel_tool_calls") is False, body
|
|
|
|
|
|
def test_openai_compat_omits_unset_optionals(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
body = _drive_openai_compat(captured)
|
|
# Optional knobs default to None / unset -> never appear.
|
|
assert "frequency_penalty" not in body, body
|
|
assert "seed" not in body, body
|
|
assert "stop" not in body, body
|
|
assert "service_tier" not in body, body
|
|
assert "parallel_tool_calls" not in body, body
|
|
|
|
|
|
# ── OpenAI Responses (gpt-5.x via /v1/responses) ─────────────────────────
|
|
|
|
|
|
def _responses_done_payload() -> bytes:
|
|
return (
|
|
b"event: response.completed\n"
|
|
b'data: {"type":"response.completed","response":{"usage":{}}}\n\n'
|
|
)
|
|
|
|
|
|
def _drive_openai_responses(captured, **kwargs) -> dict:
|
|
async def run():
|
|
client = ExternalProviderClient(
|
|
provider_type = "openai",
|
|
base_url = "https://api.openai.com/v1",
|
|
api_key = "sk-test",
|
|
)
|
|
async for _ in client.stream_chat_completion(
|
|
messages = [{"role": "user", "content": "hi"}],
|
|
model = "gpt-5.5",
|
|
temperature = 1.0,
|
|
top_p = 1.0,
|
|
max_tokens = 64,
|
|
**kwargs,
|
|
):
|
|
pass
|
|
await client.close()
|
|
|
|
_drive(run())
|
|
return captured["body"]
|
|
|
|
|
|
def test_openai_responses_drops_temperature_top_p(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
|
body = _drive_openai_responses(captured)
|
|
assert "temperature" not in body, body
|
|
assert "top_p" not in body, body
|
|
|
|
|
|
def test_openai_responses_drops_frequency_penalty_seed_stop(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
|
body = _drive_openai_responses(
|
|
captured,
|
|
frequency_penalty = 1.5,
|
|
seed = 99,
|
|
stop = ["END"],
|
|
)
|
|
# Responses 400s on any of these; the dispatch must drop them
|
|
# before they hit the wire.
|
|
assert "frequency_penalty" not in body, body
|
|
assert "seed" not in body, body
|
|
assert "stop" not in body, body
|
|
assert "stop_sequences" not in body, body
|
|
|
|
|
|
def test_openai_responses_forwards_service_tier(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
|
body = _drive_openai_responses(captured, service_tier = "priority")
|
|
assert body.get("service_tier") == "priority", body
|
|
|
|
|
|
@pytest.mark.parametrize("value", ["auto", "default", "flex", "priority"])
|
|
def test_openai_responses_forwards_documented_service_tiers(monkeypatch, value):
|
|
"""The live OpenAI Responses API reference lists `service_tier` as
|
|
`auto|default|flex|priority` for /v1/responses. Pin that every value
|
|
in the documented enum forwards untouched."""
|
|
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
|
body = _drive_openai_responses(captured, service_tier = value)
|
|
assert body.get("service_tier") == value, body
|
|
|
|
|
|
@pytest.mark.parametrize("bogus", ["scale", "standard_only", "bogus", ""])
|
|
def test_openai_responses_drops_undocumented_service_tier(monkeypatch, bogus):
|
|
"""`scale` and `standard_only` are not in the documented Responses
|
|
request enum; drop them client-side so a stale frontend never
|
|
sends an upstream-rejected value."""
|
|
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
|
body = _drive_openai_responses(captured, service_tier = bogus)
|
|
assert "service_tier" not in body, body
|
|
|
|
|
|
def test_openai_responses_forwards_parallel_tool_calls(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
|
body = _drive_openai_responses(captured, parallel_tool_calls = False)
|
|
assert body.get("parallel_tool_calls") is False, body
|
|
|
|
|
|
def test_openai_responses_omits_unset_optionals(monkeypatch):
|
|
captured = _install_mock(monkeypatch, sse_payload = _responses_done_payload())
|
|
body = _drive_openai_responses(captured)
|
|
assert "service_tier" not in body, body
|
|
assert "parallel_tool_calls" not in body, body
|
|
|
|
|
|
# ── Schema-level smoke tests ─────────────────────────────────────────────
|
|
|
|
|
|
def test_chat_completion_request_accepts_new_sampling_fields():
|
|
from models.inference import ChatCompletionRequest
|
|
|
|
payload = ChatCompletionRequest.model_validate(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"frequency_penalty": -1.0,
|
|
"seed": 0,
|
|
"stop": ["END"],
|
|
"service_tier": "auto",
|
|
"parallel_tool_calls": True,
|
|
}
|
|
)
|
|
assert payload.frequency_penalty == -1.0
|
|
assert payload.seed == 0
|
|
assert payload.stop == ["END"]
|
|
assert payload.service_tier == "auto"
|
|
assert payload.parallel_tool_calls is True
|
|
|
|
|
|
def test_chat_completion_request_rejects_bad_service_tier():
|
|
import pydantic
|
|
from models.inference import ChatCompletionRequest
|
|
|
|
with pytest.raises(pydantic.ValidationError):
|
|
ChatCompletionRequest.model_validate(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"service_tier": "bogus",
|
|
}
|
|
)
|
|
|
|
|
|
def test_chat_completion_request_clamps_frequency_penalty_range():
|
|
import pydantic
|
|
from models.inference import ChatCompletionRequest
|
|
|
|
with pytest.raises(pydantic.ValidationError):
|
|
ChatCompletionRequest.model_validate(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"frequency_penalty": 3.0,
|
|
}
|
|
)
|
|
with pytest.raises(pydantic.ValidationError):
|
|
ChatCompletionRequest.model_validate(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"frequency_penalty": -3.0,
|
|
}
|
|
)
|
|
|
|
|
|
# ── Kimi web-search bypass forwards new sampling fields ────────────────
|
|
|
|
|
|
def test_kimi_web_search_bypass_forwards_new_sampling_fields(monkeypatch):
|
|
"""The Kimi `enabled_tools=["web_search"]` path takes an early
|
|
return into `_stream_kimi_web_search` BEFORE the default OAI-compat
|
|
body builder runs. PR #5711 added new sampling fields to the
|
|
default builder; this test pins that the web-search bypass also
|
|
forwards them so Kimi-with-search and Kimi-without-search behave
|
|
consistently."""
|
|
captured = _install_mock(monkeypatch, sse_payload = _oai_done_payload())
|
|
|
|
async def run():
|
|
client = ExternalProviderClient(
|
|
provider_type = "kimi",
|
|
base_url = "https://api.moonshot.ai/v1",
|
|
api_key = "kimi-test",
|
|
)
|
|
async for _ in client.stream_chat_completion(
|
|
messages = [{"role": "user", "content": "hi"}],
|
|
model = "kimi-k2.6",
|
|
temperature = 1.0,
|
|
top_p = 1.0,
|
|
max_tokens = 256,
|
|
enabled_tools = ["web_search"],
|
|
presence_penalty = 0.5,
|
|
frequency_penalty = 1.25,
|
|
seed = 7,
|
|
stop = ["END"],
|
|
parallel_tool_calls = False,
|
|
):
|
|
pass
|
|
await client.close()
|
|
|
|
_drive(run())
|
|
body = captured["body"]
|
|
assert body.get("frequency_penalty") == 1.25, body
|
|
assert body.get("seed") == 7, body
|
|
assert body.get("stop") == ["END"], body
|
|
assert body.get("parallel_tool_calls") is False, body
|
|
assert body.get("presence_penalty") == 0.5, body
|
|
# body_omit still strips temperature / top_p for Kimi.
|
|
assert "temperature" not in body, body
|
|
assert "top_p" not in body, body
|
|
|
|
|
|
# ── Local OpenAI passthrough forwards new sampling fields ──────────────
|
|
|
|
|
|
def test_local_openai_passthrough_forwards_new_sampling_fields():
|
|
"""`_build_openai_passthrough_body` forwards frequency_penalty,
|
|
seed, stop, and parallel_tool_calls to llama-server."""
|
|
from models.inference import ChatCompletionRequest
|
|
from routes.inference import _build_openai_passthrough_body
|
|
|
|
payload = ChatCompletionRequest.model_validate(
|
|
{
|
|
"messages": [{"role": "user", "content": "hi"}],
|
|
"stream": True,
|
|
"frequency_penalty": 1.25,
|
|
"seed": 123,
|
|
"stop": ["END"],
|
|
"parallel_tool_calls": False,
|
|
}
|
|
)
|
|
body = _build_openai_passthrough_body(payload, backend_ctx = 4096)
|
|
assert body["frequency_penalty"] == 1.25, body
|
|
assert body["seed"] == 123, body
|
|
assert body["stop"] == ["END"], body
|
|
assert body["parallel_tool_calls"] is False, body
|
|
|
|
|
|
# ── Responses → ChatCompletions bridge preserves parallel_tool_calls ──
|
|
|
|
|
|
def test_responses_to_chat_bridge_preserves_parallel_tool_calls():
|
|
"""`_build_chat_request` (the /v1/responses to /v1/chat/completions
|
|
translator) must forward parallel_tool_calls so a Responses-API
|
|
caller's preference reaches llama-server."""
|
|
from models.inference import ChatMessage, ResponsesRequest
|
|
from routes.inference import _build_chat_request, _build_openai_passthrough_body
|
|
|
|
payload = ResponsesRequest(
|
|
input = "hi",
|
|
stream = True,
|
|
parallel_tool_calls = False,
|
|
)
|
|
chat_req = _build_chat_request(
|
|
payload,
|
|
[ChatMessage(role = "user", content = "hi")],
|
|
stream = True,
|
|
)
|
|
assert chat_req.parallel_tool_calls is False, chat_req
|
|
body = _build_openai_passthrough_body(chat_req, backend_ctx = 4096)
|
|
assert body["parallel_tool_calls"] is False, body
|
|
|
|
|
|
def test_responses_to_chat_bridge_omits_unset_parallel_tool_calls():
|
|
"""Unset parallel_tool_calls (None) must not appear on the
|
|
translated body; the upstream default is true everywhere so
|
|
forwarding None would over-specify."""
|
|
from models.inference import ChatMessage, ResponsesRequest
|
|
from routes.inference import _build_chat_request, _build_openai_passthrough_body
|
|
|
|
payload = ResponsesRequest(input = "hi", stream = True)
|
|
chat_req = _build_chat_request(
|
|
payload,
|
|
[ChatMessage(role = "user", content = "hi")],
|
|
stream = True,
|
|
)
|
|
assert chat_req.parallel_tool_calls is None, chat_req
|
|
body = _build_openai_passthrough_body(chat_req, backend_ctx = 4096)
|
|
assert "parallel_tool_calls" not in body, body
|
|
|
|
|
|
# ── Backend ChatInferenceSettings schema accepts new fields ────────────
|
|
|
|
|
|
def test_chat_settings_payload_accepts_new_sampling_keys():
|
|
"""ChatSettingsPayload has extra="forbid" so the new keys must be
|
|
listed explicitly; otherwise every settings save with any of them
|
|
422s. Pin the round-trip."""
|
|
from routes.chat_history import ChatSettingsPayload
|
|
|
|
parsed = ChatSettingsPayload.model_validate(
|
|
{
|
|
"inferenceParams": {
|
|
"frequencyPenalty": 0.7,
|
|
"seed": 42,
|
|
"stop": ["END"],
|
|
"serviceTier": "standard_only",
|
|
"parallelToolCalls": False,
|
|
}
|
|
}
|
|
)
|
|
ip = parsed.inferenceParams
|
|
assert ip is not None
|
|
assert ip.frequencyPenalty == 0.7
|
|
assert ip.seed == 42
|
|
assert ip.stop == ["END"]
|
|
assert ip.serviceTier == "standard_only"
|
|
assert ip.parallelToolCalls is False
|