diff --git a/studio/backend/assets/configs/inference_defaults.json b/studio/backend/assets/configs/inference_defaults.json index 1b10b557e4..c42b53b876 100644 --- a/studio/backend/assets/configs/inference_defaults.json +++ b/studio/backend/assets/configs/inference_defaults.json @@ -215,14 +215,14 @@ "repetition_penalty": 1.0 }, "ministral": { - "temperature": 0.15, + "temperature": 0.05, "top_p": 0.95, "top_k": -1, "min_p": 0.01, "repetition_penalty": 1.0 }, "devstral": { - "temperature": 0.7, + "temperature": 0.15, "top_p": 0.95, "top_k": -1, "min_p": 0.01, @@ -275,7 +275,10 @@ "top_p": 1.0, "top_k": -1, "min_p": 0.01, - "repetition_penalty": 1.0 + "repetition_penalty": 1.0, + "chat_template_kwargs": { + "enable_thinking": true + } }, "minimax-m2.5": { "temperature": 1.0, @@ -296,7 +299,10 @@ "top_p": 1.0, "top_k": 0, "min_p": 0.01, - "repetition_penalty": 1.0 + "repetition_penalty": 1.0, + "chat_template_kwargs": { + "reasoning_effort": "medium" + } }, "granite-4": { "temperature": 0.0, diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 02270ab405..c61479034c 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -3948,7 +3948,9 @@ async def _responses_stream( ) body = _build_openai_passthrough_body( - chat_req, backend_ctx = llama_backend.context_length + chat_req, + backend_ctx = llama_backend.context_length, + model_identifier = llama_backend.model_identifier, ) target_url = f"{llama_backend.base_url}/v1/chat/completions" @@ -5315,21 +5317,54 @@ def _extract_response_format(payload): return rf if isinstance(rf, dict) else None -def _build_openai_passthrough_body(payload, backend_ctx = None) -> dict: +def _build_openai_passthrough_body( + payload, + backend_ctx = None, + model_identifier: str | None = None, +) -> dict: """Assemble the llama-server request body from a ChatCompletionRequest. Only explicitly-known OpenAI / llama-server fields are forwarded so that Studio-specific extensions (``enable_tools``, ``enabled_tools``, ``session_id``, ...) never leak to the backend. + + ``model_identifier`` (when supplied) gates a registry lookup so the + family default ``chat_template_kwargs`` (e.g. gpt-oss + ``reasoning_effort=medium``) reaches llama-server even when the + inbound request did not set the keys explicitly. Per-request keys + still win on conflict. """ messages = _openai_messages_for_passthrough(payload) tool_choice = payload.tool_choice if payload.tool_choice is not None else "auto" - # When the caller asked for a specific reasoning mode, forward it to - # llama-server via chat_template_kwargs so the Jinja template renders - # with (or without) the reasoning preamble. - tpl_kwargs = None + # Merge family-default chat_template_kwargs (base) with per-request + # overrides. Per-request always wins. + tpl_kwargs: dict = {} + if model_identifier: + try: + family_defaults = load_inference_config(model_identifier) + family_kw = family_defaults.get("chat_template_kwargs") + if isinstance(family_kw, dict): + tpl_kwargs.update(family_kw) + except Exception as exc: + logger.warning( + "openai_passthrough.family_defaults_lookup_failed model=%s err=%s", + model_identifier, + exc, + ) + # Per-request enable_thinking already lifted from extra_body upstream. if payload.enable_thinking is not None: - tpl_kwargs = {"enable_thinking": bool(payload.enable_thinking)} + tpl_kwargs["enable_thinking"] = bool(payload.enable_thinking) + if payload.reasoning_effort is not None: + tpl_kwargs["reasoning_effort"] = payload.reasoning_effort + # Inbound extra_body chat_template_kwargs win outright (highest priority). + _extra = getattr(payload, "model_extra", None) + if isinstance(_extra, dict): + inbound_kw = _extra.get("chat_template_kwargs") + if isinstance(inbound_kw, dict): + tpl_kwargs.update(inbound_kw) + # Collapse empty dict to None so we don't emit an empty + # ``chat_template_kwargs`` field to llama-server. + tpl_kwargs = tpl_kwargs or None return _build_passthrough_payload( messages, payload.tools, @@ -5367,7 +5402,9 @@ async def _openai_passthrough_stream( """ target_url = f"{llama_backend.base_url}/v1/chat/completions" body = _build_openai_passthrough_body( - payload, backend_ctx = llama_backend.context_length + payload, + backend_ctx = llama_backend.context_length, + model_identifier = llama_backend.model_identifier, ) _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) @@ -5525,7 +5562,9 @@ async def _openai_passthrough_non_streaming( """ target_url = f"{llama_backend.base_url}/v1/chat/completions" body = _build_openai_passthrough_body( - payload, backend_ctx = llama_backend.context_length + payload, + backend_ctx = llama_backend.context_length, + model_identifier = llama_backend.model_identifier, ) try: diff --git a/studio/backend/tests/test_inference_defaults_chat_template_kwargs.py b/studio/backend/tests/test_inference_defaults_chat_template_kwargs.py new file mode 100644 index 0000000000..4f8d6e2d2e --- /dev/null +++ b/studio/backend/tests/test_inference_defaults_chat_template_kwargs.py @@ -0,0 +1,104 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Tests for the ``chat_template_kwargs`` extension to the per-family +inference defaults registry. + +Adding nested template kwargs to ``inference_defaults.json`` lets the +family resolver push values like ``reasoning_effort`` and +``enable_thinking`` for modern reasoning models without requiring every +client to set ``extra_body.chat_template_kwargs`` explicitly. +""" + +from utils.inference.inference_config import ( + get_family_inference_params, + load_inference_config, +) + + +class TestRegistry: + def test_gpt_oss_carries_reasoning_effort(self): + params = get_family_inference_params("unsloth/gpt-oss-120b-GGUF") + assert "chat_template_kwargs" in params + assert params["chat_template_kwargs"]["reasoning_effort"] == "medium" + + def test_nemotron_carries_enable_thinking(self): + params = get_family_inference_params( + "unsloth/NVIDIA-Nemotron-3-Super-120B-A12B-GGUF" + ) + assert "chat_template_kwargs" in params + assert params["chat_template_kwargs"]["enable_thinking"] is True + + def test_family_without_kwargs_returns_no_field(self): + # qwen3 family has scalar sampling fields but no chat_template_kwargs. + params = get_family_inference_params("unsloth/Qwen3-8B-GGUF") + assert "chat_template_kwargs" not in params + + def test_unknown_family_returns_empty(self): + params = get_family_inference_params("unsloth/CompletelyMadeUp-99B") + assert params == {} + + +class TestLoadInferenceConfig: + def test_gpt_oss_load_surfaces_dict(self): + cfg = load_inference_config("unsloth/gpt-oss-120b-GGUF") + assert cfg["chat_template_kwargs"] == {"reasoning_effort": "medium"} + + def test_nemotron_load_surfaces_dict(self): + cfg = load_inference_config("unsloth/NVIDIA-Nemotron-3-Super-120B-A12B-GGUF") + assert cfg["chat_template_kwargs"] == {"enable_thinking": True} + + def test_qwen3_load_returns_none_for_kwargs(self): + cfg = load_inference_config("unsloth/Qwen3-8B-GGUF") + assert cfg["chat_template_kwargs"] is None + + def test_load_returns_a_fresh_dict_per_call(self): + # Mutating the returned dict must not poison the registry. + a = load_inference_config("unsloth/gpt-oss-120b-GGUF") + a["chat_template_kwargs"]["reasoning_effort"] = "high" + b = load_inference_config("unsloth/gpt-oss-120b-GGUF") + assert b["chat_template_kwargs"]["reasoning_effort"] == "medium" + + def test_scalar_sampling_fields_unchanged_when_kwargs_added(self): + # Adding a nested key must not regress the existing scalar field + # extraction for families that gained chat_template_kwargs. + cfg = load_inference_config("unsloth/gpt-oss-120b-GGUF") + assert cfg["temperature"] == 1.0 + assert cfg["top_p"] == 1.0 + assert cfg["top_k"] == 0 + + def test_repeated_calls_reuse_cached_result(self): + """The passthrough request path calls ``load_inference_config`` + on every chat-completion / Responses request. Cache the heavy + work (YAML reads + recursive ``rglob`` inside + ``_has_specific_yaml``) so the hot path doesn't pay the full + scan each time.""" + from utils.inference.inference_config import ( + _has_specific_yaml, + _load_inference_config_cached, + ) + + _load_inference_config_cached.cache_clear() + _has_specific_yaml.cache_clear() + + ident = "unsloth/gpt-oss-120b-GGUF" + load_inference_config(ident) + load_inference_config(ident) + load_inference_config(ident) + + # Three calls, one underlying miss; the rest served from cache. + info = _load_inference_config_cached.cache_info() + assert info.misses == 1, info + assert info.hits >= 2, info + + +class TestTemperatureBumps: + def test_devstral_temperature_lowered_to_card_value(self): + # Devstral-Small-2 card recommends T=0.15. + cfg = load_inference_config("unsloth/Devstral-Small-2-24B-Instruct-2512-GGUF") + assert cfg["temperature"] == 0.15 + + def test_ministral_temperature_below_one_tenth(self): + # Ministral-3 card says "temperature below 0.1 for production". + cfg = load_inference_config("unsloth/Ministral-3-8B-Instruct-2512-GGUF") + assert cfg["temperature"] < 0.1 diff --git a/studio/backend/tests/test_passthrough_chat_template_kwargs.py b/studio/backend/tests/test_passthrough_chat_template_kwargs.py new file mode 100644 index 0000000000..15eca30a40 --- /dev/null +++ b/studio/backend/tests/test_passthrough_chat_template_kwargs.py @@ -0,0 +1,101 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Tests that ``_build_openai_passthrough_body`` consults the family +registry for ``chat_template_kwargs`` and merges them with per-request +overrides in the correct priority order. + +Without the registry consult, external OpenAI SDK clients that hit +``/v1/chat/completions`` with ``tools=[...]`` on a gpt-oss / Nemotron +model never see the recommended template kwargs reach llama-server, +even though Studio Chat (which uses the non-passthrough path) gets +them via a different route. +""" + +from models.inference import ChatCompletionRequest +from routes.inference import _build_openai_passthrough_body + + +def _make_payload(**fields): + base = { + "model": "gpt-oss", + "messages": [{"role": "user", "content": "hi"}], + "stream": False, + } + base.update(fields) + return ChatCompletionRequest(**base) + + +class TestFamilyDefaults: + def test_gpt_oss_family_default_reaches_outbound(self): + payload = _make_payload() + body = _build_openai_passthrough_body( + payload, model_identifier = "unsloth/gpt-oss-120b-GGUF" + ) + assert body.get("chat_template_kwargs") == {"reasoning_effort": "medium"} + + def test_nemotron_family_default_reaches_outbound(self): + payload = _make_payload() + body = _build_openai_passthrough_body( + payload, model_identifier = "unsloth/NVIDIA-Nemotron-3-Super-120B-A12B-GGUF" + ) + assert body.get("chat_template_kwargs") == {"enable_thinking": True} + + def test_no_model_identifier_skips_registry(self): + # Without a model_identifier we cannot look up the family, so + # behaviour collapses to the prior (request-only) path. + payload = _make_payload() + body = _build_openai_passthrough_body(payload) + assert "chat_template_kwargs" not in body + + def test_unknown_family_yields_no_template_kwargs(self): + payload = _make_payload() + body = _build_openai_passthrough_body( + payload, model_identifier = "unsloth/CompletelyMadeUp-99B" + ) + assert "chat_template_kwargs" not in body + + +class TestPerRequestOverrides: + def test_explicit_reasoning_effort_wins_over_family(self): + payload = _make_payload(reasoning_effort = "high") + body = _build_openai_passthrough_body( + payload, model_identifier = "unsloth/gpt-oss-120b-GGUF" + ) + assert body["chat_template_kwargs"]["reasoning_effort"] == "high" + + def test_extra_body_chat_template_kwargs_wins_outright(self): + payload = _make_payload(chat_template_kwargs = {"reasoning_effort": "low"}) + body = _build_openai_passthrough_body( + payload, model_identifier = "unsloth/gpt-oss-120b-GGUF" + ) + assert body["chat_template_kwargs"]["reasoning_effort"] == "low" + + def test_enable_thinking_propagates_alongside_family_keys(self): + # Family carries reasoning_effort, the caller sets enable_thinking. + # Both should end up in the outbound dict. + payload = _make_payload(enable_thinking = False) + body = _build_openai_passthrough_body( + payload, model_identifier = "unsloth/gpt-oss-120b-GGUF" + ) + kw = body["chat_template_kwargs"] + assert kw["reasoning_effort"] == "medium" + assert kw["enable_thinking"] is False + + +class TestNoSpuriousKwargs: + def test_qwen3_no_family_kwargs_emits_no_field(self): + payload = _make_payload() + body = _build_openai_passthrough_body( + payload, model_identifier = "unsloth/Qwen3-8B-GGUF" + ) + assert "chat_template_kwargs" not in body + + def test_empty_overrides_collapse_to_no_field(self): + # No family kwargs and no per-request keys means no outbound + # field (not an empty dict). + payload = _make_payload() + body = _build_openai_passthrough_body( + payload, model_identifier = "unsloth/Qwen3-8B-GGUF" + ) + assert "chat_template_kwargs" not in body diff --git a/studio/backend/utils/inference/inference_config.py b/studio/backend/utils/inference/inference_config.py index 9efc281b0b..23e0a849ec 100644 --- a/studio/backend/utils/inference/inference_config.py +++ b/studio/backend/utils/inference/inference_config.py @@ -9,6 +9,8 @@ from model YAML configuration files, with fallback to default.yaml. Includes family-based lookup from inference_defaults.json for GGUF models. """ +from copy import deepcopy +from functools import lru_cache from pathlib import Path from typing import Dict, Any, Optional import json @@ -83,8 +85,16 @@ def get_family_inference_params(model_id: str) -> Dict[str, Any]: return {} +@lru_cache(maxsize = 256) def _has_specific_yaml(model_identifier: str) -> bool: - """Check if a model has its own YAML config (not just default.yaml).""" + """Check if a model has its own YAML config (not just default.yaml). + + Cached because the lookup walks ``defaults_dir`` recursively via + ``rglob`` on every miss, and every chat-completion / Responses + passthrough request asks the same question for the same loaded + model. The defaults directory is shipped with the package and does + not mutate at runtime, so an LRU cache is safe. + """ from utils.models.model_config import _REVERSE_MODEL_MAPPING script_dir = Path(__file__).parent.parent.parent @@ -144,6 +154,17 @@ def load_inference_config(model_identifier: str) -> Dict[str, Any]: "min_p": float } """ + # The heavy work (YAML reads + recursive scan inside + # `_has_specific_yaml`) is memoised on the model identifier; this + # function is called from the hot path of every passthrough + # request. Callers are documented to treat the returned dict as + # immutable, but tests historically mutate it — so deepcopy the + # snapshot before returning to preserve that contract. + return deepcopy(_load_inference_config_cached(model_identifier)) + + +@lru_cache(maxsize = 256) +def _load_inference_config_cached(model_identifier: str) -> Dict[str, Any]: # Load model defaults to get inference parameters model_defaults = load_model_defaults(model_identifier) @@ -185,12 +206,29 @@ def load_inference_config(model_identifier: str) -> Dict[str, Any]: return family_params[key] return default_inference.get(key, hardcoded_default) + def _get_dict_param(key): + # Dict-valued defaults (e.g. ``chat_template_kwargs``). Same + # priority chain as _get_param but with type-safe dict checks. + # Returns None when nothing is configured anywhere. + if has_own_yaml: + val = model_inference.get(key) + if isinstance(val, dict) and val: + return dict(val) + fam = family_params.get(key) + if isinstance(fam, dict) and fam: + return dict(fam) + dfl = default_inference.get(key) + if isinstance(dfl, dict) and dfl: + return dict(dfl) + return None + inference_config = { "temperature": _get_param("temperature", 0.7), "top_p": _get_param("top_p", 0.95), "top_k": _get_param("top_k", -1), "min_p": _get_param("min_p", 0.01), "presence_penalty": _get_param("presence_penalty", 0.0), + "chat_template_kwargs": _get_dict_param("chat_template_kwargs"), "trust_remote_code": model_inference.get( "trust_remote_code", default_inference.get("trust_remote_code", False) ),