Memoise load_inference_config to skip per-request disk scans

Every passthrough request through `/v1/chat/completions` and the
streaming `/v1/responses` path calls `load_inference_config(model)`
to fold family-default `chat_template_kwargs` (e.g. gpt-oss
`reasoning_effort=medium`) into the outbound body. The function
walks the model_defaults directory recursively via `rglob` inside
`_has_specific_yaml` plus reads two YAML files on every call, so the
hot path was paying the full lookup cost for every token-budget
poll, every tool turn, every reasoning sub-step.

The defaults directory is shipped with the package and does not
mutate at runtime, so wrap both `_has_specific_yaml` and the
expensive inner work of `load_inference_config` in `lru_cache`. The
public entry point still returns a fresh deepcopy of the cached
snapshot so the existing `test_load_returns_a_fresh_dict_per_call`
contract (callers may safely mutate the dict) is preserved.

Adds a regression test pinning the cache-hit count over repeated
calls for the same identifier.

studio/backend/assets/configs/inference_defaults.json

@@ -215,14 +215,14 @@
       "repetition_penalty": 1.0
     },
     "ministral": {
-      "temperature": 0.15,
+      "temperature": 0.05,
       "top_p": 0.95,
       "top_k": -1,
       "min_p": 0.01,
       "repetition_penalty": 1.0
     },
     "devstral": {
-      "temperature": 0.7,
+      "temperature": 0.15,
       "top_p": 0.95,
       "top_k": -1,
       "min_p": 0.01,
@@ -275,7 +275,10 @@
       "top_p": 1.0,
       "top_k": -1,
       "min_p": 0.01,
-      "repetition_penalty": 1.0
+      "repetition_penalty": 1.0,
+      "chat_template_kwargs": {
+        "enable_thinking": true
+      }
     },
     "minimax-m2.5": {
       "temperature": 1.0,
@@ -296,7 +299,10 @@
       "top_p": 1.0,
       "top_k": 0,
       "min_p": 0.01,
-      "repetition_penalty": 1.0
+      "repetition_penalty": 1.0,
+      "chat_template_kwargs": {
+        "reasoning_effort": "medium"
+      }
     },
     "granite-4": {
       "temperature": 0.0,

studio/backend/routes/inference.py

@@ -3948,7 +3948,9 @@ async def _responses_stream(
         )
 
     body = _build_openai_passthrough_body(
-        chat_req, backend_ctx = llama_backend.context_length
+        chat_req,
+        backend_ctx = llama_backend.context_length,
+        model_identifier = llama_backend.model_identifier,
     )
     target_url = f"{llama_backend.base_url}/v1/chat/completions"
 
@@ -5315,21 +5317,54 @@ def _extract_response_format(payload):
     return rf if isinstance(rf, dict) else None
 
 
-def _build_openai_passthrough_body(payload, backend_ctx = None) -> dict:
+def _build_openai_passthrough_body(
+    payload,
+    backend_ctx = None,
+    model_identifier: str | None = None,
+) -> dict:
     """Assemble the llama-server request body from a ChatCompletionRequest.
 
     Only explicitly-known OpenAI / llama-server fields are forwarded so that
     Studio-specific extensions (``enable_tools``, ``enabled_tools``,
     ``session_id``, ...) never leak to the backend.
+
+    ``model_identifier`` (when supplied) gates a registry lookup so the
+    family default ``chat_template_kwargs`` (e.g. gpt-oss
+    ``reasoning_effort=medium``) reaches llama-server even when the
+    inbound request did not set the keys explicitly. Per-request keys
+    still win on conflict.
     """
     messages = _openai_messages_for_passthrough(payload)
     tool_choice = payload.tool_choice if payload.tool_choice is not None else "auto"
-    # When the caller asked for a specific reasoning mode, forward it to
-    # llama-server via chat_template_kwargs so the Jinja template renders
-    # with (or without) the reasoning preamble.
-    tpl_kwargs = None
+    # Merge family-default chat_template_kwargs (base) with per-request
+    # overrides. Per-request always wins.
+    tpl_kwargs: dict = {}
+    if model_identifier:
+        try:
+            family_defaults = load_inference_config(model_identifier)
+            family_kw = family_defaults.get("chat_template_kwargs")
+            if isinstance(family_kw, dict):
+                tpl_kwargs.update(family_kw)
+        except Exception as exc:
+            logger.warning(
+                "openai_passthrough.family_defaults_lookup_failed model=%s err=%s",
+                model_identifier,
+                exc,
+            )
+    # Per-request enable_thinking already lifted from extra_body upstream.
     if payload.enable_thinking is not None:
-        tpl_kwargs = {"enable_thinking": bool(payload.enable_thinking)}
+        tpl_kwargs["enable_thinking"] = bool(payload.enable_thinking)
+    if payload.reasoning_effort is not None:
+        tpl_kwargs["reasoning_effort"] = payload.reasoning_effort
+    # Inbound extra_body chat_template_kwargs win outright (highest priority).
+    _extra = getattr(payload, "model_extra", None)
+    if isinstance(_extra, dict):
+        inbound_kw = _extra.get("chat_template_kwargs")
+        if isinstance(inbound_kw, dict):
+            tpl_kwargs.update(inbound_kw)
+    # Collapse empty dict to None so we don't emit an empty
+    # ``chat_template_kwargs`` field to llama-server.
+    tpl_kwargs = tpl_kwargs or None
     return _build_passthrough_payload(
         messages,
         payload.tools,
@@ -5367,7 +5402,9 @@ async def _openai_passthrough_stream(
     """
     target_url = f"{llama_backend.base_url}/v1/chat/completions"
     body = _build_openai_passthrough_body(
-        payload, backend_ctx = llama_backend.context_length
+        payload,
+        backend_ctx = llama_backend.context_length,
+        model_identifier = llama_backend.model_identifier,
     )
 
     _cancel_keys = (payload.cancel_id, payload.session_id, completion_id)
@@ -5525,7 +5562,9 @@ async def _openai_passthrough_non_streaming(
     """
     target_url = f"{llama_backend.base_url}/v1/chat/completions"
     body = _build_openai_passthrough_body(
-        payload, backend_ctx = llama_backend.context_length
+        payload,
+        backend_ctx = llama_backend.context_length,
+        model_identifier = llama_backend.model_identifier,
     )
 
     try:

studio/backend/tests/test_inference_defaults_chat_template_kwargs.py

@@ -0,0 +1,104 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Tests for the ``chat_template_kwargs`` extension to the per-family
+inference defaults registry.
+
+Adding nested template kwargs to ``inference_defaults.json`` lets the
+family resolver push values like ``reasoning_effort`` and
+``enable_thinking`` for modern reasoning models without requiring every
+client to set ``extra_body.chat_template_kwargs`` explicitly.
+"""
+
+from utils.inference.inference_config import (
+    get_family_inference_params,
+    load_inference_config,
+)
+
+
+class TestRegistry:
+    def test_gpt_oss_carries_reasoning_effort(self):
+        params = get_family_inference_params("unsloth/gpt-oss-120b-GGUF")
+        assert "chat_template_kwargs" in params
+        assert params["chat_template_kwargs"]["reasoning_effort"] == "medium"
+
+    def test_nemotron_carries_enable_thinking(self):
+        params = get_family_inference_params(
+            "unsloth/NVIDIA-Nemotron-3-Super-120B-A12B-GGUF"
+        )
+        assert "chat_template_kwargs" in params
+        assert params["chat_template_kwargs"]["enable_thinking"] is True
+
+    def test_family_without_kwargs_returns_no_field(self):
+        # qwen3 family has scalar sampling fields but no chat_template_kwargs.
+        params = get_family_inference_params("unsloth/Qwen3-8B-GGUF")
+        assert "chat_template_kwargs" not in params
+
+    def test_unknown_family_returns_empty(self):
+        params = get_family_inference_params("unsloth/CompletelyMadeUp-99B")
+        assert params == {}
+
+
+class TestLoadInferenceConfig:
+    def test_gpt_oss_load_surfaces_dict(self):
+        cfg = load_inference_config("unsloth/gpt-oss-120b-GGUF")
+        assert cfg["chat_template_kwargs"] == {"reasoning_effort": "medium"}
+
+    def test_nemotron_load_surfaces_dict(self):
+        cfg = load_inference_config("unsloth/NVIDIA-Nemotron-3-Super-120B-A12B-GGUF")
+        assert cfg["chat_template_kwargs"] == {"enable_thinking": True}
+
+    def test_qwen3_load_returns_none_for_kwargs(self):
+        cfg = load_inference_config("unsloth/Qwen3-8B-GGUF")
+        assert cfg["chat_template_kwargs"] is None
+
+    def test_load_returns_a_fresh_dict_per_call(self):
+        # Mutating the returned dict must not poison the registry.
+        a = load_inference_config("unsloth/gpt-oss-120b-GGUF")
+        a["chat_template_kwargs"]["reasoning_effort"] = "high"
+        b = load_inference_config("unsloth/gpt-oss-120b-GGUF")
+        assert b["chat_template_kwargs"]["reasoning_effort"] == "medium"
+
+    def test_scalar_sampling_fields_unchanged_when_kwargs_added(self):
+        # Adding a nested key must not regress the existing scalar field
+        # extraction for families that gained chat_template_kwargs.
+        cfg = load_inference_config("unsloth/gpt-oss-120b-GGUF")
+        assert cfg["temperature"] == 1.0
+        assert cfg["top_p"] == 1.0
+        assert cfg["top_k"] == 0
+
+    def test_repeated_calls_reuse_cached_result(self):
+        """The passthrough request path calls ``load_inference_config``
+        on every chat-completion / Responses request. Cache the heavy
+        work (YAML reads + recursive ``rglob`` inside
+        ``_has_specific_yaml``) so the hot path doesn't pay the full
+        scan each time."""
+        from utils.inference.inference_config import (
+            _has_specific_yaml,
+            _load_inference_config_cached,
+        )
+
+        _load_inference_config_cached.cache_clear()
+        _has_specific_yaml.cache_clear()
+
+        ident = "unsloth/gpt-oss-120b-GGUF"
+        load_inference_config(ident)
+        load_inference_config(ident)
+        load_inference_config(ident)
+
+        # Three calls, one underlying miss; the rest served from cache.
+        info = _load_inference_config_cached.cache_info()
+        assert info.misses == 1, info
+        assert info.hits >= 2, info
+
+
+class TestTemperatureBumps:
+    def test_devstral_temperature_lowered_to_card_value(self):
+        # Devstral-Small-2 card recommends T=0.15.
+        cfg = load_inference_config("unsloth/Devstral-Small-2-24B-Instruct-2512-GGUF")
+        assert cfg["temperature"] == 0.15
+
+    def test_ministral_temperature_below_one_tenth(self):
+        # Ministral-3 card says "temperature below 0.1 for production".
+        cfg = load_inference_config("unsloth/Ministral-3-8B-Instruct-2512-GGUF")
+        assert cfg["temperature"] < 0.1

studio/backend/tests/test_passthrough_chat_template_kwargs.py

@@ -0,0 +1,101 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Tests that ``_build_openai_passthrough_body`` consults the family
+registry for ``chat_template_kwargs`` and merges them with per-request
+overrides in the correct priority order.
+
+Without the registry consult, external OpenAI SDK clients that hit
+``/v1/chat/completions`` with ``tools=[...]`` on a gpt-oss / Nemotron
+model never see the recommended template kwargs reach llama-server,
+even though Studio Chat (which uses the non-passthrough path) gets
+them via a different route.
+"""
+
+from models.inference import ChatCompletionRequest
+from routes.inference import _build_openai_passthrough_body
+
+
+def _make_payload(**fields):
+    base = {
+        "model": "gpt-oss",
+        "messages": [{"role": "user", "content": "hi"}],
+        "stream": False,
+    }
+    base.update(fields)
+    return ChatCompletionRequest(**base)
+
+
+class TestFamilyDefaults:
+    def test_gpt_oss_family_default_reaches_outbound(self):
+        payload = _make_payload()
+        body = _build_openai_passthrough_body(
+            payload, model_identifier = "unsloth/gpt-oss-120b-GGUF"
+        )
+        assert body.get("chat_template_kwargs") == {"reasoning_effort": "medium"}
+
+    def test_nemotron_family_default_reaches_outbound(self):
+        payload = _make_payload()
+        body = _build_openai_passthrough_body(
+            payload, model_identifier = "unsloth/NVIDIA-Nemotron-3-Super-120B-A12B-GGUF"
+        )
+        assert body.get("chat_template_kwargs") == {"enable_thinking": True}
+
+    def test_no_model_identifier_skips_registry(self):
+        # Without a model_identifier we cannot look up the family, so
+        # behaviour collapses to the prior (request-only) path.
+        payload = _make_payload()
+        body = _build_openai_passthrough_body(payload)
+        assert "chat_template_kwargs" not in body
+
+    def test_unknown_family_yields_no_template_kwargs(self):
+        payload = _make_payload()
+        body = _build_openai_passthrough_body(
+            payload, model_identifier = "unsloth/CompletelyMadeUp-99B"
+        )
+        assert "chat_template_kwargs" not in body
+
+
+class TestPerRequestOverrides:
+    def test_explicit_reasoning_effort_wins_over_family(self):
+        payload = _make_payload(reasoning_effort = "high")
+        body = _build_openai_passthrough_body(
+            payload, model_identifier = "unsloth/gpt-oss-120b-GGUF"
+        )
+        assert body["chat_template_kwargs"]["reasoning_effort"] == "high"
+
+    def test_extra_body_chat_template_kwargs_wins_outright(self):
+        payload = _make_payload(chat_template_kwargs = {"reasoning_effort": "low"})
+        body = _build_openai_passthrough_body(
+            payload, model_identifier = "unsloth/gpt-oss-120b-GGUF"
+        )
+        assert body["chat_template_kwargs"]["reasoning_effort"] == "low"
+
+    def test_enable_thinking_propagates_alongside_family_keys(self):
+        # Family carries reasoning_effort, the caller sets enable_thinking.
+        # Both should end up in the outbound dict.
+        payload = _make_payload(enable_thinking = False)
+        body = _build_openai_passthrough_body(
+            payload, model_identifier = "unsloth/gpt-oss-120b-GGUF"
+        )
+        kw = body["chat_template_kwargs"]
+        assert kw["reasoning_effort"] == "medium"
+        assert kw["enable_thinking"] is False
+
+
+class TestNoSpuriousKwargs:
+    def test_qwen3_no_family_kwargs_emits_no_field(self):
+        payload = _make_payload()
+        body = _build_openai_passthrough_body(
+            payload, model_identifier = "unsloth/Qwen3-8B-GGUF"
+        )
+        assert "chat_template_kwargs" not in body
+
+    def test_empty_overrides_collapse_to_no_field(self):
+        # No family kwargs and no per-request keys means no outbound
+        # field (not an empty dict).
+        payload = _make_payload()
+        body = _build_openai_passthrough_body(
+            payload, model_identifier = "unsloth/Qwen3-8B-GGUF"
+        )
+        assert "chat_template_kwargs" not in body

studio/backend/utils/inference/inference_config.py

@@ -9,6 +9,8 @@ from model YAML configuration files, with fallback to default.yaml.
 Includes family-based lookup from inference_defaults.json for GGUF models.
 """
 
+from copy import deepcopy
+from functools import lru_cache
 from pathlib import Path
 from typing import Dict, Any, Optional
 import json
@@ -83,8 +85,16 @@ def get_family_inference_params(model_id: str) -> Dict[str, Any]:
     return {}
 
 
+@lru_cache(maxsize = 256)
 def _has_specific_yaml(model_identifier: str) -> bool:
-    """Check if a model has its own YAML config (not just default.yaml)."""
+    """Check if a model has its own YAML config (not just default.yaml).
+
+    Cached because the lookup walks ``defaults_dir`` recursively via
+    ``rglob`` on every miss, and every chat-completion / Responses
+    passthrough request asks the same question for the same loaded
+    model. The defaults directory is shipped with the package and does
+    not mutate at runtime, so an LRU cache is safe.
+    """
     from utils.models.model_config import _REVERSE_MODEL_MAPPING
 
     script_dir = Path(__file__).parent.parent.parent
@@ -144,6 +154,17 @@ def load_inference_config(model_identifier: str) -> Dict[str, Any]:
             "min_p": float
         }
     """
+    # The heavy work (YAML reads + recursive scan inside
+    # `_has_specific_yaml`) is memoised on the model identifier; this
+    # function is called from the hot path of every passthrough
+    # request. Callers are documented to treat the returned dict as
+    # immutable, but tests historically mutate it — so deepcopy the
+    # snapshot before returning to preserve that contract.
+    return deepcopy(_load_inference_config_cached(model_identifier))
+
+
+@lru_cache(maxsize = 256)
+def _load_inference_config_cached(model_identifier: str) -> Dict[str, Any]:
     # Load model defaults to get inference parameters
     model_defaults = load_model_defaults(model_identifier)
 
@@ -185,12 +206,29 @@ def load_inference_config(model_identifier: str) -> Dict[str, Any]:
                 return family_params[key]
             return default_inference.get(key, hardcoded_default)
 
+    def _get_dict_param(key):
+        # Dict-valued defaults (e.g. ``chat_template_kwargs``). Same
+        # priority chain as _get_param but with type-safe dict checks.
+        # Returns None when nothing is configured anywhere.
+        if has_own_yaml:
+            val = model_inference.get(key)
+            if isinstance(val, dict) and val:
+                return dict(val)
+        fam = family_params.get(key)
+        if isinstance(fam, dict) and fam:
+            return dict(fam)
+        dfl = default_inference.get(key)
+        if isinstance(dfl, dict) and dfl:
+            return dict(dfl)
+        return None
+
     inference_config = {
         "temperature": _get_param("temperature", 0.7),
         "top_p": _get_param("top_p", 0.95),
         "top_k": _get_param("top_k", -1),
         "min_p": _get_param("min_p", 0.01),
         "presence_penalty": _get_param("presence_penalty", 0.0),
+        "chat_template_kwargs": _get_dict_param("chat_template_kwargs"),
         "trust_remote_code": model_inference.get(
             "trust_remote_code", default_inference.get("trust_remote_code", False)
         ),