From 70021b1f6a66a8f24fa9d133d17d593639513810 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 22 May 2026 14:35:37 +0000 Subject: [PATCH 1/3] Studio: family-default chat_template_kwargs for gpt-oss + Nemotron The per-family inference defaults registry (inference_defaults.json) exposes scalar sampling fields only (temperature, top_p, top_k, min_p, presence_penalty). Modern reasoning models also want template-level kwargs: gpt-oss takes reasoning_effort (low|medium|high); Nemotron-3 takes enable_thinking. Today those reach llama-server only when an OpenAI SDK client sets extra_body.chat_template_kwargs explicitly, so external clients hitting /v1/chat/completions never benefit from the recommended defaults. This patch: * Adds an optional ``chat_template_kwargs`` block per family in inference_defaults.json. Bootstrap rows: gpt-oss -> reasoning_effort: medium nemotron -> enable_thinking: true * Extends load_inference_config to surface chat_template_kwargs in the returned dict (None when unset). * Extends _build_openai_passthrough_body to merge family defaults (lookup via load_inference_config(model_identifier)) with per-request overrides. Priority order, base to top: family_defaults -> payload.enable_thinking -> payload.reasoning_effort -> extra_body.chat_template_kwargs * Bumps two family temperatures to match upstream HF model cards: devstral 0.7 -> 0.15 (Devstral-Small-2-Instruct card) ministral 0.15 -> 0.05 (Ministral-3 card: "below 0.1 for production") --- .../assets/configs/inference_defaults.json | 14 ++- studio/backend/routes/inference.py | 54 +++++++-- ...inference_defaults_chat_template_kwargs.py | 86 ++++++++++++++ .../test_passthrough_chat_template_kwargs.py | 105 ++++++++++++++++++ .../utils/inference/inference_config.py | 17 +++ 5 files changed, 263 insertions(+), 13 deletions(-) create mode 100644 studio/backend/tests/test_inference_defaults_chat_template_kwargs.py create mode 100644 studio/backend/tests/test_passthrough_chat_template_kwargs.py diff --git a/studio/backend/assets/configs/inference_defaults.json b/studio/backend/assets/configs/inference_defaults.json index 1b10b557e4..c42b53b876 100644 --- a/studio/backend/assets/configs/inference_defaults.json +++ b/studio/backend/assets/configs/inference_defaults.json @@ -215,14 +215,14 @@ "repetition_penalty": 1.0 }, "ministral": { - "temperature": 0.15, + "temperature": 0.05, "top_p": 0.95, "top_k": -1, "min_p": 0.01, "repetition_penalty": 1.0 }, "devstral": { - "temperature": 0.7, + "temperature": 0.15, "top_p": 0.95, "top_k": -1, "min_p": 0.01, @@ -275,7 +275,10 @@ "top_p": 1.0, "top_k": -1, "min_p": 0.01, - "repetition_penalty": 1.0 + "repetition_penalty": 1.0, + "chat_template_kwargs": { + "enable_thinking": true + } }, "minimax-m2.5": { "temperature": 1.0, @@ -296,7 +299,10 @@ "top_p": 1.0, "top_k": 0, "min_p": 0.01, - "repetition_penalty": 1.0 + "repetition_penalty": 1.0, + "chat_template_kwargs": { + "reasoning_effort": "medium" + } }, "granite-4": { "temperature": 0.0, diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 02270ab405..4d23e73df1 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -3948,7 +3948,9 @@ async def _responses_stream( ) body = _build_openai_passthrough_body( - chat_req, backend_ctx = llama_backend.context_length + chat_req, + backend_ctx = llama_backend.context_length, + model_identifier = llama_backend.model_identifier, ) target_url = f"{llama_backend.base_url}/v1/chat/completions" @@ -5315,21 +5317,51 @@ def _extract_response_format(payload): return rf if isinstance(rf, dict) else None -def _build_openai_passthrough_body(payload, backend_ctx = None) -> dict: +def _build_openai_passthrough_body( + payload, backend_ctx = None, model_identifier: str | None = None, +) -> dict: """Assemble the llama-server request body from a ChatCompletionRequest. Only explicitly-known OpenAI / llama-server fields are forwarded so that Studio-specific extensions (``enable_tools``, ``enabled_tools``, ``session_id``, ...) never leak to the backend. + + ``model_identifier`` (when supplied) gates a registry lookup so the + family default ``chat_template_kwargs`` (e.g. gpt-oss + ``reasoning_effort=medium``) reaches llama-server even when the + inbound request did not set the keys explicitly. Per-request keys + still win on conflict. """ messages = _openai_messages_for_passthrough(payload) tool_choice = payload.tool_choice if payload.tool_choice is not None else "auto" - # When the caller asked for a specific reasoning mode, forward it to - # llama-server via chat_template_kwargs so the Jinja template renders - # with (or without) the reasoning preamble. - tpl_kwargs = None + # Merge family-default chat_template_kwargs (base) with per-request + # overrides. Per-request always wins. + tpl_kwargs: dict = {} + if model_identifier: + try: + family_defaults = load_inference_config(model_identifier) + family_kw = family_defaults.get("chat_template_kwargs") + if isinstance(family_kw, dict): + tpl_kwargs.update(family_kw) + except Exception as exc: + logger.warning( + "openai_passthrough.family_defaults_lookup_failed model=%s err=%s", + model_identifier, exc, + ) + # Per-request enable_thinking already lifted from extra_body upstream. if payload.enable_thinking is not None: - tpl_kwargs = {"enable_thinking": bool(payload.enable_thinking)} + tpl_kwargs["enable_thinking"] = bool(payload.enable_thinking) + if payload.reasoning_effort is not None: + tpl_kwargs["reasoning_effort"] = payload.reasoning_effort + # Inbound extra_body chat_template_kwargs win outright (highest priority). + _extra = getattr(payload, "model_extra", None) + if isinstance(_extra, dict): + inbound_kw = _extra.get("chat_template_kwargs") + if isinstance(inbound_kw, dict): + tpl_kwargs.update(inbound_kw) + # Collapse empty dict to None so we don't emit an empty + # ``chat_template_kwargs`` field to llama-server. + tpl_kwargs = tpl_kwargs or None return _build_passthrough_payload( messages, payload.tools, @@ -5367,7 +5399,9 @@ async def _openai_passthrough_stream( """ target_url = f"{llama_backend.base_url}/v1/chat/completions" body = _build_openai_passthrough_body( - payload, backend_ctx = llama_backend.context_length + payload, + backend_ctx = llama_backend.context_length, + model_identifier = llama_backend.model_identifier, ) _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) @@ -5525,7 +5559,9 @@ async def _openai_passthrough_non_streaming( """ target_url = f"{llama_backend.base_url}/v1/chat/completions" body = _build_openai_passthrough_body( - payload, backend_ctx = llama_backend.context_length + payload, + backend_ctx = llama_backend.context_length, + model_identifier = llama_backend.model_identifier, ) try: diff --git a/studio/backend/tests/test_inference_defaults_chat_template_kwargs.py b/studio/backend/tests/test_inference_defaults_chat_template_kwargs.py new file mode 100644 index 0000000000..e9ac216dd6 --- /dev/null +++ b/studio/backend/tests/test_inference_defaults_chat_template_kwargs.py @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Tests for the ``chat_template_kwargs`` extension to the per-family +inference defaults registry. + +Adding nested template kwargs to ``inference_defaults.json`` lets the +family resolver push values like ``reasoning_effort`` and +``enable_thinking`` for modern reasoning models without requiring every +client to set ``extra_body.chat_template_kwargs`` explicitly. +""" + +from utils.inference.inference_config import ( + get_family_inference_params, + load_inference_config, +) + + +class TestRegistry: + def test_gpt_oss_carries_reasoning_effort(self): + params = get_family_inference_params("unsloth/gpt-oss-120b-GGUF") + assert "chat_template_kwargs" in params + assert params["chat_template_kwargs"]["reasoning_effort"] == "medium" + + def test_nemotron_carries_enable_thinking(self): + params = get_family_inference_params( + "unsloth/NVIDIA-Nemotron-3-Super-120B-A12B-GGUF" + ) + assert "chat_template_kwargs" in params + assert params["chat_template_kwargs"]["enable_thinking"] is True + + def test_family_without_kwargs_returns_no_field(self): + # qwen3 family has scalar sampling fields but no chat_template_kwargs. + params = get_family_inference_params("unsloth/Qwen3-8B-GGUF") + assert "chat_template_kwargs" not in params + + def test_unknown_family_returns_empty(self): + params = get_family_inference_params("unsloth/CompletelyMadeUp-99B") + assert params == {} + + +class TestLoadInferenceConfig: + def test_gpt_oss_load_surfaces_dict(self): + cfg = load_inference_config("unsloth/gpt-oss-120b-GGUF") + assert cfg["chat_template_kwargs"] == {"reasoning_effort": "medium"} + + def test_nemotron_load_surfaces_dict(self): + cfg = load_inference_config( + "unsloth/NVIDIA-Nemotron-3-Super-120B-A12B-GGUF" + ) + assert cfg["chat_template_kwargs"] == {"enable_thinking": True} + + def test_qwen3_load_returns_none_for_kwargs(self): + cfg = load_inference_config("unsloth/Qwen3-8B-GGUF") + assert cfg["chat_template_kwargs"] is None + + def test_load_returns_a_fresh_dict_per_call(self): + # Mutating the returned dict must not poison the registry. + a = load_inference_config("unsloth/gpt-oss-120b-GGUF") + a["chat_template_kwargs"]["reasoning_effort"] = "high" + b = load_inference_config("unsloth/gpt-oss-120b-GGUF") + assert b["chat_template_kwargs"]["reasoning_effort"] == "medium" + + def test_scalar_sampling_fields_unchanged_when_kwargs_added(self): + # Adding a nested key must not regress the existing scalar field + # extraction for families that gained chat_template_kwargs. + cfg = load_inference_config("unsloth/gpt-oss-120b-GGUF") + assert cfg["temperature"] == 1.0 + assert cfg["top_p"] == 1.0 + assert cfg["top_k"] == 0 + + +class TestTemperatureBumps: + def test_devstral_temperature_lowered_to_card_value(self): + # Devstral-Small-2 card recommends T=0.15. + cfg = load_inference_config( + "unsloth/Devstral-Small-2-24B-Instruct-2512-GGUF" + ) + assert cfg["temperature"] == 0.15 + + def test_ministral_temperature_below_one_tenth(self): + # Ministral-3 card says "temperature below 0.1 for production". + cfg = load_inference_config( + "unsloth/Ministral-3-8B-Instruct-2512-GGUF" + ) + assert cfg["temperature"] < 0.1 diff --git a/studio/backend/tests/test_passthrough_chat_template_kwargs.py b/studio/backend/tests/test_passthrough_chat_template_kwargs.py new file mode 100644 index 0000000000..2d1d4043fb --- /dev/null +++ b/studio/backend/tests/test_passthrough_chat_template_kwargs.py @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Tests that ``_build_openai_passthrough_body`` consults the family +registry for ``chat_template_kwargs`` and merges them with per-request +overrides in the correct priority order. + +Without the registry consult, external OpenAI SDK clients that hit +``/v1/chat/completions`` with ``tools=[...]`` on a gpt-oss / Nemotron +model never see the recommended template kwargs reach llama-server, +even though Studio Chat (which uses the non-passthrough path) gets +them via a different route. +""" + +from models.inference import ChatCompletionRequest +from routes.inference import _build_openai_passthrough_body + + +def _make_payload(**fields): + base = { + "model": "gpt-oss", + "messages": [{"role": "user", "content": "hi"}], + "stream": False, + } + base.update(fields) + return ChatCompletionRequest(**base) + + +class TestFamilyDefaults: + def test_gpt_oss_family_default_reaches_outbound(self): + payload = _make_payload() + body = _build_openai_passthrough_body( + payload, model_identifier = "unsloth/gpt-oss-120b-GGUF" + ) + assert body.get("chat_template_kwargs") == { + "reasoning_effort": "medium" + } + + def test_nemotron_family_default_reaches_outbound(self): + payload = _make_payload() + body = _build_openai_passthrough_body( + payload, model_identifier = "unsloth/NVIDIA-Nemotron-3-Super-120B-A12B-GGUF" + ) + assert body.get("chat_template_kwargs") == {"enable_thinking": True} + + def test_no_model_identifier_skips_registry(self): + # Without a model_identifier we cannot look up the family, so + # behaviour collapses to the prior (request-only) path. + payload = _make_payload() + body = _build_openai_passthrough_body(payload) + assert "chat_template_kwargs" not in body + + def test_unknown_family_yields_no_template_kwargs(self): + payload = _make_payload() + body = _build_openai_passthrough_body( + payload, model_identifier = "unsloth/CompletelyMadeUp-99B" + ) + assert "chat_template_kwargs" not in body + + +class TestPerRequestOverrides: + def test_explicit_reasoning_effort_wins_over_family(self): + payload = _make_payload(reasoning_effort = "high") + body = _build_openai_passthrough_body( + payload, model_identifier = "unsloth/gpt-oss-120b-GGUF" + ) + assert body["chat_template_kwargs"]["reasoning_effort"] == "high" + + def test_extra_body_chat_template_kwargs_wins_outright(self): + payload = _make_payload( + chat_template_kwargs = {"reasoning_effort": "low"} + ) + body = _build_openai_passthrough_body( + payload, model_identifier = "unsloth/gpt-oss-120b-GGUF" + ) + assert body["chat_template_kwargs"]["reasoning_effort"] == "low" + + def test_enable_thinking_propagates_alongside_family_keys(self): + # Family carries reasoning_effort, the caller sets enable_thinking. + # Both should end up in the outbound dict. + payload = _make_payload(enable_thinking = False) + body = _build_openai_passthrough_body( + payload, model_identifier = "unsloth/gpt-oss-120b-GGUF" + ) + kw = body["chat_template_kwargs"] + assert kw["reasoning_effort"] == "medium" + assert kw["enable_thinking"] is False + + +class TestNoSpuriousKwargs: + def test_qwen3_no_family_kwargs_emits_no_field(self): + payload = _make_payload() + body = _build_openai_passthrough_body( + payload, model_identifier = "unsloth/Qwen3-8B-GGUF" + ) + assert "chat_template_kwargs" not in body + + def test_empty_overrides_collapse_to_no_field(self): + # No family kwargs and no per-request keys means no outbound + # field (not an empty dict). + payload = _make_payload() + body = _build_openai_passthrough_body( + payload, model_identifier = "unsloth/Qwen3-8B-GGUF" + ) + assert "chat_template_kwargs" not in body diff --git a/studio/backend/utils/inference/inference_config.py b/studio/backend/utils/inference/inference_config.py index 9efc281b0b..5e01f405df 100644 --- a/studio/backend/utils/inference/inference_config.py +++ b/studio/backend/utils/inference/inference_config.py @@ -185,12 +185,29 @@ def load_inference_config(model_identifier: str) -> Dict[str, Any]: return family_params[key] return default_inference.get(key, hardcoded_default) + def _get_dict_param(key): + # Dict-valued defaults (e.g. ``chat_template_kwargs``). Same + # priority chain as _get_param but with type-safe dict checks. + # Returns None when nothing is configured anywhere. + if has_own_yaml: + val = model_inference.get(key) + if isinstance(val, dict) and val: + return dict(val) + fam = family_params.get(key) + if isinstance(fam, dict) and fam: + return dict(fam) + dfl = default_inference.get(key) + if isinstance(dfl, dict) and dfl: + return dict(dfl) + return None + inference_config = { "temperature": _get_param("temperature", 0.7), "top_p": _get_param("top_p", 0.95), "top_k": _get_param("top_k", -1), "min_p": _get_param("min_p", 0.01), "presence_penalty": _get_param("presence_penalty", 0.0), + "chat_template_kwargs": _get_dict_param("chat_template_kwargs"), "trust_remote_code": model_inference.get( "trust_remote_code", default_inference.get("trust_remote_code", False) ), From de3032eb07a6fe94f749f14db2e9da536eba8465 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 14:36:25 +0000 Subject: [PATCH 2/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/routes/inference.py | 7 +++++-- .../test_inference_defaults_chat_template_kwargs.py | 12 +++--------- .../tests/test_passthrough_chat_template_kwargs.py | 8 ++------ 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 4d23e73df1..c61479034c 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -5318,7 +5318,9 @@ def _extract_response_format(payload): def _build_openai_passthrough_body( - payload, backend_ctx = None, model_identifier: str | None = None, + payload, + backend_ctx = None, + model_identifier: str | None = None, ) -> dict: """Assemble the llama-server request body from a ChatCompletionRequest. @@ -5346,7 +5348,8 @@ def _build_openai_passthrough_body( except Exception as exc: logger.warning( "openai_passthrough.family_defaults_lookup_failed model=%s err=%s", - model_identifier, exc, + model_identifier, + exc, ) # Per-request enable_thinking already lifted from extra_body upstream. if payload.enable_thinking is not None: diff --git a/studio/backend/tests/test_inference_defaults_chat_template_kwargs.py b/studio/backend/tests/test_inference_defaults_chat_template_kwargs.py index e9ac216dd6..4e54bf562d 100644 --- a/studio/backend/tests/test_inference_defaults_chat_template_kwargs.py +++ b/studio/backend/tests/test_inference_defaults_chat_template_kwargs.py @@ -45,9 +45,7 @@ class TestLoadInferenceConfig: assert cfg["chat_template_kwargs"] == {"reasoning_effort": "medium"} def test_nemotron_load_surfaces_dict(self): - cfg = load_inference_config( - "unsloth/NVIDIA-Nemotron-3-Super-120B-A12B-GGUF" - ) + cfg = load_inference_config("unsloth/NVIDIA-Nemotron-3-Super-120B-A12B-GGUF") assert cfg["chat_template_kwargs"] == {"enable_thinking": True} def test_qwen3_load_returns_none_for_kwargs(self): @@ -73,14 +71,10 @@ class TestLoadInferenceConfig: class TestTemperatureBumps: def test_devstral_temperature_lowered_to_card_value(self): # Devstral-Small-2 card recommends T=0.15. - cfg = load_inference_config( - "unsloth/Devstral-Small-2-24B-Instruct-2512-GGUF" - ) + cfg = load_inference_config("unsloth/Devstral-Small-2-24B-Instruct-2512-GGUF") assert cfg["temperature"] == 0.15 def test_ministral_temperature_below_one_tenth(self): # Ministral-3 card says "temperature below 0.1 for production". - cfg = load_inference_config( - "unsloth/Ministral-3-8B-Instruct-2512-GGUF" - ) + cfg = load_inference_config("unsloth/Ministral-3-8B-Instruct-2512-GGUF") assert cfg["temperature"] < 0.1 diff --git a/studio/backend/tests/test_passthrough_chat_template_kwargs.py b/studio/backend/tests/test_passthrough_chat_template_kwargs.py index 2d1d4043fb..15eca30a40 100644 --- a/studio/backend/tests/test_passthrough_chat_template_kwargs.py +++ b/studio/backend/tests/test_passthrough_chat_template_kwargs.py @@ -32,9 +32,7 @@ class TestFamilyDefaults: body = _build_openai_passthrough_body( payload, model_identifier = "unsloth/gpt-oss-120b-GGUF" ) - assert body.get("chat_template_kwargs") == { - "reasoning_effort": "medium" - } + assert body.get("chat_template_kwargs") == {"reasoning_effort": "medium"} def test_nemotron_family_default_reaches_outbound(self): payload = _make_payload() @@ -67,9 +65,7 @@ class TestPerRequestOverrides: assert body["chat_template_kwargs"]["reasoning_effort"] == "high" def test_extra_body_chat_template_kwargs_wins_outright(self): - payload = _make_payload( - chat_template_kwargs = {"reasoning_effort": "low"} - ) + payload = _make_payload(chat_template_kwargs = {"reasoning_effort": "low"}) body = _build_openai_passthrough_body( payload, model_identifier = "unsloth/gpt-oss-120b-GGUF" ) From 4577c9c5655c1522064a4be37f53fd45729c15da Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 22 May 2026 15:52:48 +0000 Subject: [PATCH 3/3] Memoise load_inference_config to skip per-request disk scans Every passthrough request through `/v1/chat/completions` and the streaming `/v1/responses` path calls `load_inference_config(model)` to fold family-default `chat_template_kwargs` (e.g. gpt-oss `reasoning_effort=medium`) into the outbound body. The function walks the model_defaults directory recursively via `rglob` inside `_has_specific_yaml` plus reads two YAML files on every call, so the hot path was paying the full lookup cost for every token-budget poll, every tool turn, every reasoning sub-step. The defaults directory is shipped with the package and does not mutate at runtime, so wrap both `_has_specific_yaml` and the expensive inner work of `load_inference_config` in `lru_cache`. The public entry point still returns a fresh deepcopy of the cached snapshot so the existing `test_load_returns_a_fresh_dict_per_call` contract (callers may safely mutate the dict) is preserved. Adds a regression test pinning the cache-hit count over repeated calls for the same identifier. --- ...inference_defaults_chat_template_kwargs.py | 24 +++++++++++++++++++ .../utils/inference/inference_config.py | 23 +++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/studio/backend/tests/test_inference_defaults_chat_template_kwargs.py b/studio/backend/tests/test_inference_defaults_chat_template_kwargs.py index 4e54bf562d..4f8d6e2d2e 100644 --- a/studio/backend/tests/test_inference_defaults_chat_template_kwargs.py +++ b/studio/backend/tests/test_inference_defaults_chat_template_kwargs.py @@ -67,6 +67,30 @@ class TestLoadInferenceConfig: assert cfg["top_p"] == 1.0 assert cfg["top_k"] == 0 + def test_repeated_calls_reuse_cached_result(self): + """The passthrough request path calls ``load_inference_config`` + on every chat-completion / Responses request. Cache the heavy + work (YAML reads + recursive ``rglob`` inside + ``_has_specific_yaml``) so the hot path doesn't pay the full + scan each time.""" + from utils.inference.inference_config import ( + _has_specific_yaml, + _load_inference_config_cached, + ) + + _load_inference_config_cached.cache_clear() + _has_specific_yaml.cache_clear() + + ident = "unsloth/gpt-oss-120b-GGUF" + load_inference_config(ident) + load_inference_config(ident) + load_inference_config(ident) + + # Three calls, one underlying miss; the rest served from cache. + info = _load_inference_config_cached.cache_info() + assert info.misses == 1, info + assert info.hits >= 2, info + class TestTemperatureBumps: def test_devstral_temperature_lowered_to_card_value(self): diff --git a/studio/backend/utils/inference/inference_config.py b/studio/backend/utils/inference/inference_config.py index 5e01f405df..23e0a849ec 100644 --- a/studio/backend/utils/inference/inference_config.py +++ b/studio/backend/utils/inference/inference_config.py @@ -9,6 +9,8 @@ from model YAML configuration files, with fallback to default.yaml. Includes family-based lookup from inference_defaults.json for GGUF models. """ +from copy import deepcopy +from functools import lru_cache from pathlib import Path from typing import Dict, Any, Optional import json @@ -83,8 +85,16 @@ def get_family_inference_params(model_id: str) -> Dict[str, Any]: return {} +@lru_cache(maxsize = 256) def _has_specific_yaml(model_identifier: str) -> bool: - """Check if a model has its own YAML config (not just default.yaml).""" + """Check if a model has its own YAML config (not just default.yaml). + + Cached because the lookup walks ``defaults_dir`` recursively via + ``rglob`` on every miss, and every chat-completion / Responses + passthrough request asks the same question for the same loaded + model. The defaults directory is shipped with the package and does + not mutate at runtime, so an LRU cache is safe. + """ from utils.models.model_config import _REVERSE_MODEL_MAPPING script_dir = Path(__file__).parent.parent.parent @@ -144,6 +154,17 @@ def load_inference_config(model_identifier: str) -> Dict[str, Any]: "min_p": float } """ + # The heavy work (YAML reads + recursive scan inside + # `_has_specific_yaml`) is memoised on the model identifier; this + # function is called from the hot path of every passthrough + # request. Callers are documented to treat the returned dict as + # immutable, but tests historically mutate it — so deepcopy the + # snapshot before returning to preserve that contract. + return deepcopy(_load_inference_config_cached(model_identifier)) + + +@lru_cache(maxsize = 256) +def _load_inference_config_cached(model_identifier: str) -> Dict[str, Any]: # Load model defaults to get inference parameters model_defaults = load_model_defaults(model_identifier)