From 4577c9c5655c1522064a4be37f53fd45729c15da Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 22 May 2026 15:52:48 +0000 Subject: [PATCH] Memoise load_inference_config to skip per-request disk scans Every passthrough request through `/v1/chat/completions` and the streaming `/v1/responses` path calls `load_inference_config(model)` to fold family-default `chat_template_kwargs` (e.g. gpt-oss `reasoning_effort=medium`) into the outbound body. The function walks the model_defaults directory recursively via `rglob` inside `_has_specific_yaml` plus reads two YAML files on every call, so the hot path was paying the full lookup cost for every token-budget poll, every tool turn, every reasoning sub-step. The defaults directory is shipped with the package and does not mutate at runtime, so wrap both `_has_specific_yaml` and the expensive inner work of `load_inference_config` in `lru_cache`. The public entry point still returns a fresh deepcopy of the cached snapshot so the existing `test_load_returns_a_fresh_dict_per_call` contract (callers may safely mutate the dict) is preserved. Adds a regression test pinning the cache-hit count over repeated calls for the same identifier. --- ...inference_defaults_chat_template_kwargs.py | 24 +++++++++++++++++++ .../utils/inference/inference_config.py | 23 +++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/studio/backend/tests/test_inference_defaults_chat_template_kwargs.py b/studio/backend/tests/test_inference_defaults_chat_template_kwargs.py index 4e54bf562d..4f8d6e2d2e 100644 --- a/studio/backend/tests/test_inference_defaults_chat_template_kwargs.py +++ b/studio/backend/tests/test_inference_defaults_chat_template_kwargs.py @@ -67,6 +67,30 @@ class TestLoadInferenceConfig: assert cfg["top_p"] == 1.0 assert cfg["top_k"] == 0 + def test_repeated_calls_reuse_cached_result(self): + """The passthrough request path calls ``load_inference_config`` + on every chat-completion / Responses request. Cache the heavy + work (YAML reads + recursive ``rglob`` inside + ``_has_specific_yaml``) so the hot path doesn't pay the full + scan each time.""" + from utils.inference.inference_config import ( + _has_specific_yaml, + _load_inference_config_cached, + ) + + _load_inference_config_cached.cache_clear() + _has_specific_yaml.cache_clear() + + ident = "unsloth/gpt-oss-120b-GGUF" + load_inference_config(ident) + load_inference_config(ident) + load_inference_config(ident) + + # Three calls, one underlying miss; the rest served from cache. + info = _load_inference_config_cached.cache_info() + assert info.misses == 1, info + assert info.hits >= 2, info + class TestTemperatureBumps: def test_devstral_temperature_lowered_to_card_value(self): diff --git a/studio/backend/utils/inference/inference_config.py b/studio/backend/utils/inference/inference_config.py index 5e01f405df..23e0a849ec 100644 --- a/studio/backend/utils/inference/inference_config.py +++ b/studio/backend/utils/inference/inference_config.py @@ -9,6 +9,8 @@ from model YAML configuration files, with fallback to default.yaml. Includes family-based lookup from inference_defaults.json for GGUF models. """ +from copy import deepcopy +from functools import lru_cache from pathlib import Path from typing import Dict, Any, Optional import json @@ -83,8 +85,16 @@ def get_family_inference_params(model_id: str) -> Dict[str, Any]: return {} +@lru_cache(maxsize = 256) def _has_specific_yaml(model_identifier: str) -> bool: - """Check if a model has its own YAML config (not just default.yaml).""" + """Check if a model has its own YAML config (not just default.yaml). + + Cached because the lookup walks ``defaults_dir`` recursively via + ``rglob`` on every miss, and every chat-completion / Responses + passthrough request asks the same question for the same loaded + model. The defaults directory is shipped with the package and does + not mutate at runtime, so an LRU cache is safe. + """ from utils.models.model_config import _REVERSE_MODEL_MAPPING script_dir = Path(__file__).parent.parent.parent @@ -144,6 +154,17 @@ def load_inference_config(model_identifier: str) -> Dict[str, Any]: "min_p": float } """ + # The heavy work (YAML reads + recursive scan inside + # `_has_specific_yaml`) is memoised on the model identifier; this + # function is called from the hot path of every passthrough + # request. Callers are documented to treat the returned dict as + # immutable, but tests historically mutate it — so deepcopy the + # snapshot before returning to preserve that contract. + return deepcopy(_load_inference_config_cached(model_identifier)) + + +@lru_cache(maxsize = 256) +def _load_inference_config_cached(model_identifier: str) -> Dict[str, Any]: # Load model defaults to get inference parameters model_defaults = load_model_defaults(model_identifier)