From dd0b1d58a737e3e3f85cf8caf4eeee7282c7d7eb Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 15 May 2026 02:15:50 +0000 Subject: [PATCH] Studio: track requested n_ctx so Auto-slider flips trigger a reload Review feedback on PR #5427 from gemini-code-assist. The original short-circuit compared ``request.max_seq_length`` against ``llama_backend.context_length`` (the effective context). VRAM-fit logic can cap the running server below what the caller asked for, so this comparison incorrectly returns ``already_loaded`` when the user flips the slider from an explicit length (e.g. 8192) back to "Auto" (0): the explicit request was capped to, say, 4096, and the new "Auto" request reads ``backend.context_length == 4096`` and decides nothing changed. Track the originally requested ``n_ctx`` on the backend instead and compare against that. ``requested_n_ctx == 0`` means the last load asked for the model's native length; ``request.max_seq_length == 0`` matches it. Verified in the sandbox suite (now 90 tests): - ``test_explicit_to_auto_triggers_reload`` -- loaded with explicit 8192, then Apply with ``max_seq_length=0`` falls through to a real reload and the new server runs at the native 40960. - ``test_auto_to_explicit_triggers_reload`` -- inverse direction. - ``test_explicit_to_same_explicit_short_circuits`` -- re-Apply with the same explicit value still short-circuits (no needless reload). - Existing scenarios (kv change, spec change, template change, extra args inherit, parallel-load stress, frontend Apply flow) unchanged. ``pytest studio/backend/tests`` still green on the same set of tests; the pre-existing ``test_help_output`` failure and ``test_studio_api`` fixture errors are unaffected. --- studio/backend/core/inference/llama_cpp.py | 29 ++++++++++++++++++++++ studio/backend/routes/inference.py | 11 +++++--- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 5443960936..750618b764 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -486,6 +486,15 @@ class LlamaCppBackend: # round-trip those flags) can inherit the values the CLI / first # load supplied. See issue #5401. self._extra_args: Optional[List[str]] = None + # The n_ctx the most recent load_model() was *invoked with* (NOT + # the effective context the server is running at -- that one + # lives in ``_effective_context_length`` and may have been capped + # by VRAM-fit logic). The route layer compares against this so an + # Apply that flips the slider between "Auto" (0) and an explicit + # length is detected as a settings change even when the running + # server happens to be at the same effective context. See issue + # #5401 and the gemini-code-assist review on PR #5427. + self._requested_n_ctx: int = 0 self._stdout_lines: list[str] = [] self._stdout_thread: Optional[threading.Thread] = None self._cancel_event = threading.Event() @@ -533,6 +542,21 @@ class LlamaCppBackend: """ return list(self._extra_args) if self._extra_args is not None else None + @property + def requested_n_ctx(self) -> int: + """The ``n_ctx`` value the last ``load_model()`` was *invoked + with* (not the effective context the server is running at). + + Returned as an int -- 0 means the caller asked for the model's + native length. The route layer compares an incoming + ``request.max_seq_length`` against this so a slider flip from + explicit (e.g. 8192) to "Auto" (0) is detected as a real + settings change even when the running server's effective context + happens to match the explicit value (because VRAM-fit may have + capped it). See issue #5401. + """ + return self._requested_n_ctx + @property def context_length(self) -> Optional[int]: """Return the effective context length the server is running at.""" @@ -2033,6 +2057,11 @@ class LlamaCppBackend: # See issue #5401. if extra_args is not None: self._extra_args = list(extra_args) + # Track the requested n_ctx so the route comparator can + # distinguish "user picked Auto" (0) from "user picked an + # explicit length that happens to equal the running effective + # context". See requested_n_ctx property docstring. + self._requested_n_ctx = int(n_ctx) self._cancel_event.clear() diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index c10939491c..169a5ef630 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -423,10 +423,13 @@ def _request_matches_loaded_settings( runtime setting falls through to a real reload instead of silently returning ``status="already_loaded"`` (issue #5401). """ - # max_seq_length == 0 means "use model default" -- treat as a match - # against the running effective context length. - backend_ctx = llama_backend.context_length or 0 - if request.max_seq_length and request.max_seq_length != backend_ctx: + # Compare against the *requested* n_ctx, not the effective context. + # The effective context can be lower than the request if VRAM-fit + # capped it, so comparing against the effective value would + # incorrectly call an explicit-to-Auto slider flip a "match". + # ``requested_n_ctx == 0`` means the last load asked for the model's + # native length; an incoming ``max_seq_length == 0`` matches that. + if request.max_seq_length != llama_backend.requested_n_ctx: return False if _normalise_settings_str(request.cache_type_kv) != _normalise_settings_str( llama_backend.cache_type_kv