From 434dee96185a6d1018e5593f4829bd4ba001476e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 1 Apr 2026 12:49:41 +0000 Subject: [PATCH] Tighten _can_estimate_kv gate and treat sliding_window=0 as disabled Two additional fixes from review round 1 (5/8 and 4/8 reviewer consensus): 1. _can_estimate_kv now requires BOTH key_length AND value_length for the explicit-dims path. Previously key_length alone was enough, which could cause silent fallthrough to the legacy formula with fabricated defaults (n_kv=1, head_dim=128) when value_length was absent from the GGUF. 2. SWA path now requires sliding_window > 0. Some GGUFs use 0 as a disabled sentinel. Without this guard, min(ctx, 0) would zero out all SWA layer contributions, severely underestimating KV cache. --- studio/backend/core/inference/llama_cpp.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index a1b872c8e6..6635764e01 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -358,12 +358,12 @@ class LlamaCppBackend: """True if we have enough GGUF metadata to estimate KV cache size.""" if self._n_layers is None: return False - # New-style: explicit key/value dimensions from GGUF - if self._kv_key_length is not None: - return True - # MLA: kv_lora_rank is sufficient + # MLA: kv_lora_rank is sufficient (K-only cache) if self._kv_lora_rank is not None: return True + # New-style: need both explicit key AND value dimensions + if self._kv_key_length is not None and self._kv_value_length is not None: + return True # Legacy: need embedding_length + head count return self._embedding_length is not None and ( self._n_kv_heads is not None or self._n_heads is not None @@ -434,6 +434,7 @@ class LlamaCppBackend: # which is still far more accurate than the legacy formula (which ignores SWA). if ( self._sliding_window is not None + and self._sliding_window > 0 and key_len is not None and val_len is not None ):