diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 873b72bba1..5f974b0af3 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -116,6 +116,11 @@ class LlamaCppBackend: """Return the maximum context currently available on this hardware.""" return self._max_context_length or self._context_length + @property + def native_context_length(self) -> Optional[int]: + """Return the model's native context length from GGUF metadata.""" + return self._context_length + @property def chat_template(self) -> Optional[str]: return self._chat_template @@ -315,11 +320,11 @@ class LlamaCppBackend: """Pick GPU(s) for a model based on estimated VRAM and free memory. ``model_size_bytes`` should include both model weights and estimated - KV cache. The 70% threshold provides headroom for compute buffers, + KV cache. The 90% threshold provides headroom for compute buffers, CUDA context, and other runtime overhead. Returns (gpu_indices, use_fit): - - ([1], False) model fits on 1 GPU at 70% of free + - ([1], False) model fits on 1 GPU at 90% of free - ([1, 2], False) model needs 2 GPUs - (None, True) model too large, let --fit handle it """ @@ -331,8 +336,8 @@ class LlamaCppBackend: # Sort GPUs by free memory descending ranked = sorted(gpus, key = lambda g: g[1], reverse = True) - # Try fitting on 1 GPU (70% of free memory threshold) - if ranked[0][1] * 0.70 >= model_size_mib: + # Try fitting on 1 GPU (90% of free memory threshold) + if ranked[0][1] * 0.90 >= model_size_mib: return [ranked[0][0]], False # Try fitting on N GPUs (accumulate free memory from most-free) @@ -340,7 +345,7 @@ class LlamaCppBackend: selected = [] for idx, free_mib in ranked: selected.append(idx) - cumulative += free_mib * 0.70 + cumulative += free_mib * 0.90 if cumulative >= model_size_mib: return sorted(selected), False @@ -467,8 +472,8 @@ class LlamaCppBackend: ) -> int: """Return the largest context length that fits in GPU VRAM. - Uses 70% of available VRAM as the budget (matching _select_gpus - threshold -- 30% reserved for compute buffers, CUDA context, + Uses 90% of available VRAM as the budget (matching _select_gpus + threshold -- 10% reserved for compute buffers, CUDA context, scratch space, flash-attn workspace, etc.). If the model weights alone don't fit, returns min_ctx unchanged. """ @@ -480,7 +485,7 @@ class LlamaCppBackend: ) return requested_ctx - budget_bytes = available_mib * 1024 * 1024 * 0.70 + budget_bytes = available_mib * 1024 * 1024 * 0.90 model_footprint = model_size_bytes # Check if requested context already fits @@ -1133,7 +1138,7 @@ class LlamaCppBackend: ) kv = self._estimate_kv_cache_bytes(capped, cache_type_kv) total_mib = (model_size + kv) / (1024 * 1024) - if total_mib <= pool_mib * 0.70: + if total_mib <= pool_mib * 0.90: best_cap = max(best_cap, capped) if best_cap > 0: max_available_ctx = best_cap @@ -1162,7 +1167,7 @@ class LlamaCppBackend: capped, cache_type_kv ) total_mib = (model_size + kv) / (1024 * 1024) - if total_mib <= pool_mib * 0.70: + if total_mib <= pool_mib * 0.90: effective_ctx = capped gpu_indices = sorted(idx for idx, _ in subset) use_fit = False @@ -1181,7 +1186,7 @@ class LlamaCppBackend: ) kv = self._estimate_kv_cache_bytes(capped, cache_type_kv) total_mib = (model_size + kv) / (1024 * 1024) - if total_mib <= pool_mib * 0.70: + if total_mib <= pool_mib * 0.90: effective_ctx = capped gpu_indices = sorted(idx for idx, _ in subset) use_fit = False diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index 77f70b9bd6..3094df4169 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -139,6 +139,10 @@ class LoadResponse(BaseModel): max_context_length: Optional[int] = Field( None, description = "Maximum context length currently available on this hardware" ) + native_context_length: Optional[int] = Field( + None, + description = "Model's native context length from GGUF metadata (not capped by VRAM)", + ) supports_reasoning: bool = Field( False, description = "Whether model supports thinking/reasoning mode (enable_thinking)", @@ -217,6 +221,10 @@ class InferenceStatusResponse(BaseModel): None, description = "Maximum context length currently available for the active model", ) + native_context_length: Optional[int] = Field( + None, + description = "Model's native context length from GGUF metadata (not capped by VRAM)", + ) # ===================================================================== diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 9bce371775..99a52dfe2d 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -163,6 +163,7 @@ async def load_model( inference = inference_config, context_length = llama_backend.context_length, max_context_length = llama_backend.max_context_length, + native_context_length = llama_backend.native_context_length, supports_reasoning = llama_backend.supports_reasoning, reasoning_always_on = llama_backend.reasoning_always_on, chat_template = llama_backend.chat_template, @@ -298,6 +299,7 @@ async def load_model( inference = inference_config, context_length = llama_backend.context_length, max_context_length = llama_backend.max_context_length, + native_context_length = llama_backend.native_context_length, supports_reasoning = llama_backend.supports_reasoning, reasoning_always_on = llama_backend.reasoning_always_on, supports_tools = llama_backend.supports_tools, @@ -637,6 +639,7 @@ async def get_status( supports_tools = llama_backend.supports_tools, context_length = llama_backend.context_length, max_context_length = llama_backend.max_context_length, + native_context_length = llama_backend.native_context_length, ) # Otherwise, report Unsloth backend status diff --git a/studio/backend/tests/test_native_context_length.py b/studio/backend/tests/test_native_context_length.py new file mode 100644 index 0000000000..7c69e56f89 --- /dev/null +++ b/studio/backend/tests/test_native_context_length.py @@ -0,0 +1,518 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Tests for the native_context_length feature (PR #4746). + +Verifies that the new `native_context_length` property on LlamaCppBackend +and the corresponding Pydantic model fields work correctly. The raw GGUF +`_context_length` must never be overwritten by VRAM-capping logic. + +Requires no GPU, network, or external libraries beyond pytest and pydantic. +""" + +import io +import json +import struct +import sys +import types as _types +from pathlib import Path +from unittest.mock import patch + +import pytest + +# --------------------------------------------------------------------------- +# Stub heavy / unavailable external dependencies before importing the +# module under test. Same pattern as test_kv_cache_estimation.py. +# --------------------------------------------------------------------------- + +_BACKEND_DIR = str(Path(__file__).resolve().parent.parent) +if _BACKEND_DIR not in sys.path: + sys.path.insert(0, _BACKEND_DIR) + +# loggers +_loggers_stub = _types.ModuleType("loggers") +_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name) +sys.modules.setdefault("loggers", _loggers_stub) + +# structlog +_structlog_stub = _types.ModuleType("structlog") +sys.modules.setdefault("structlog", _structlog_stub) + +# httpx -- stub only the names referenced at import / class-definition time +_httpx_stub = _types.ModuleType("httpx") +for _exc_name in ( + "ConnectError", + "TimeoutException", + "ReadTimeout", + "ReadError", + "RemoteProtocolError", + "CloseError", +): + setattr(_httpx_stub, _exc_name, type(_exc_name, (Exception,), {})) + + +class _FakeTimeout: + def __init__(self, *a, **kw): + pass + + +_httpx_stub.Timeout = _FakeTimeout +_httpx_stub.Client = type( + "Client", + (), + { + "__init__": lambda self, **kw: None, + "__enter__": lambda self: self, + "__exit__": lambda self, *a: None, + }, +) +sys.modules.setdefault("httpx", _httpx_stub) + +from core.inference.llama_cpp import LlamaCppBackend +from models.inference import LoadResponse, InferenceStatusResponse + + +# ── Helpers ────────────────────────────────────────────────────────── + + +def _write_kv(buf: io.BytesIO, key: str, value, vtype: int) -> None: + """Append a single GGUF KV pair to *buf*.""" + key_bytes = key.encode("utf-8") + buf.write(struct.pack(" str: + """Create a minimal valid GGUF v3 binary in *tmp_path*.""" + buf = io.BytesIO() + buf.write(struct.pack("= max >= effective holds when VRAM-capped.""" + backend._context_length = 131072 + backend._max_context_length = 65536 + backend._effective_context_length = 32768 + assert backend.native_context_length >= backend.max_context_length + assert backend.max_context_length >= backend.context_length + + def test_all_equal_when_uncapped(self, backend): + """All three equal when no VRAM constraint.""" + backend._context_length = 8192 + # No effective or max set -- properties fall back to _context_length + assert backend.native_context_length == 8192 + assert backend.max_context_length == 8192 + assert backend.context_length == 8192 + + def test_fit_context_does_not_modify(self, backend): + """_fit_context_to_vram() does not touch _context_length.""" + backend._context_length = 131072 + backend._n_layers = 32 + backend._n_kv_heads = 8 + backend._n_heads = 32 + backend._embedding_length = 4096 + original = backend._context_length + + # Simulate a very small VRAM budget that forces capping + result = backend._fit_context_to_vram( + requested_ctx = 131072, + available_mib = 512, # very small + model_size_bytes = 0, + ) + # _fit_context_to_vram returns the capped value, not modifying _context_length + assert backend._context_length == original + assert backend.native_context_length == original + # The returned capped value should be <= requested + assert result <= 131072 + + def test_native_gt_context_when_capped(self, backend): + """native_context_length > context_length after VRAM capping.""" + backend._context_length = 131072 + backend._effective_context_length = 16384 + assert backend.native_context_length > backend.context_length + + +# ===================================================================== +# C. TestPydanticModels -- LoadResponse & InferenceStatusResponse +# ===================================================================== + + +class TestPydanticModels: + """Tests native_context_length field on Pydantic models.""" + + def test_load_response_has_field(self): + """Field exists in LoadResponse.model_fields.""" + assert "native_context_length" in LoadResponse.model_fields + + def test_load_response_defaults_none(self): + """Omitting native_context_length defaults to None.""" + resp = LoadResponse( + status = "loaded", + model = "test", + display_name = "Test", + inference = {}, + ) + assert resp.native_context_length is None + + def test_load_response_accepts_int(self): + """native_context_length=131072 stores correctly.""" + resp = LoadResponse( + status = "loaded", + model = "test", + display_name = "Test", + inference = {}, + native_context_length = 131072, + ) + assert resp.native_context_length == 131072 + + def test_load_response_json_null(self): + """None serializes to JSON null.""" + resp = LoadResponse( + status = "loaded", + model = "test", + display_name = "Test", + inference = {}, + ) + data = json.loads(resp.model_dump_json()) + assert data["native_context_length"] is None + + def test_load_response_json_int(self): + """131072 serializes to JSON number.""" + resp = LoadResponse( + status = "loaded", + model = "test", + display_name = "Test", + inference = {}, + native_context_length = 131072, + ) + data = json.loads(resp.model_dump_json()) + assert data["native_context_length"] == 131072 + + def test_status_response_has_field(self): + """Field exists in InferenceStatusResponse.model_fields.""" + assert "native_context_length" in InferenceStatusResponse.model_fields + + def test_status_response_defaults_none(self): + """Omitting native_context_length defaults to None.""" + resp = InferenceStatusResponse() + assert resp.native_context_length is None + + def test_roundtrip_preserves_value(self): + """model_validate_json(model_dump_json()) round-trips.""" + resp = LoadResponse( + status = "loaded", + model = "test", + display_name = "Test", + inference = {}, + native_context_length = 131072, + ) + roundtripped = LoadResponse.model_validate_json(resp.model_dump_json()) + assert roundtripped.native_context_length == 131072 + + +# ===================================================================== +# D. TestRouteCompleteness -- source-level verification +# ===================================================================== + + +class TestRouteCompleteness: + """All response construction sites in routes/inference.py include native_context_length.""" + + @pytest.fixture(autouse = True) + def _load_source(self): + """Read routes/inference.py source once.""" + routes_path = Path(__file__).resolve().parent.parent / "routes" / "inference.py" + self._source = routes_path.read_text() + + def _find_construction_blocks(self, class_name: str) -> list[str]: + """Extract all code blocks that construct a given response class.""" + blocks = [] + idx = 0 + while True: + start = self._source.find(f"{class_name}(", idx) + if start == -1: + break + # Find matching closing paren (simple depth counter) + depth = 0 + end = start + for i, ch in enumerate(self._source[start:], start): + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth == 0: + end = i + 1 + break + blocks.append(self._source[start:end]) + idx = end + return blocks + + def test_gguf_load_responses_have_field(self): + """Every GGUF LoadResponse (is_gguf = True) includes native_context_length.""" + blocks = self._find_construction_blocks("LoadResponse") + gguf_blocks = [ + b for b in blocks if "is_gguf = True" in b or "is_gguf=True" in b + ] + assert ( + len(gguf_blocks) >= 2 + ), f"Expected at least 2 GGUF LoadResponse blocks, found {len(gguf_blocks)}" + for i, block in enumerate(gguf_blocks): + assert ( + "native_context_length" in block + ), f"GGUF LoadResponse block #{i} missing native_context_length:\n{block[:200]}" + + def test_non_gguf_load_responses_omit_field(self): + """Non-GGUF LoadResponse blocks do not set native_context_length (defaults to None).""" + blocks = self._find_construction_blocks("LoadResponse") + non_gguf = [ + b for b in blocks if "is_gguf = True" not in b and "is_gguf=True" not in b + ] + # Non-GGUF paths should not reference native_context_length + # (Pydantic defaults it to None, so not setting it is correct) + for block in non_gguf: + assert ( + "native_context_length" not in block + ), f"Non-GGUF LoadResponse should not set native_context_length:\n{block[:200]}" + + def test_status_path(self): + """InferenceStatusResponse construction with llama_backend has the field.""" + blocks = self._find_construction_blocks("InferenceStatusResponse") + found = False + for block in blocks: + if "llama_backend" in block and "native_context_length" in block: + found = True + break + assert found, "No InferenceStatusResponse block with llama_backend has native_context_length" + + +# ===================================================================== +# E. TestEdgeCases +# ===================================================================== + + +class TestNativeContextEdgeCases: + """Edge cases for native_context_length.""" + + def test_context_length_zero(self, tmp_path, backend): + """GGUF context_length=0 returns 0, not None.""" + path = make_gguf(tmp_path, "llama", [("context_length", 0, 4)]) + backend._read_gguf_metadata(path) + assert backend.native_context_length == 0 + + def test_context_length_uint32_max(self, tmp_path, backend): + """2^32 - 1 survives without truncation.""" + val = 2**32 - 1 + path = make_gguf(tmp_path, "llama", [("context_length", val, 4)]) + backend._read_gguf_metadata(path) + assert backend.native_context_length == val + + def test_context_length_uint64(self, tmp_path, backend): + """UINT64 type context_length parsed correctly.""" + val = 2**33 # exceeds UINT32 range + path = make_gguf(tmp_path, "llama", [("context_length", val, 10)]) + backend._read_gguf_metadata(path) + assert backend.native_context_length == val + + def test_no_context_length_in_gguf(self, tmp_path, backend): + """GGUF without context_length key yields None.""" + path = make_gguf(tmp_path, "llama", [("block_count", 32, 4)]) + backend._read_gguf_metadata(path) + assert backend.native_context_length is None + + def test_native_equals_context_when_uncapped(self, backend): + """Both equal when no VRAM cap applied.""" + backend._context_length = 8192 + assert backend.native_context_length == backend.context_length + + def test_native_survives_parse_then_cap(self, tmp_path, backend): + """Parse then set effective cap: native unchanged.""" + path = make_gguf( + tmp_path, + "llama", + [ + ("context_length", 131072, 4), + ("block_count", 32, 4), + ("attention.head_count", 32, 4), + ("attention.head_count_kv", 8, 4), + ("embedding_length", 4096, 4), + ], + ) + backend._read_gguf_metadata(path) + assert backend.native_context_length == 131072 + + # Simulate VRAM capping by setting effective and max + backend._effective_context_length = 16384 + backend._max_context_length = 32768 + assert backend.native_context_length == 131072 + + +# ===================================================================== +# F. TestCrossPlatform -- binary I/O and serialization +# ===================================================================== + + +class TestCrossPlatform: + """Binary I/O and serialization correctness across platforms.""" + + def test_le_uint32_context_length(self, tmp_path, backend): + """Little-endian UINT32 parsed correctly.""" + path = make_gguf(tmp_path, "llama", [("context_length", 16384, 4)]) + backend._read_gguf_metadata(path) + assert backend.native_context_length == 16384 + + def test_le_uint64_context_length(self, tmp_path, backend): + """Little-endian UINT64 parsed correctly.""" + path = make_gguf(tmp_path, "llama", [("context_length", 16384, 10)]) + backend._read_gguf_metadata(path) + assert backend.native_context_length == 16384 + + def test_gguf_magic_le_byte_order(self, tmp_path): + """Magic 0x46554747 matches GGUF spec (little-endian 'GGUF').""" + path = tmp_path / "magic_check.gguf" + buf = io.BytesIO() + buf.write(struct.pack(" s.ggufMaxContextLength, ); + const ggufNativeContextLength = useChatRuntimeStore( + (s) => s.ggufNativeContextLength, + ); const kvCacheDtype = useChatRuntimeStore((s) => s.kvCacheDtype); const setKvCacheDtype = useChatRuntimeStore((s) => s.setKvCacheDtype); const loadedKvCacheDtype = useChatRuntimeStore((s) => s.loadedKvCacheDtype); @@ -293,7 +296,7 @@ export function ChatSettingsPanel({ ); const ctxDisplayValue = customContextLength ?? ggufContextLength ?? ""; - const ctxMaxValue = ggufMaxContextLength ?? ggufContextLength ?? null; + const ctxMaxValue = ggufNativeContextLength ?? ggufContextLength ?? null; const kvDirty = kvCacheDtype !== loadedKvCacheDtype; const ctxDirty = customContextLength !== null; const modelSettingsDirty = kvDirty || ctxDirty; @@ -544,6 +547,13 @@ export function ChatSettingsPanel({ ); }} /> + {ggufMaxContextLength != null && + typeof ctxDisplayValue === "number" && + ctxDisplayValue > ggufMaxContextLength && ( +

+ Exceeds estimated VRAM capacity ({ggufMaxContextLength.toLocaleString()} tokens). The model may use system RAM. +

+ )}
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts index 2c585a18b8..fc0e392596 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts @@ -246,12 +246,16 @@ export function useChatModelRuntime() { const ggufMaxContextLength = statusRes.is_gguf ? (statusRes.max_context_length ?? null) : null; + const ggufNativeContextLength = statusRes.is_gguf + ? (statusRes.native_context_length ?? null) + : null; useChatRuntimeStore.setState({ supportsReasoning, reasoningAlwaysOn, supportsTools, ggufContextLength: currentGgufContextLength, ggufMaxContextLength, + ggufNativeContextLength, }); // Set reasoning default for Qwen3.5 small models @@ -425,6 +429,9 @@ export function useChatModelRuntime() { const reportedMaxCtx = loadResponse.is_gguf ? (loadResponse.max_context_length ?? null) : null; + const reportedNativeCtx = loadResponse.is_gguf + ? (loadResponse.native_context_length ?? null) + : null; // A successful reload has applied settings, so clear pending custom // context state and display the backend-reported effective context. const keepCustomCtx = null; @@ -433,6 +440,7 @@ export function useChatModelRuntime() { useChatRuntimeStore.setState({ ggufContextLength: nativeCtx, ggufMaxContextLength, + ggufNativeContextLength: reportedNativeCtx, supportsReasoning: loadResponse.supports_reasoning ?? false, reasoningAlwaysOn, reasoningEnabled: reasoningAlwaysOn ? true : reasoningDefault, diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts index 8cea234f21..48abaf7580 100644 --- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts +++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts @@ -151,6 +151,7 @@ type ChatRuntimeStore = { activeGgufVariant: string | null; ggufContextLength: number | null; ggufMaxContextLength: number | null; + ggufNativeContextLength: number | null; supportsReasoning: boolean; reasoningAlwaysOn: boolean; reasoningEnabled: boolean; @@ -215,6 +216,7 @@ export const useChatRuntimeStore = create((set) => ({ activeGgufVariant: null, ggufContextLength: null, ggufMaxContextLength: null, + ggufNativeContextLength: null, supportsReasoning: false, reasoningAlwaysOn: false, reasoningEnabled: true, @@ -290,6 +292,7 @@ export const useChatRuntimeStore = create((set) => ({ activeGgufVariant: null, ggufContextLength: null, ggufMaxContextLength: null, + ggufNativeContextLength: null, contextUsage: null, supportsReasoning: false, reasoningEnabled: true, diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index dcc0a980c8..8f0839615f 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -87,6 +87,7 @@ export interface LoadModelResponse { }; context_length?: number | null; max_context_length?: number | null; + native_context_length?: number | null; supports_reasoning?: boolean; reasoning_always_on?: boolean; supports_tools?: boolean; @@ -121,6 +122,7 @@ export interface InferenceStatusResponse { supports_tools?: boolean; context_length?: number | null; max_context_length?: number | null; + native_context_length?: number | null; } export interface AudioGenerationResponse {