diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 05e038dbb7..ca39054ec0 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -48,6 +48,7 @@ class LlamaCppBackend: self._healthy = False self._context_length: Optional[int] = None self._effective_context_length: Optional[int] = None + self._max_context_length: Optional[int] = None self._chat_template: Optional[str] = None self._supports_reasoning: bool = False self._reasoning_always_on: bool = False @@ -100,6 +101,11 @@ class LlamaCppBackend: """Return the effective context length the server is running at.""" return self._effective_context_length or self._context_length + @property + def max_context_length(self) -> Optional[int]: + """Return the maximum context currently available on this hardware.""" + return self._max_context_length or self._context_length + @property def chat_template(self) -> Optional[str]: return self._chat_template @@ -960,7 +966,11 @@ class LlamaCppBackend: self._port = self._find_free_port() - # Select GPU(s) based on model size + estimated KV cache + # Select GPU(s) based on model size + estimated KV cache. + # Seed safe defaults before GPU probing so the except path + # still has valid state to publish. + effective_ctx = n_ctx if n_ctx > 0 else (self._context_length or 0) + max_available_ctx = self._context_length or effective_ctx try: model_size = self._get_gguf_size_bytes(model_path) gpus = self._get_gpu_free_memory() @@ -975,6 +985,9 @@ class LlamaCppBackend: else: effective_ctx = 0 original_ctx = effective_ctx + # Default UI ceiling to the model's native context length. + # GPU/VRAM-fit logic below may shrink this if hardware is limited. + max_available_ctx = self._context_length or effective_ctx # Auto-cap context to fit in GPU VRAM and select GPUs. # @@ -993,6 +1006,29 @@ class LlamaCppBackend: explicit_ctx = n_ctx > 0 if gpus and self._can_estimate_kv() and effective_ctx > 0: + # Compute the largest hardware-aware cap from the model's + # native context across all usable GPU subsets (for UI + # bounds), independent of the currently requested context. + native_ctx_for_cap = self._context_length or effective_ctx + if native_ctx_for_cap > 0: + ranked_for_cap = sorted(gpus, key = lambda g: g[1], reverse = True) + best_cap = 0 + for n_gpus in range(1, len(ranked_for_cap) + 1): + subset = ranked_for_cap[:n_gpus] + pool_mib = sum(free for _, free in subset) + capped = self._fit_context_to_vram( + native_ctx_for_cap, + pool_mib, + model_size, + cache_type_kv, + ) + kv = self._estimate_kv_cache_bytes(capped, cache_type_kv) + total_mib = (model_size + kv) / (1024 * 1024) + if total_mib <= pool_mib * 0.70: + best_cap = max(best_cap, capped) + if best_cap > 0: + max_available_ctx = best_cap + if explicit_ctx: # Try to honor the user's requested context exactly. requested_total = model_size + self._estimate_kv_cache_bytes( @@ -1043,7 +1079,9 @@ class LlamaCppBackend: break elif gpus: - # Can't estimate KV -- fall back to file-size-only check + # Can't estimate KV -- fall back to file-size-only check. + # Without KV estimation we cannot prove a hardware cap, so + # keep the ceiling at the native context (already the default). gpu_indices, use_fit = self._select_gpus(model_size, gpus) if effective_ctx < original_ctx: @@ -1313,6 +1351,11 @@ class LlamaCppBackend: self._effective_context_length = ( effective_ctx if effective_ctx > 0 else self._context_length ) + self._max_context_length = ( + max_available_ctx + if max_available_ctx > 0 + else self._effective_context_length + ) # Wait for llama-server to become healthy if not self._wait_for_health(timeout = 600.0): @@ -1347,6 +1390,7 @@ class LlamaCppBackend: self._healthy = False self._context_length = None self._effective_context_length = None + self._max_context_length = None self._chat_template = None self._supports_reasoning = False self._reasoning_always_on = False diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index accdcc1290..a20c2052aa 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -132,6 +132,9 @@ class LoadResponse(BaseModel): context_length: Optional[int] = Field( None, description = "Model's native context length (from GGUF metadata)" ) + max_context_length: Optional[int] = Field( + None, description = "Maximum context length currently available on this hardware" + ) supports_reasoning: bool = Field( False, description = "Whether model supports thinking/reasoning mode (enable_thinking)", @@ -206,6 +209,10 @@ class InferenceStatusResponse(BaseModel): context_length: Optional[int] = Field( None, description = "Context length of the active model" ) + max_context_length: Optional[int] = Field( + None, + description = "Maximum context length currently available for the active model", + ) # ===================================================================== diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 6f44a3c69f..7d48198d42 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -155,6 +155,7 @@ async def load_model( else False, inference = inference_config, context_length = llama_backend.context_length, + max_context_length = llama_backend.max_context_length, supports_reasoning = llama_backend.supports_reasoning, reasoning_always_on = llama_backend.reasoning_always_on, chat_template = llama_backend.chat_template, @@ -280,6 +281,7 @@ async def load_model( has_audio_input = is_audio_input_type(_gguf_audio), inference = inference_config, context_length = llama_backend.context_length, + max_context_length = llama_backend.max_context_length, supports_reasoning = llama_backend.supports_reasoning, reasoning_always_on = llama_backend.reasoning_always_on, supports_tools = llama_backend.supports_tools, @@ -614,6 +616,7 @@ async def get_status( reasoning_always_on = llama_backend.reasoning_always_on, supports_tools = llama_backend.supports_tools, context_length = llama_backend.context_length, + max_context_length = llama_backend.max_context_length, ) # Otherwise, report Unsloth backend status diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index ab4cdd9d6a..2b8a259930 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -306,6 +306,7 @@ async function autoLoadSmallestModel(): Promise { } useChatRuntimeStore.setState({ ggufContextLength: loadResp.context_length ?? 131072, + ggufMaxContextLength: loadResp.max_context_length ?? loadResp.context_length ?? 131072, supportsReasoning: loadResp.supports_reasoning ?? false, reasoningAlwaysOn: loadResp.reasoning_always_on ?? false, reasoningEnabled: loadResp.supports_reasoning ?? false, @@ -392,6 +393,7 @@ async function autoLoadSmallestModel(): Promise { } useChatRuntimeStore.setState({ ggufContextLength: loadResp.context_length ?? 131072, + ggufMaxContextLength: loadResp.max_context_length ?? loadResp.context_length ?? 131072, supportsReasoning: loadResp.supports_reasoning ?? false, reasoningAlwaysOn: loadResp.reasoning_always_on ?? false, reasoningEnabled: loadResp.supports_reasoning ?? false, diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index a315aeb005..3f3557b34f 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -277,6 +277,7 @@ export function ChatSettingsPanel({ const isMobile = useIsMobile(); const isGguf = useChatRuntimeStore((s) => s.activeGgufVariant) != null; const ggufContextLength = useChatRuntimeStore((s) => s.ggufContextLength); + const ggufMaxContextLength = useChatRuntimeStore((s) => s.ggufMaxContextLength); const kvCacheDtype = useChatRuntimeStore((s) => s.kvCacheDtype); const setKvCacheDtype = useChatRuntimeStore((s) => s.setKvCacheDtype); const loadedKvCacheDtype = useChatRuntimeStore((s) => s.loadedKvCacheDtype); @@ -284,6 +285,7 @@ export function ChatSettingsPanel({ const setCustomContextLength = useChatRuntimeStore((s) => s.setCustomContextLength); const ctxDisplayValue = customContextLength ?? ggufContextLength ?? ""; + const ctxMaxValue = ggufMaxContextLength ?? ggufContextLength ?? null; const kvDirty = kvCacheDtype !== loadedKvCacheDtype; const ctxDirty = customContextLength !== null; const modelSettingsDirty = kvDirty || ctxDirty; @@ -483,7 +485,7 @@ export function ChatSettingsPanel({ value={typeof ctxDisplayValue === "number" ? ctxDisplayValue : (ggufContextLength ?? "")} placeholder="..." min={128} - max={ggufContextLength ?? undefined} + max={ctxMaxValue ?? undefined} step={1024} className="h-6 w-[100px] text-right text-xs tabular-nums" onChange={(e) => { @@ -494,7 +496,7 @@ export function ChatSettingsPanel({ } const v = parseInt(raw, 10); if (!Number.isNaN(v) && v >= 0) { - const maxCtx = ggufContextLength ?? Infinity; + const maxCtx = ctxMaxValue ?? Infinity; const clamped = Math.min(v, maxCtx); setCustomContextLength(clamped === (ggufContextLength ?? 0) ? null : clamped); } @@ -503,9 +505,9 @@ export function ChatSettingsPanel({ { setCustomContextLength(v === (ggufContextLength ?? 0) ? null : v); }} diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts index cc9ec3971d..2c585a18b8 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts @@ -240,11 +240,18 @@ export function useChatModelRuntime() { const supportsReasoning = statusRes.supports_reasoning ?? false; const reasoningAlwaysOn = statusRes.reasoning_always_on ?? false; const supportsTools = statusRes.supports_tools ?? false; + const currentGgufContextLength = statusRes.is_gguf + ? (statusRes.context_length ?? null) + : null; + const ggufMaxContextLength = statusRes.is_gguf + ? (statusRes.max_context_length ?? null) + : null; useChatRuntimeStore.setState({ supportsReasoning, reasoningAlwaysOn, supportsTools, - ggufContextLength: statusRes.is_gguf ? (statusRes.context_length ?? null) : null, + ggufContextLength: currentGgufContextLength, + ggufMaxContextLength, }); // Set reasoning default for Qwen3.5 small models @@ -415,16 +422,17 @@ export function useChatModelRuntime() { const nativeCtx = loadResponse.is_gguf ? (loadResponse.context_length ?? 131072) : null; - // Keep customContextLength if the user set one and it differs - // from the model's native context; otherwise clear it so the - // display shows the native value without a dirty marker. - const keepCustomCtx = customContextLength != null - && customContextLength !== nativeCtx - ? customContextLength + const reportedMaxCtx = loadResponse.is_gguf + ? (loadResponse.max_context_length ?? null) : null; + // A successful reload has applied settings, so clear pending custom + // context state and display the backend-reported effective context. + const keepCustomCtx = null; const reasoningAlwaysOn = loadResponse.reasoning_always_on ?? false; + const ggufMaxContextLength = reportedMaxCtx; useChatRuntimeStore.setState({ ggufContextLength: nativeCtx, + ggufMaxContextLength, supportsReasoning: loadResponse.supports_reasoning ?? false, reasoningAlwaysOn, reasoningEnabled: reasoningAlwaysOn ? true : reasoningDefault, diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts index 35a7ab068b..ca1044b3dc 100644 --- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts +++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts @@ -150,6 +150,7 @@ type ChatRuntimeStore = { modelsError: string | null; activeGgufVariant: string | null; ggufContextLength: number | null; + ggufMaxContextLength: number | null; supportsReasoning: boolean; reasoningAlwaysOn: boolean; reasoningEnabled: boolean; @@ -213,6 +214,7 @@ export const useChatRuntimeStore = create((set) => ({ modelsError: null, activeGgufVariant: null, ggufContextLength: null, + ggufMaxContextLength: null, supportsReasoning: false, reasoningAlwaysOn: false, reasoningEnabled: true, @@ -287,6 +289,7 @@ export const useChatRuntimeStore = create((set) => ({ }, activeGgufVariant: null, ggufContextLength: null, + ggufMaxContextLength: null, contextUsage: null, supportsReasoning: false, reasoningEnabled: true, diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index f41f1279a7..dcc0a980c8 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -86,6 +86,7 @@ export interface LoadModelResponse { trust_remote_code?: boolean; }; context_length?: number | null; + max_context_length?: number | null; supports_reasoning?: boolean; reasoning_always_on?: boolean; supports_tools?: boolean; @@ -119,6 +120,7 @@ export interface InferenceStatusResponse { reasoning_always_on?: boolean; supports_tools?: boolean; context_length?: number | null; + max_context_length?: number | null; } export interface AudioGenerationResponse {