ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(studio): preserve GGUF context max after apply and refresh (#4691)

Browse source 
Fixes #4670

Separates the GGUF context slider ceiling from the currently active context length so lowering context via Chat Settings no longer locks the slider max to the reduced value.

- Backend: adds `max_context_length` to GGUF load/status responses, computed from the largest VRAM/KV-fit cap across all usable GPU subsets
- Frontend: stores `ggufMaxContextLength` and uses it for Context Length slider/input bounds; hydrates from both `/api/inference/load` and `/api/inference/status`
- Defaults UI ceiling to native context for CPU-only and fallback paths
- Seeds `effective_ctx` and `max_available_ctx` before GPU probing to prevent `UnboundLocalError` on probe failure
- Property fallback uses native `_context_length`, not effective `context_length`
This commit is contained in:
Lee Jackson 2026-03-30 09:33:16 +01:00 • committed by GitHub
commit 2f0a5baa87
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 84 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

48
studio/backend/core/inference/llama_cpp.py
View file

@ -48,6 +48,7 @@ class LlamaCppBackend:
self._healthy = False
self._context_length: Optional[int] = None
self._effective_context_length: Optional[int] = None
self._max_context_length: Optional[int] = None
self._chat_template: Optional[str] = None
self._supports_reasoning: bool = False
self._reasoning_always_on: bool = False
@ -100,6 +101,11 @@ class LlamaCppBackend:
"""Return the effective context length the server is running at."""
return self._effective_context_length or self._context_length
@property
def max_context_length(self) -> Optional[int]:
"""Return the maximum context currently available on this hardware."""
return self._max_context_length or self._context_length
@property
def chat_template(self) -> Optional[str]:
return self._chat_template
@ -960,7 +966,11 @@ class LlamaCppBackend:
self._port = self._find_free_port()
# Select GPU(s) based on model size + estimated KV cache
# Select GPU(s) based on model size + estimated KV cache.
# Seed safe defaults before GPU probing so the except path
# still has valid state to publish.
effective_ctx = n_ctx if n_ctx > 0 else (self._context_length or 0)
max_available_ctx = self._context_length or effective_ctx
try:
model_size = self._get_gguf_size_bytes(model_path)
gpus = self._get_gpu_free_memory()
@ -975,6 +985,9 @@ class LlamaCppBackend:
else:
effective_ctx = 0
original_ctx = effective_ctx
# Default UI ceiling to the model's native context length.
# GPU/VRAM-fit logic below may shrink this if hardware is limited.
max_available_ctx = self._context_length or effective_ctx
# Auto-cap context to fit in GPU VRAM and select GPUs.
#
@ -993,6 +1006,29 @@ class LlamaCppBackend:
explicit_ctx = n_ctx > 0
if gpus and self._can_estimate_kv() and effective_ctx > 0:
# Compute the largest hardware-aware cap from the model's
# native context across all usable GPU subsets (for UI
# bounds), independent of the currently requested context.
native_ctx_for_cap = self._context_length or effective_ctx
if native_ctx_for_cap > 0:
ranked_for_cap = sorted(gpus, key = lambda g: g[1], reverse = True)
best_cap = 0
for n_gpus in range(1, len(ranked_for_cap) + 1):
subset = ranked_for_cap[:n_gpus]
pool_mib = sum(free for _, free in subset)
capped = self._fit_context_to_vram(
native_ctx_for_cap,
pool_mib,
model_size,
cache_type_kv,
)
kv = self._estimate_kv_cache_bytes(capped, cache_type_kv)
total_mib = (model_size + kv) / (1024 * 1024)
if total_mib <= pool_mib * 0.70:
best_cap = max(best_cap, capped)
if best_cap > 0:
max_available_ctx = best_cap
if explicit_ctx:
# Try to honor the user's requested context exactly.
requested_total = model_size + self._estimate_kv_cache_bytes(
@ -1043,7 +1079,9 @@ class LlamaCppBackend:
break
elif gpus:
# Can't estimate KV -- fall back to file-size-only check
# Can't estimate KV -- fall back to file-size-only check.
# Without KV estimation we cannot prove a hardware cap, so
# keep the ceiling at the native context (already the default).
gpu_indices, use_fit = self._select_gpus(model_size, gpus)
if effective_ctx < original_ctx:
@ -1313,6 +1351,11 @@ class LlamaCppBackend:
self._effective_context_length = (
effective_ctx if effective_ctx > 0 else self._context_length
)
self._max_context_length = (
max_available_ctx
if max_available_ctx > 0
else self._effective_context_length
)
# Wait for llama-server to become healthy
if not self._wait_for_health(timeout = 600.0):
@ -1347,6 +1390,7 @@ class LlamaCppBackend:
self._healthy = False
self._context_length = None
self._effective_context_length = None
self._max_context_length = None
self._chat_template = None
self._supports_reasoning = False
self._reasoning_always_on = False

7
studio/backend/models/inference.py
View file

@ -132,6 +132,9 @@ class LoadResponse(BaseModel):
context_length: Optional[int] = Field(
None, description = "Model's native context length (from GGUF metadata)"
)
max_context_length: Optional[int] = Field(
None, description = "Maximum context length currently available on this hardware"
)
supports_reasoning: bool = Field(
False,
description = "Whether model supports thinking/reasoning mode (enable_thinking)",
@ -206,6 +209,10 @@ class InferenceStatusResponse(BaseModel):
context_length: Optional[int] = Field(
None, description = "Context length of the active model"
)
max_context_length: Optional[int] = Field(
None,
description = "Maximum context length currently available for the active model",
)
# =====================================================================

3
studio/backend/routes/inference.py
View file

@ -155,6 +155,7 @@ async def load_model(
else False,
inference = inference_config,
context_length = llama_backend.context_length,
max_context_length = llama_backend.max_context_length,
supports_reasoning = llama_backend.supports_reasoning,
reasoning_always_on = llama_backend.reasoning_always_on,
chat_template = llama_backend.chat_template,
@ -280,6 +281,7 @@ async def load_model(
has_audio_input = is_audio_input_type(_gguf_audio),
inference = inference_config,
context_length = llama_backend.context_length,
max_context_length = llama_backend.max_context_length,
supports_reasoning = llama_backend.supports_reasoning,
reasoning_always_on = llama_backend.reasoning_always_on,
supports_tools = llama_backend.supports_tools,
@ -614,6 +616,7 @@ async def get_status(
reasoning_always_on = llama_backend.reasoning_always_on,
supports_tools = llama_backend.supports_tools,
context_length = llama_backend.context_length,
max_context_length = llama_backend.max_context_length,
)
# Otherwise, report Unsloth backend status

2
studio/frontend/src/features/chat/api/chat-adapter.ts
View file

@ -306,6 +306,7 @@ async function autoLoadSmallestModel(): Promise<boolean> {
}
useChatRuntimeStore.setState({
ggufContextLength: loadResp.context_length ?? 131072,
ggufMaxContextLength: loadResp.max_context_length ?? loadResp.context_length ?? 131072,
supportsReasoning: loadResp.supports_reasoning ?? false,
reasoningAlwaysOn: loadResp.reasoning_always_on ?? false,
reasoningEnabled: loadResp.supports_reasoning ?? false,
@ -392,6 +393,7 @@ async function autoLoadSmallestModel(): Promise<boolean> {
}
useChatRuntimeStore.setState({
ggufContextLength: loadResp.context_length ?? 131072,
ggufMaxContextLength: loadResp.max_context_length ?? loadResp.context_length ?? 131072,
supportsReasoning: loadResp.supports_reasoning ?? false,
reasoningAlwaysOn: loadResp.reasoning_always_on ?? false,
reasoningEnabled: loadResp.supports_reasoning ?? false,

10
studio/frontend/src/features/chat/chat-settings-sheet.tsx
View file

@ -277,6 +277,7 @@ export function ChatSettingsPanel({
const isMobile = useIsMobile();
const isGguf = useChatRuntimeStore((s) => s.activeGgufVariant) != null;
const ggufContextLength = useChatRuntimeStore((s) => s.ggufContextLength);
const ggufMaxContextLength = useChatRuntimeStore((s) => s.ggufMaxContextLength);
const kvCacheDtype = useChatRuntimeStore((s) => s.kvCacheDtype);
const setKvCacheDtype = useChatRuntimeStore((s) => s.setKvCacheDtype);
const loadedKvCacheDtype = useChatRuntimeStore((s) => s.loadedKvCacheDtype);
@ -284,6 +285,7 @@ export function ChatSettingsPanel({
const setCustomContextLength = useChatRuntimeStore((s) => s.setCustomContextLength);
const ctxDisplayValue = customContextLength ?? ggufContextLength ?? "";
const ctxMaxValue = ggufMaxContextLength ?? ggufContextLength ?? null;
const kvDirty = kvCacheDtype !== loadedKvCacheDtype;
const ctxDirty = customContextLength !== null;
const modelSettingsDirty = kvDirty || ctxDirty;
@ -483,7 +485,7 @@ export function ChatSettingsPanel({
value={typeof ctxDisplayValue === "number" ? ctxDisplayValue : (ggufContextLength ?? "")}
placeholder="..."
min={128}
max={ggufContextLength ?? undefined}
max={ctxMaxValue ?? undefined}
step={1024}
className="h-6 w-[100px] text-right text-xs tabular-nums"
onChange={(e) => {
@ -494,7 +496,7 @@ export function ChatSettingsPanel({
}
const v = parseInt(raw, 10);
if (!Number.isNaN(v) && v >= 0) {
const maxCtx = ggufContextLength ?? Infinity;
const maxCtx = ctxMaxValue ?? Infinity;
const clamped = Math.min(v, maxCtx);
setCustomContextLength(clamped === (ggufContextLength ?? 0) ? null : clamped);
}
@ -503,9 +505,9 @@ export function ChatSettingsPanel({
</div>
<Slider
min={1024}
max={ggufContextLength ?? 4096}
max={ctxMaxValue ?? 4096}
step={1024}
value={[Math.min(typeof ctxDisplayValue === "number" ? ctxDisplayValue : (ggufContextLength ?? 4096), ggufContextLength ?? 4096)]}
value={[Math.min(typeof ctxDisplayValue === "number" ? ctxDisplayValue : (ggufContextLength ?? 4096), ctxMaxValue ?? 4096)]}
onValueChange={([v]) => {
setCustomContextLength(v === (ggufContextLength ?? 0) ? null : v);
}}

22
studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
View file

@ -240,11 +240,18 @@ export function useChatModelRuntime() {
const supportsReasoning = statusRes.supports_reasoning ?? false;
const reasoningAlwaysOn = statusRes.reasoning_always_on ?? false;
const supportsTools = statusRes.supports_tools ?? false;
const currentGgufContextLength = statusRes.is_gguf
? (statusRes.context_length ?? null)
: null;
const ggufMaxContextLength = statusRes.is_gguf
? (statusRes.max_context_length ?? null)
: null;
useChatRuntimeStore.setState({
supportsReasoning,
reasoningAlwaysOn,
supportsTools,
ggufContextLength: statusRes.is_gguf ? (statusRes.context_length ?? null) : null,
ggufContextLength: currentGgufContextLength,
ggufMaxContextLength,
});
// Set reasoning default for Qwen3.5 small models
@ -415,16 +422,17 @@ export function useChatModelRuntime() {
const nativeCtx = loadResponse.is_gguf
? (loadResponse.context_length ?? 131072)
: null;
// Keep customContextLength if the user set one and it differs
// from the model's native context; otherwise clear it so the
// display shows the native value without a dirty marker.
const keepCustomCtx = customContextLength != null
&& customContextLength !== nativeCtx
? customContextLength
const reportedMaxCtx = loadResponse.is_gguf
? (loadResponse.max_context_length ?? null)
: null;
// A successful reload has applied settings, so clear pending custom
// context state and display the backend-reported effective context.
const keepCustomCtx = null;
const reasoningAlwaysOn = loadResponse.reasoning_always_on ?? false;
const ggufMaxContextLength = reportedMaxCtx;
useChatRuntimeStore.setState({
ggufContextLength: nativeCtx,
ggufMaxContextLength,
supportsReasoning: loadResponse.supports_reasoning ?? false,
reasoningAlwaysOn,
reasoningEnabled: reasoningAlwaysOn ? true : reasoningDefault,

3
studio/frontend/src/features/chat/stores/chat-runtime-store.ts
View file

@ -150,6 +150,7 @@ type ChatRuntimeStore = {
modelsError: string | null;
activeGgufVariant: string | null;
ggufContextLength: number | null;
ggufMaxContextLength: number | null;
supportsReasoning: boolean;
reasoningAlwaysOn: boolean;
reasoningEnabled: boolean;
@ -213,6 +214,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set) => ({
modelsError: null,
activeGgufVariant: null,
ggufContextLength: null,
ggufMaxContextLength: null,
supportsReasoning: false,
reasoningAlwaysOn: false,
reasoningEnabled: true,
@ -287,6 +289,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set) => ({
},
activeGgufVariant: null,
ggufContextLength: null,
ggufMaxContextLength: null,
contextUsage: null,
supportsReasoning: false,
reasoningEnabled: true,

2
studio/frontend/src/features/chat/types/api.ts
View file

@ -86,6 +86,7 @@ export interface LoadModelResponse {
trust_remote_code?: boolean;
};
context_length?: number | null;
max_context_length?: number | null;
supports_reasoning?: boolean;
reasoning_always_on?: boolean;
supports_tools?: boolean;
@ -119,6 +120,7 @@ export interface InferenceStatusResponse {
reasoning_always_on?: boolean;
supports_tools?: boolean;
context_length?: number | null;
max_context_length?: number | null;
}
export interface AudioGenerationResponse {