Studio: add 'Load on selection' toggle to configure load options before loading (#6348)

* Studio: add 'Load on selection' toggle to configure load options before loading

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Studio: seed staged speculative decoding from the standing default

* Studio: address PR review for load-on-selection staging

* Studio: handle direct GGUF staging and stale-stage edge cases from load-on-selection review

* Studio: cancel replaced staged downloads and keep staged pick on load failure

* Studio: centralize staged-download cancel and guard staged-load restore

* fix: address staged GGUF load review

* fix: honor staged GGUF load metadata

* fix: clarify load-on-selection tooltip

Keep the load-on-selection hint visually anchored to the control and make the on/off behavior explicit without changing the broader deferred-load flow.

* Studio: reset orphaned staged knobs on abandon and cap Max Tokens to staged context

* Studio: remove dead code and cancel staged download when loading a different model

* fix: surface staged model in run settings before deferred load

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Lee Jackson <130007945+Imagineer99@users.noreply.github.com>
Co-authored-by: imagineer99 <samleejackson0@gmail.com>
This commit is contained in:
oobabooga 2026-06-17 12:24:10 -03:00 committed by GitHub
commit c6cf53759b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
23 changed files with 948 additions and 81 deletions

View file

@ -353,6 +353,20 @@ def list_partial_gguf_variants_from_state(
return variants, has_vision
def resolve_local_gguf_path(repo_id: str, gguf_variant: Optional[str]) -> Optional[str]:
"""Absolute path to the (shard-1) GGUF file for ``repo_id`` + ``gguf_variant``
if it is already downloaded in the HF cache, else ``None``. Read-only never
triggers a download. Lets callers read header metadata before a load."""
for snapshot in iter_hf_cache_snapshots(repo_id):
variants, _ = list_local_gguf_variants(str(snapshot))
for variant in variants:
if gguf_variant is None or variant.quant == gguf_variant:
candidate = snapshot / variant.filename
if candidate.is_file():
return str(candidate)
return None
def list_gguf_variants(
repo_id: str, hf_token: Optional[str] = None
) -> tuple[list[GgufVariantInfo], bool, Optional[list]]:

View file

@ -125,6 +125,11 @@ class ValidateModelRequest(BaseModel):
gguf_variant: Optional[str] = Field(
None, description = "GGUF quantization variant (e.g. 'Q4_K_M')"
)
include_context_length: bool = Field(
False,
description = "Also read the native context length from the local GGUF header. "
"Opt-in so the normal load preflight doesn't pay for a cache scan it doesn't need.",
)
class ValidateModelResponse(BaseModel):
@ -144,6 +149,11 @@ class ValidateModelResponse(BaseModel):
False,
description = "Whether the model defaults require trust_remote_code to be enabled for loading.",
)
context_length: Optional[int] = Field(
None,
description = "Native training context length, read from the GGUF header when the file "
"is already downloaded locally; None for non-GGUF, gated, or not-yet-downloaded models.",
)
class GenerateRequest(BaseModel):

View file

@ -2536,6 +2536,33 @@ async def validate_model(
detail = f"Invalid model identifier: {model_log_label}",
)
is_gguf = getattr(config, "is_gguf", False)
# Native context length, read from the local GGUF header when present.
# Lets the staged ("Load on selection" off) flow populate the context
# slider before the GPU load; None until the file is downloaded.
context_length: Optional[int] = None
if request.include_context_length and is_gguf:
from hub.utils.gguf import resolve_local_gguf_path
from utils.models.gguf_metadata import read_gguf_context_length
# Best-effort: a header-read failure must never fail validation of an
# otherwise-valid model (the outer except turns it into a 400).
try:
if native_grant_backed:
# model_identifier is the resolved canonical .gguf path.
local_gguf = model_identifier
else:
# Local folder / exported GGUFs already have their file
# resolved on the config (gguf_file is None for HF repos, so
# those fall back to the HF-cache lookup).
local_gguf = config.gguf_file or resolve_local_gguf_path(
model_identifier, request.gguf_variant
)
if local_gguf:
context_length = read_gguf_context_length(local_gguf)
except Exception as e:
logger.debug("Context-length probe failed for %s: %s", model_log_label, e)
return ValidateModelResponse(
valid = True,
message = "Model identifier is valid.",
@ -2543,12 +2570,13 @@ async def validate_model(
display_name = model_log_label
if native_grant_backed
else getattr(config, "display_name", config.identifier),
is_gguf = getattr(config, "is_gguf", False),
is_gguf = is_gguf,
is_lora = getattr(config, "is_lora", False),
is_vision = getattr(config, "is_vision", False),
requires_trust_remote_code = bool(
load_inference_config(config.identifier).get("trust_remote_code", False)
),
context_length = context_length,
)
except HTTPException:

View file

@ -13,6 +13,7 @@ from typing import Iterable, Mapping
from utils.models.gguf_metadata import (
is_mmproj_by_metadata,
pairing_score,
read_gguf_context_length,
read_gguf_general_metadata,
read_mmproj_audio_capability,
)
@ -21,6 +22,7 @@ from utils.models.gguf_metadata import (
_GGUF_MAGIC = 0x46554747
_VTYPE_STRING = 8
_VTYPE_UINT32 = 4
_VTYPE_UINT64 = 10
_VTYPE_ARRAY = 9
_VTYPE_BOOL = 7
@ -38,6 +40,10 @@ def _enc_kv_uint32(key: str, value: int) -> bytes:
return _enc_string(key) + struct.pack("<I", _VTYPE_UINT32) + struct.pack("<I", value)
def _enc_kv_uint64(key: str, value: int) -> bytes:
return _enc_string(key) + struct.pack("<I", _VTYPE_UINT64) + struct.pack("<Q", value)
def _enc_kv_bool(key: str, value: bool) -> bytes:
return _enc_string(key) + struct.pack("<I", _VTYPE_BOOL) + struct.pack("<B", 1 if value else 0)
@ -56,21 +62,29 @@ def _write_synthetic_gguf(
general_strings: Mapping[str, str],
*,
extra_uint32: Mapping[str, int] | None = None,
extra_uint64: Mapping[str, int] | None = None,
extra_string_arrays: Mapping[str, Iterable[str]] | None = None,
extra_bools: Mapping[str, bool] | None = None,
) -> Path:
"""Minimal GGUF: header + KV body, no tensors."""
extra_uint32 = extra_uint32 or {}
extra_uint64 = extra_uint64 or {}
extra_string_arrays = extra_string_arrays or {}
extra_bools = extra_bools or {}
kv_count = (
len(general_strings) + len(extra_uint32) + len(extra_string_arrays) + len(extra_bools)
len(general_strings)
+ len(extra_uint32)
+ len(extra_uint64)
+ len(extra_string_arrays)
+ len(extra_bools)
)
body = b""
for k, v in general_strings.items():
body += _enc_kv_string(k, v)
for k, v in extra_uint32.items():
body += _enc_kv_uint32(k, v)
for k, v in extra_uint64.items():
body += _enc_kv_uint64(k, v)
for k, v in extra_string_arrays.items():
body += _enc_kv_string_array(k, v)
for k, v in extra_bools.items():
@ -100,6 +114,66 @@ def test_returns_none_for_non_gguf(tmp_path: Path):
assert read_gguf_general_metadata(str(p)) is None
def test_context_length_none_for_missing_file(tmp_path: Path):
assert read_gguf_context_length(str(tmp_path / "nope.gguf")) is None
def test_context_length_none_for_non_gguf(tmp_path: Path):
p = tmp_path / "garbage.gguf"
p.write_bytes(b"not a gguf file at all, just bytes")
assert read_gguf_context_length(str(p)) is None
def test_context_length_read_from_arch_namespaced_key(tmp_path: Path):
p = _write_synthetic_gguf(
tmp_path / "model.gguf",
{"general.architecture": "llama"},
extra_uint32 = {"llama.context_length": 4096, "llama.block_count": 32},
)
assert read_gguf_context_length(str(p)) == 4096
def test_context_length_none_when_absent(tmp_path: Path):
# Architecture present but no <arch>.context_length key.
p = _write_synthetic_gguf(
tmp_path / "model.gguf",
{"general.architecture": "llama"},
extra_uint32 = {"llama.block_count": 32},
)
assert read_gguf_context_length(str(p)) is None
def test_context_length_ignores_foreign_arch_key(tmp_path: Path):
# A context_length under a different arch namespace must not match.
p = _write_synthetic_gguf(
tmp_path / "model.gguf",
{"general.architecture": "llama"},
extra_uint32 = {"qwen2.context_length": 8192},
)
assert read_gguf_context_length(str(p)) is None
def test_context_length_read_from_uint64(tmp_path: Path):
# Some models store context_length as a uint64 (vtype 10).
p = _write_synthetic_gguf(
tmp_path / "model.gguf",
{"general.architecture": "qwen3"},
extra_uint64 = {"qwen3.context_length": 262144},
)
assert read_gguf_context_length(str(p)) == 262144
def test_context_length_zero_treated_as_absent(tmp_path: Path):
# A zero/garbage ceiling must read as None so the UI can't build a slider
# with max < min.
p = _write_synthetic_gguf(
tmp_path / "model.gguf",
{"general.architecture": "llama"},
extra_uint32 = {"llama.context_length": 0},
)
assert read_gguf_context_length(str(p)) is None
def test_extracts_general_string_fields(tmp_path: Path):
p = _write_synthetic_gguf(
tmp_path / "model.gguf",

View file

@ -50,6 +50,10 @@ _CACHE_MAX_ENTRIES = 4096
# keyed by (file cache key, wanted key). None = key absent / file unreadable.
_BOOL_CACHE: Dict[Tuple[_CacheKey, str], Optional[bool]] = {}
# Native training context length (``{arch}.context_length``). None = absent /
# unreadable. Lets the UI show the real context ceiling before a model loads.
_CONTEXT_CACHE: Dict[_CacheKey, Optional[int]] = {}
def _cache_key(path: str) -> Optional[_CacheKey]:
try:
@ -138,6 +142,92 @@ def _parse_gguf_header(path: str) -> Optional[Dict[str, str]]:
return out
def read_gguf_context_length(path: str) -> Optional[int]:
"""Return the GGUF's native training context length (``{arch}.context_length``),
or ``None`` if missing/unreadable/not a GGUF. Cached by (path, mtime, size).
Lets the UI populate the context slider before the model is loaded."""
key = _cache_key(path)
if key is None:
return None
with _CACHE_LOCK:
if key in _CONTEXT_CACHE:
return _CONTEXT_CACHE[key]
result = _parse_gguf_context_length(path)
with _CACHE_LOCK:
while len(_CONTEXT_CACHE) >= _CACHE_MAX_ENTRIES:
try:
_CONTEXT_CACHE.pop(next(iter(_CONTEXT_CACHE)))
except StopIteration:
break
_CONTEXT_CACHE[key] = result
return result
def _parse_gguf_context_length(path: str) -> Optional[int]:
# The context key is architecture-namespaced (``llama.context_length`` etc.),
# so we learn the key only after reading ``general.architecture``. GGUF writes
# general.* before arch.* keys, matching the loader's own parser.
ctx_key: Optional[str] = None
try:
with open(path, "rb") as f:
head = f.read(24)
if len(head) < 24:
return None
magic, _version, _tcount, kv_count = struct.unpack("<IIQQ", head)
if magic != _GGUF_MAGIC:
return None
for _ in range(kv_count):
try:
klen_bytes = f.read(8)
if len(klen_bytes) < 8:
break
klen = struct.unpack("<Q", klen_bytes)[0]
if klen > 1 << 20: # 1 MB sanity bound
break
kbytes = f.read(klen)
if len(kbytes) < klen:
break
key = kbytes.decode("utf-8", "replace")
vt_bytes = f.read(4)
if len(vt_bytes) < 4:
break
vtype = struct.unpack("<I", vt_bytes)[0]
if vtype == 8 and key == "general.architecture":
slen_bytes = f.read(8)
if len(slen_bytes) < 8:
break
slen = struct.unpack("<Q", slen_bytes)[0]
if slen > 1 << 22: # 4 MB sanity bound
break
sbytes = f.read(slen)
if len(sbytes) < slen:
break
ctx_key = f"{sbytes.decode('utf-8', 'replace')}.context_length"
elif ctx_key is not None and key == ctx_key and vtype in (4, 10):
width = 4 if vtype == 4 else 8
n_bytes = f.read(width)
if len(n_bytes) < width:
break
value = struct.unpack("<I" if vtype == 4 else "<Q", n_bytes)[0]
# A real context length is positive; treat 0/garbage as
# absent so the UI never builds a slider with max < min.
return value if value > 0 else None
else:
if not _skip_gguf_value(f, vtype):
break
except (struct.error, UnicodeDecodeError):
break
except OSError as e:
logger.debug(f"read_gguf_context_length: cannot open {path}: {e}")
return None
except Exception as e:
logger.debug(f"read_gguf_context_length: parse failure on {path}: {e}")
return None
return None
# Strings (8) and arrays (9) are handled inline.
_FIXED_VTYPE_SIZES: Dict[int, int] = {
0: 1, # uint8

View file

@ -119,6 +119,7 @@ function RootLayout() {
chatRuntime.setActiveThreadId(null);
chatRuntime.setActiveProjectId(null);
chatRuntime.setIncognito(false);
if (chatRuntime.pendingSelection) chatRuntime.abandonStagedModel();
void navigate({
to: "/chat",
search: { new: crypto.randomUUID() },
@ -135,6 +136,7 @@ function RootLayout() {
chatRuntime.setActiveProjectId(null);
chatRuntime.setActiveThreadId(null);
chatRuntime.setIncognito(false);
if (chatRuntime.pendingSelection) chatRuntime.abandonStagedModel();
}, [isChatRoute]);
return (

View file

@ -8,6 +8,8 @@ import {
PopoverContent,
PopoverTrigger,
} from "@/components/ui/popover";
import { InfoHint } from "@/components/ui/info-hint";
import { Switch } from "@/components/ui/switch";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { usePlatformStore } from "@/config/env";
import { isCustomProviderType } from "@/features/chat/external-providers";
@ -108,6 +110,11 @@ interface ModelSelectorProps {
activeGgufVariant?: string | null;
onValueChange?: (value: string, meta: ModelSelectorChangeMeta) => void;
onEject?: () => void;
/** When provided, renders a persisted "Load on selection" toggle in the
* popover. Off picking a model stages it for a deferred, configured load
* instead of loading immediately. */
loadOnSelection?: boolean;
onLoadOnSelectionChange?: (value: boolean) => void;
onFoldersChange?: () => void;
onPickLocalModel?: () => void | Promise<void>;
onModelsChange?: (deletedModel?: DeletedModelRef) => void;
@ -210,6 +217,8 @@ function ModelSelectorContent({
onPickLocalModel,
onModelsChange,
deleteDisabled,
loadOnSelection,
onLoadOnSelectionChange,
className,
dataTour,
}: {
@ -223,6 +232,8 @@ function ModelSelectorContent({
onPickLocalModel?: () => void;
onModelsChange?: (deletedModel?: DeletedModelRef) => void;
deleteDisabled?: boolean;
loadOnSelection?: boolean;
onLoadOnSelectionChange?: (value: boolean) => void;
className?: string;
dataTour?: string;
}) {
@ -378,6 +389,37 @@ function ModelSelectorContent({
</button>
</div>
) : null}
{onLoadOnSelectionChange ? (
<div className="mt-1.5 border-t border-border/70 pt-1.5">
<div className="flex w-full items-center justify-between gap-2 rounded-md px-2 py-1.5 text-xs text-muted-foreground">
<div className="flex min-w-0 flex-col gap-0.5">
<div className="flex min-w-0 items-center gap-1.5">
<span>Load on selection</span>
<InfoHint>
<div className="space-y-1">
<div>
<span className="font-medium">On:</span> load the model
immediately after selection.
</div>
<div>
<span className="font-medium">Off:</span> configure options
first, then click Load model.
</div>
</div>
</InfoHint>
</div>
<span className="text-[10px] leading-none text-muted-foreground/70">
Local GGUF models only
</span>
</div>
<Switch
className="panel-switch shrink-0"
checked={loadOnSelection ?? true}
onCheckedChange={onLoadOnSelectionChange}
/>
</div>
</div>
) : null}
</PopoverContent>
);
}
@ -395,6 +437,8 @@ export function ModelSelector({
onPickLocalModel,
onModelsChange,
deleteDisabled,
loadOnSelection,
onLoadOnSelectionChange,
variant = "outline",
size = "default",
className,
@ -513,6 +557,8 @@ export function ModelSelector({
onPickLocalModel={onPickLocalModel ? handlePickLocalModel : undefined}
onModelsChange={onModelsChange}
deleteDisabled={deleteDisabled}
loadOnSelection={loadOnSelection}
onLoadOnSelectionChange={onLoadOnSelectionChange}
className={contentClassName}
dataTour={contentDataTour}
/>

View file

@ -1357,15 +1357,14 @@ export function HubModelPicker({
<>
<ListLabel>LM Studio</ListLabel>
{lmStudioModels.map((m) => {
const isGgufFile = m.path.toLowerCase().endsWith(".gguf");
const isGguf = isGgufRepo(m.id) || isGgufRepo(m.display_name);
const optionKey = makeModelOptionKey("lm-studio", m.id);
return (
<div key={m.id}>
<ModelRow
label={m.model_id ?? m.display_name}
meta={
isGguf || m.path.toLowerCase().endsWith(".gguf") ? "GGUF" : "Local"
}
meta={isGguf || isGgufFile ? "GGUF" : "Local"}
selected={value === m.id}
optionProps={hubModelList.getOptionProps(
optionKey,
@ -1381,6 +1380,7 @@ export function HubModelPicker({
source: "local",
isLora: false,
isDownloaded: true,
isGguf: isGgufFile,
});
}
}}
@ -1594,6 +1594,7 @@ export function HubModelPicker({
source: "local",
isLora: false,
isDownloaded: true,
isGguf: true,
});
} else if (isGguf) {
setExpandedGguf((prev) =>

View file

@ -31,6 +31,9 @@ export interface ModelSelectorChangeMeta {
ggufVariant?: string;
isDownloaded?: boolean;
expectedBytes?: number;
/** Direct local .gguf file picked without a variant (custom folder / LM
* Studio). Marks it as a GGUF source for the deferred-load staging flow. */
isGguf?: boolean;
}
export interface DeletedModelRef {

View file

@ -0,0 +1,43 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import type { ReactNode } from "react";
import {
Tooltip,
TooltipContent,
TooltipTrigger,
} from "@/components/ui/tooltip";
import { InformationCircleIcon } from "@hugeicons/core-free-icons";
import { HugeiconsIcon } from "@hugeicons/react";
/** Small "i" affordance that reveals a styled tooltip on hover/focus. The
* standard inline help control across the settings UI. */
export function InfoHint({ children }: { children: ReactNode }) {
return (
<Tooltip>
<TooltipTrigger asChild>
<button
type="button"
aria-label="More info"
className="inline-flex size-4 shrink-0 cursor-help items-center justify-center rounded-full text-muted-foreground/70 transition-colors hover:text-[#383835] dark:hover:text-[#e8e8e8] focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
>
<HugeiconsIcon
icon={InformationCircleIcon}
strokeWidth={1.75}
className="size-3.5"
/>
</button>
</TooltipTrigger>
<TooltipContent
side="top"
align="center"
sideOffset={6}
collisionPadding={12}
className="[&_span>svg]:hidden! duration-0 max-w-[240px] text-left"
>
{children}
</TooltipContent>
</Tooltip>
);
}

View file

@ -2,6 +2,7 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { authFetch } from "@/features/auth";
import { consumeNativePathToken } from "@/features/native-intents/api";
import { formatFastApiDetail } from "@/lib/format-fastapi-error";
import type {
MessageRecord,
@ -115,6 +116,45 @@ export async function validateModel(
return parseJsonOrThrow<ValidateModelResponse>(response);
}
/**
* Read a GGUF's native context length from its local header (no GPU load, no
* download). Returns null when the file isn't downloaded yet, the model isn't a
* GGUF, or it's gated. For a native (drag-drop / picked) file, pass
* `nativePathToken` so the backend reads the granted local path. Used by the
* deferred-load staging flow to fill the context slider before the single load.
*/
export async function fetchGgufContextLength(payload: {
model_path: string;
gguf_variant?: string | null;
hf_token?: string | null;
nativePathToken?: string | null;
}): Promise<number | null> {
let nativePathLease: string | null = null;
if (payload.nativePathToken) {
try {
nativePathLease = (
await consumeNativePathToken(payload.nativePathToken, "validate-model")
).nativePathLease;
} catch {
// Lease expired / revoked: degrade to no context (the load can re-mint).
return null;
}
}
const response = await authFetch("/api/inference/validate", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
model_path: payload.model_path,
gguf_variant: payload.gguf_variant ?? null,
hf_token: payload.hf_token ?? null,
native_path_lease: nativePathLease,
include_context_length: true,
}),
});
const res = await parseJsonOrThrow<ValidateModelResponse>(response);
return res.context_length ?? null;
}
export async function unloadModel(payload: UnloadModelRequest): Promise<void> {
const response = await authFetch("/api/inference/unload", {
method: "POST",

View file

@ -62,7 +62,10 @@ import {
parseExternalModelId,
} from "./external-providers";
import { useChatModelRuntime } from "./hooks/use-chat-model-runtime";
import type { SelectedModelInput } from "./hooks/use-chat-model-runtime";
import { useChatProjects } from "./hooks/use-chat-projects";
import { useStagedModelPreparation } from "./hooks/use-staged-model-preparation";
import { useLatestRef } from "@/features/hub/hooks/use-latest-ref";
import {
type SidebarItem,
useChatSidebarItems,
@ -93,6 +96,7 @@ import {
CHAT_IMAGE_TOOLS_ENABLED_KEY,
CHAT_TOOLS_ENABLED_KEY,
CHAT_WEB_FETCH_TOOLS_ENABLED_KEY,
hasGgufSource,
loadOptionalBool,
useChatRuntimeStore,
} from "./stores/chat-runtime-store";
@ -1024,6 +1028,20 @@ export function ChatPage(): ReactElement {
const settingsOpen = useChatRuntimeStore((s) => s.settingsPanelOpen);
const setSettingsOpen = useChatRuntimeStore((s) => s.setSettingsPanelOpen);
const loadOnSelection = useChatRuntimeStore((s) => s.loadOnSelection);
const setLoadOnSelection = useChatRuntimeStore((s) => s.setLoadOnSelection);
// Deferred-load staging: downloads a staged GGUF (if needed) and reads its
// header context so the sheet can show the context slider before the load.
const stagedDownload = useStagedModelPreparation();
// Abandon a staged pick: the store action cancels its in-flight download and
// reverts the edited knobs, so nothing lingers after the user walks away.
const abandonStaged = useCallback(() => {
useChatRuntimeStore.getState().abandonStagedModel();
}, []);
// Tracks whether the chat page is still mounted, so a staged-load failure that
// resolves after the user left chat doesn't resurrect the abandoned pick.
const mountedRef = useRef(true);
useEffect(() => () => void (mountedRef.current = false), []);
const incognito = useChatRuntimeStore((s) => s.incognito);
const setIncognito = useChatRuntimeStore((s) => s.setIncognito);
const incognitoLabel = incognito
@ -1503,12 +1521,64 @@ export function ChatPage(): ReactElement {
closeArtifactSurface();
}, [activeThreadId, closeArtifactSurface, selectedArtifact, view]);
// Abandon a staged (not-yet-loaded) pick when the chat context actually
// changes — switching threads, leaving single view, or starting a new chat /
// project — so a stale Load button can't resurface in a different context.
// New Chat keeps activeThreadId null and only bumps the `new` search nonce, so
// the key includes the route identity, not just the thread. Mirrors the
// incognito reset pattern. (Route exit is handled in __root.tsx, which runs
// after this unmounts.) Clear only on a real change, never on mount: staging
// from the Hub sets pendingSelection then navigates here, and clearing on
// mount would wipe it. Comparing the previous context (rather than a first-run
// flag) is also safe under StrictMode's double-invoke and component remounts.
const chatContextKey = `${view.mode}|${activeThreadId ?? ""}|${search.new ?? ""}|${search.project ?? ""}`;
const chatContextKeyRef = useLatestRef(chatContextKey);
const prevChatContextRef = useRef<string | null>(null);
useEffect(() => {
const prev = prevChatContextRef.current;
prevChatContextRef.current = chatContextKey;
if (prev === null || prev === chatContextKey) return;
abandonStaged();
}, [chatContextKey, abandonStaged]);
const hasActiveModel = Boolean(inferenceParams.checkpoint);
// Load immediately, or — when "Load on selection" is off — stage the pick so
// its load options can be set first. Shared by the main selector, native
// drag-drop/picker, and the dropped-file chip (the Hub stages via the store).
const stageOrLoad = useCallback(
async (selection: SelectedModelInput) => {
const store = useChatRuntimeStore.getState();
// Only GGUF picks have pre-load options worth staging. Non-GGUF models
// (and the toggle-on case) load immediately, so e.g. a trust_remote_code
// approval surfaces through the normal load path.
if (store.loadOnSelection || !hasGgufSource(selection)) {
// Abandon any staged GGUF first so its edited knobs (e.g. a custom
// context length) don't leak into this immediate load -- resolveLoad
// reads customContextLength before checking the target is GGUF.
abandonStaged();
await selectModel(selection);
return;
}
// Tear down any existing staged pick first so its in-flight download is
// cancelled, not left running after we rebind to the new pick.
abandonStaged();
store.stageModel({
id: selection.id,
isLora: selection.isLora,
ggufVariant: selection.ggufVariant,
isDownloaded: selection.isDownloaded,
expectedBytes: selection.expectedBytes,
nativePathToken: selection.nativePathToken,
isGguf: selection.isGguf,
});
},
[abandonStaged, selectModel],
);
const loadNativeModelIntent = useCallback(
async (intent: NativeIntent, loadingDescription: string) => {
const label =
intent.path.displayLabel || intent.displayLabel || "Local GGUF model";
await selectModel({
await stageOrLoad({
id: label,
nativePathToken: intent.path.token,
isDownloaded: true,
@ -1518,7 +1588,7 @@ export function ChatPage(): ReactElement {
});
useNativeIntentStore.getState().clearModelIntent(intent.id);
},
[selectModel],
[stageOrLoad],
);
const handleNativeModelDropAutoLoad = useCallback(
(intent: NativeIntent) =>
@ -1567,6 +1637,7 @@ export function ChatPage(): ReactElement {
ggufVariant?: string;
isDownloaded?: boolean;
expectedBytes?: number;
isGguf?: boolean;
},
) => {
const store = useChatRuntimeStore.getState();
@ -1579,6 +1650,9 @@ export function ChatPage(): ReactElement {
)
return;
if (meta?.source === "external" || isExternalModelId(value)) {
// Switching to an external model abandons any staged local pick: cancel
// its download too (setCheckpoint below only clears the pending + knobs).
abandonStaged();
const selectedExternal = parseExternalModelId(value);
const selectedProvider = selectedExternal
? externalProvidersForChat.find(
@ -1746,20 +1820,27 @@ export function ChatPage(): ReactElement {
duration: 6000,
});
}
await selectModel({
const selection = {
id: value,
isLora: meta?.isLora,
ggufVariant: meta?.ggufVariant,
isDownloaded: meta?.isDownloaded,
expectedBytes: meta?.expectedBytes,
});
isGguf: meta?.isGguf,
};
// "Load on selection" off: stage the model and open settings so its
// load knobs (tensor parallel, context length…) can be set, then it
// loads once via the sheet's Load button. The currently loaded model
// stays put until the user commits.
await stageOrLoad(selection);
})();
},
[
abandonStaged,
activeThreadId,
externalProvidersForChat,
modelsFromStore,
selectModel,
stageOrLoad,
view,
],
);
@ -2139,6 +2220,8 @@ export function ChatPage(): ReactElement {
activeGgufVariant={activeGgufVariant}
onValueChange={handleCheckpointChange}
onEject={handleEject}
loadOnSelection={loadOnSelection}
onLoadOnSelectionChange={setLoadOnSelection}
onFoldersChange={refreshLocalModels}
onPickLocalModel={isTauri ? chooseNativeModel : undefined}
onModelsChange={refreshModelLists}
@ -2180,7 +2263,7 @@ export function ChatPage(): ReactElement {
<NativeModelChip
intent={pendingNativeModelIntent}
nativeReadsDisabled={!nativePathLeasesSupported}
onLoad={(selection) => selectModel(selection)}
onLoad={(selection) => stageOrLoad(selection)}
/>
) : null}
{loadingModel && loadToastDismissed ? (
@ -2330,7 +2413,13 @@ export function ChatPage(): ReactElement {
<ChatSettingsPanel
open={settingsOpen}
onOpenChange={setSettingsOpen}
onOpenChange={(open) => {
setSettingsOpen(open);
// Closing the sheet abandons a staged (not-yet-loaded) pick: cancel its
// download and revert the staged knobs so nothing lingers as a dirty
// edit (or a background download) on the loaded model.
if (!open) abandonStaged();
}}
params={inferenceParams}
onParamsChange={setInferenceParams}
isExternalModel={isExternalModel}
@ -2356,6 +2445,47 @@ export function ChatPage(): ReactElement {
});
}
}}
onLoadPendingModel={() => {
const pending = useChatRuntimeStore.getState().pendingSelection;
if (!pending) return;
const keyAtLoad = chatContextKey;
// forceReload: the staged model isn't loaded yet, so bypass the
// same-checkpoint dedupe (and selectModel clears pendingSelection).
// keepSpeculative: honor the speculative mode set on the sidebar.
void selectModel({
...pending,
forceReload: true,
keepSpeculative: true,
throwOnError: true,
}).catch(() => {
// Recoverable failure (expired token, gated repo, OOM…): selectModel
// cleared the pick but left the edited knobs intact.
const store = useChatRuntimeStore.getState();
// A pick staged meanwhile owns the knobs now; leave it untouched.
if (store.pendingSelection) return;
// Restore (not re-stage, which would reset the knobs) only if the
// staged-load is still wanted: same chat context, sheet still open,
// page still mounted.
const stillWanted =
mountedRef.current &&
store.settingsPanelOpen &&
chatContextKeyRef.current === keyAtLoad;
if (stillWanted) {
store.setPendingSelection(pending);
} else {
// Abandoned (closed the sheet / switched chats / left chat): drop
// the orphaned staged knob edits so they don't linger as dirty
// settings over the loaded model.
store.resetModelSettingsToLoaded();
}
});
}}
stagedDownloadFraction={stagedDownload.progress?.fraction ?? null}
onCancelStagedDownload={() =>
stagedDownload.cancelDownload(
useChatRuntimeStore.getState().pendingSelection?.ggufVariant ?? null,
)
}
/>
</div>
);

View file

@ -54,18 +54,14 @@ import {
import { Slider } from "@/components/ui/slider";
import { Switch } from "@/components/ui/switch";
import { Textarea } from "@/components/ui/textarea";
import {
Tooltip,
TooltipContent,
TooltipTrigger,
} from "@/components/ui/tooltip";
import { InfoHint } from "@/components/ui/info-hint";
import { Tooltip, TooltipContent } from "@/components/ui/tooltip";
import { useIsMobile } from "@/hooks/use-mobile";
import { useLlamaUpdateCheck } from "@/hooks/use-llama-update-check";
import { cn } from "@/lib/utils";
import {
ArrowTurnBackwardIcon,
Edit03Icon,
InformationCircleIcon,
LayoutAlignRightIcon,
} from "@hugeicons/core-free-icons";
import { ChevronDownStandardIcon } from "@/lib/chevron-icons";
@ -101,7 +97,10 @@ import {
providerSupportsBuiltinCodeExecution,
providerSupportsFastMode,
} from "./provider-capabilities";
import { useChatRuntimeStore } from "./stores/chat-runtime-store";
import {
isPendingGguf,
useChatRuntimeStore,
} from "./stores/chat-runtime-store";
import { RetrievalSettingsSection } from "@/features/rag/components/retrieval-settings-section";
import type { InferenceParams } from "./types/runtime";
@ -112,33 +111,6 @@ function canUseStorage(): boolean {
return typeof window !== "undefined";
}
export function InfoHint({ children }: { children: ReactNode }) {
return (
<Tooltip>
<TooltipTrigger asChild>
<button
type="button"
aria-label="More info"
className="inline-flex size-4 shrink-0 cursor-help items-center justify-center rounded-full text-muted-foreground/70 transition-colors hover:text-[#383835] dark:hover:text-[#e8e8e8] focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
>
<HugeiconsIcon
icon={InformationCircleIcon}
strokeWidth={1.75}
className="size-3.5"
/>
</button>
</TooltipTrigger>
<TooltipContent
side="left"
sideOffset={8}
className="tooltip-compact [&_span>svg]:hidden! duration-0 max-w-64"
>
{children}
</TooltipContent>
</Tooltip>
);
}
/**
* Editable numeric value display, shared by every slider value and the Context
* Length input. An <input> that looks like text (shows `displayValue ?? value`,
@ -466,6 +438,12 @@ interface ChatSettingsPanelProps {
*/
externalProviderType?: string | null;
onReloadModel?: () => void;
/** Loads the staged `pendingSelection` (deferred "Load on selection" flow). */
onLoadPendingModel?: () => void;
/** Download progress (01) for a staged GGUF being fetched, or null when idle. */
stagedDownloadFraction?: number | null;
/** Cancels the in-flight staged download (paired with abandoning the stage). */
onCancelStagedDownload?: () => void;
}
export function ChatSettingsPanel({
@ -479,6 +457,9 @@ export function ChatSettingsPanel({
onExternalProviderChange,
externalProviderType = null,
onReloadModel,
onLoadPendingModel,
stagedDownloadFraction,
onCancelStagedDownload,
}: ChatSettingsPanelProps) {
// Local models show every knob; providerCapabilities is only consulted when
// isExternalModel. Unknown providers fall back to the OpenAI-compat shape via
@ -493,9 +474,31 @@ export function ChatSettingsPanel({
const showPresencePenalty =
!isExternalModel || Boolean(providerCapabilities?.presencePenalty);
const isMobile = useIsMobile();
const isGguf = useChatRuntimeStore((s) => s.activeGgufVariant) != null;
const pendingSelection = useChatRuntimeStore((s) => s.pendingSelection);
const abandonStagedModel = useChatRuntimeStore((s) => s.abandonStagedModel);
const resetModelSettingsToLoaded = useChatRuntimeStore(
(s) => s.resetModelSettingsToLoaded,
);
// A staged GGUF pick (deferred load) shows the GGUF load knobs so they can be
// set before the single load.
const pendingIsGguf = isPendingGguf(pendingSelection);
// Short, human-readable name for the staged pick (HF ids carry an org prefix;
// native picks are already a display label). Drives the "staged, not loaded"
// callout so it's obvious the selection hasn't loaded yet.
const stagedLabel = (() => {
const id = pendingSelection?.id ?? "";
const slash = id.lastIndexOf("/");
const base = slash >= 0 ? id.slice(slash + 1) : id;
return base || id;
})();
const isLoadedGguf =
useChatRuntimeStore((s) => s.activeGgufVariant) != null;
const isGguf = isLoadedGguf || pendingIsGguf;
// A staged pick is always a local GGUF, so show its Model section (and the
// Load button) even when the currently active model is external.
const hasModelContent =
!isExternalModel && (isGguf || Boolean(params.checkpoint));
pendingSelection != null ||
(!isExternalModel && (isGguf || Boolean(params.checkpoint)));
const speculativeType = useChatRuntimeStore((s) => s.speculativeType);
const setSpeculativeType = useChatRuntimeStore((s) => s.setSpeculativeType);
const loadedSpeculativeType = useChatRuntimeStore(
@ -560,8 +563,25 @@ export function ChatSettingsPanel({
const setActivePreset = useChatRuntimeStore((s) => s.setActivePreset);
const settingsHydrated = useChatRuntimeStore((s) => s.settingsHydrated);
const ctxDisplayValue = customContextLength ?? ggufContextLength ?? "";
const ctxMaxValue = ggufNativeContextLength ?? ggufContextLength ?? null;
// A staged (not-yet-loaded) GGUF carries its own header context length on
// pendingSelection, so the slider can use the staged model's real ceiling
// without reading the loaded model's `ggufContextLength`.
const stagedContextLength = pendingSelection?.contextLength ?? null;
// While staging, the sheet reflects the STAGED model, so its header context
// takes precedence over the loaded model's (which may differ or be larger).
const baseContext = pendingIsGguf ? stagedContextLength : ggufContextLength;
const baseNativeContext = pendingIsGguf
? stagedContextLength
: ggufNativeContextLength;
// Context controls render once we actually have a ceiling: for a staged GGUF,
// once its header metadata arrives (post-download); otherwise post-load.
const showContextControl = pendingIsGguf
? stagedContextLength != null
: isLoadedGguf;
const stagedDownloading =
stagedDownloadFraction != null && stagedDownloadFraction < 1;
const ctxDisplayValue = customContextLength ?? baseContext ?? "";
const ctxMaxValue = baseNativeContext ?? baseContext ?? null;
const kvDirty = kvCacheDtype !== loadedKvCacheDtype;
const ctxDirty = customContextLength !== null;
const specDirty = speculativeType !== loadedSpeculativeType;
@ -569,12 +589,6 @@ export function ChatSettingsPanel({
const tpDirty = tensorParallel !== (loadedTensorParallel ?? false);
const modelSettingsDirty =
kvDirty || ctxDirty || specDirty || specDraftDirty || tpDirty;
const loadedChatTemplateOverride = useChatRuntimeStore(
(s) => s.loadedChatTemplateOverride,
);
const setChatTemplateOverride = useChatRuntimeStore(
(s) => s.setChatTemplateOverride,
);
const [presetNameInput, setPresetNameInput] = useState(activePreset);
const [systemPromptEditorOpen, setSystemPromptEditorOpen] = useState(false);
const [systemPromptDraft, setSystemPromptDraft] = useState("");
@ -846,8 +860,19 @@ export function ChatSettingsPanel({
{hasModelContent && (
<CollapsibleSection label="Model" defaultOpen={true} first>
<div className="flex flex-col gap-4 pt-1">
{pendingSelection && (
<Alert className="rounded-[14px] border-primary/30 bg-primary/5 px-3 py-2">
<AlertTitle className="text-[12px] font-medium">
{stagedLabel} is staged, not loaded yet
</AlertTitle>
<AlertDescription className="text-[11.5px] leading-[1.45] text-muted-foreground">
Set the options below, then choose Load model to load it.
</AlertDescription>
</Alert>
)}
{isGguf && (
<>
{showContextControl && (
<div className="space-y-3.5">
<div className="flex items-center justify-between gap-3">
<span className="min-w-0 text-[13px] font-medium leading-[1.25] tracking-nav text-nav-fg">
@ -857,14 +882,14 @@ export function ChatSettingsPanel({
value={
typeof ctxDisplayValue === "number"
? ctxDisplayValue
: (ggufContextLength ?? 0)
: (baseContext ?? 0)
}
min={128}
max={ctxMaxValue ?? undefined}
step={1}
onChange={(v) => {
setCustomContextLength(
v === (ggufContextLength ?? 0) ? null : v,
v === (baseContext ?? 0) ? null : v,
);
}}
ariaLabel="Context Length"
@ -879,14 +904,14 @@ export function ChatSettingsPanel({
Math.min(
typeof ctxDisplayValue === "number"
? ctxDisplayValue
: (ggufContextLength ?? 4096),
: (baseContext ?? 4096),
ctxMaxValue ?? 4096,
),
]}
onValueChange={([v]) => {
const snapped = Math.round(v);
setCustomContextLength(
snapped === (ggufContextLength ?? 0) ? null : snapped,
snapped === (baseContext ?? 0) ? null : snapped,
);
}}
className="panel-slider"
@ -901,6 +926,7 @@ export function ChatSettingsPanel({
</p>
)}
</div>
)}
<div className="flex items-center justify-between gap-3">
<div className="flex min-w-0 items-center gap-1.5">
<span className="min-w-0 text-[13px] font-medium leading-[1.25] tracking-nav text-nav-fg">
@ -937,6 +963,8 @@ export function ChatSettingsPanel({
</Select>
</div>
</div>
{isGguf && (
<>
<div className="flex items-center justify-between gap-3">
<div className="flex min-w-0 items-center gap-1.5">
<span className="min-w-0 text-[13px] font-medium leading-[1.25] tracking-nav text-nav-fg">
@ -1043,6 +1071,8 @@ export function ChatSettingsPanel({
className="h-7 w-[76px] rounded-full border-transparent bg-black/[0.04] dark:bg-white/[0.05] hover:bg-black/[0.06] dark:hover:bg-white/[0.1] pl-3 pr-2 py-0 text-[13px] font-medium text-nav-fg outline-none focus-visible:ring-0"
/>
</div>
)}
</>
)}
<div className="flex items-center justify-between gap-3">
<div className="flex min-w-0 items-center gap-1.5">
@ -1099,8 +1129,44 @@ export function ChatSettingsPanel({
{/* Apply/Reset belongs to the model-reload settings above (context
length, KV cache, speculative decoding). Render it here, before
the Chat Template row, so it never reads as attached to Chat
Template (which is edited via its own dialog). */}
{modelSettingsDirty && (
Template (which is edited via its own dialog). When a model is
staged (deferred load), Load/Cancel takes its place: there's
nothing loaded to "apply" against yet. */}
{pendingSelection ? (
<div className="flex flex-col gap-2">
{stagedDownloading && (
<p className="text-[11px] text-muted-foreground">
Downloading{" "}
{Math.round((stagedDownloadFraction ?? 0) * 100)}%
</p>
)}
<div className="flex flex-wrap gap-1.5">
<Button
type="button"
onClick={() => onLoadPendingModel?.()}
disabled={stagedDownloading}
size="sm"
className="h-7 px-3 text-[12px] font-medium tracking-nav bg-primary/92 text-primary-foreground hover:bg-primary"
>
Load model
</Button>
<Button
type="button"
variant="outline"
size="sm"
onClick={() => {
// Cancel abandons the stage; if a download is mid-flight,
// stop it too rather than leaving it running headless.
if (stagedDownloading) onCancelStagedDownload?.();
abandonStagedModel();
}}
className="h-7 px-3 text-[12px] font-medium tracking-nav text-muted-foreground"
>
Cancel
</Button>
</div>
</div>
) : modelSettingsDirty ? (
<div className="flex flex-wrap gap-1.5">
<Button
type="button"
@ -1114,20 +1180,13 @@ export function ChatSettingsPanel({
type="button"
variant="outline"
size="sm"
onClick={() => {
setCustomContextLength(null);
setKvCacheDtype(loadedKvCacheDtype);
setSpeculativeType(loadedSpeculativeType);
setSpecDraftNMax(loadedSpecDraftNMax);
setTensorParallel(loadedTensorParallel ?? false);
setChatTemplateOverride(loadedChatTemplateOverride);
}}
onClick={() => resetModelSettingsToLoaded()}
className="h-7 px-3 text-[12px] font-medium tracking-nav text-muted-foreground"
>
Reset
</Button>
</div>
)}
) : null}
<ChatTemplateFields />
</div>
</CollapsibleSection>
@ -1509,21 +1568,21 @@ export function ChatSettingsPanel({
: 64
}
max={
isExternalModel
// A staged GGUF caps to its own context even over an active
// external model (the staged model is what will load).
!pendingIsGguf && isExternalModel
? getExternalMaxOutputTokens(
externalProviderType,
externalSelection?.modelId,
)
: isGguf && ggufContextLength
? ggufContextLength
: isGguf && baseContext
? baseContext
: 32768
}
step={64}
onChange={set("maxTokens")}
displayValue={
isGguf &&
ggufContextLength &&
params.maxTokens >= ggufContextLength
isGguf && baseContext && params.maxTokens >= baseContext
? "Max"
: undefined
}

View file

@ -45,7 +45,7 @@ import {
import { CHAT_HISTORY_UPDATED_EVENT } from "../api/chat-api";
import type { ExternalProviderConfig } from "../external-providers";
import { ensureThreadRecord } from "../runtime-provider";
import { InfoHint } from "../chat-settings-sheet";
import { InfoHint } from "@/components/ui/info-hint";
import {
getStoredChatThread,
listStoredChatThreads,

View file

@ -43,12 +43,13 @@ import {
isMultimodalResponse,
} from "../types/api";
import { isExternalModelId } from "../external-providers";
import { cancelStagedModelDownload } from "@/features/hub";
import type {
ChatLoraSummary,
ChatModelSummary,
} from "../types/runtime";
type SelectedModelInput = {
export type SelectedModelInput = {
id: string;
isLora?: boolean;
ggufVariant?: string;
@ -57,7 +58,14 @@ type SelectedModelInput = {
expectedBytes?: number;
forceReload?: boolean;
nativePathToken?: string;
/** Direct local .gguf file (no HF variant / native token) still a GGUF
* source, so the staging flow treats it as one. */
isGguf?: boolean;
throwOnError?: boolean;
/** Keep the current speculative-decoding choice across the model switch
* instead of resetting it to the standing preference. Set by the deferred
* ("Load on selection") Load, where the user picked it for this model. */
keepSpeculative?: boolean;
};
const MODEL_LOAD_TOAST_CLASSNAMES = {
@ -370,8 +378,27 @@ export function useChatModelRuntime() {
typeof selection === "string" ? false : selection.forceReload ?? false;
const nativePathToken =
typeof selection === "string" ? undefined : selection.nativePathToken;
const explicitIsGguf =
typeof selection === "string" ? undefined : selection.isGguf;
const throwOnError =
typeof selection === "string" ? false : selection.throwOnError ?? false;
const keepSpeculative =
typeof selection === "string" ? false : selection.keepSpeculative ?? false;
// Picking/loading any model abandons a staged (deferred) selection.
// Before the early-returns below so even a no-op re-select clears the
// stage, and so the Load button unmounts on first click (no double-load).
const staged = useChatRuntimeStore.getState().pendingSelection;
if (staged) {
// Loading a DIFFERENT model abandons this stage, so cancel its in-flight
// download. Loading the staged pick itself keeps it (that download feeds
// this load).
const loadingStagedPick =
staged.id === modelId &&
(staged.ggufVariant ?? null) === (ggufVariant ?? null) &&
(staged.nativePathToken ?? null) === (nativePathToken ?? null);
if (!loadingStagedPick) cancelStagedModelDownload(staged);
useChatRuntimeStore.getState().setPendingSelection(null);
}
const currentVariant = useChatRuntimeStore.getState().activeGgufVariant;
if (!forceReload && (!modelId || (params.checkpoint === modelId && (ggufVariant ?? null) === (currentVariant ?? null)))) {
return;
@ -391,6 +418,7 @@ export function useChatModelRuntime() {
typeof selection === "string" ? false : selection.isDownloaded ?? false;
const model = models.find((entry) => entry.id === modelId);
const lora = loras.find((entry) => entry.id === modelId);
const isGguf = explicitIsGguf ?? model?.isGguf ?? false;
const loraIsAdapter = lora?.exportType === "lora";
const isLora =
explicitIsLora ?? model?.isLora ?? loraIsAdapter ?? false;
@ -501,7 +529,10 @@ export function useChatModelRuntime() {
// can't follow the user onto a model without an MTP head.
// spec_draft_n_max is MTP-only and always resets. The loaded
// shadow is seeded too, preventing a transient dirty Apply state.
if (currentCheckpoint && currentCheckpoint !== modelId) {
// keepSpeculative skips this for a staged Load: the user picked the
// mode for this model on the sidebar, so honor it (the backend still
// falls back at runtime if the model has no MTP head).
if (currentCheckpoint && currentCheckpoint !== modelId && !keepSpeculative) {
const persistedSpeculativeType = readPersistedSpeculativeType();
useChatRuntimeStore.setState({
speculativeType: persistedSpeculativeType,
@ -525,6 +556,7 @@ export function useChatModelRuntime() {
const effectiveMaxSeqLength = resolveLoadMaxSeqLength({
modelId,
ggufVariant,
isGguf,
customContextLength,
ggufContextLength,
currentCheckpoint,

View file

@ -0,0 +1,114 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { useCallback, useEffect } from "react";
import { useLatestRef } from "@/features/hub/hooks/use-latest-ref";
import { useRepoDownload } from "@/features/hub/download-manager/use-repo-download";
import type { DownloadJob } from "@/features/hub/download-manager/use-repo-download";
import { fetchGgufContextLength } from "../api/chat-api";
import {
isPendingGguf,
useChatRuntimeStore,
} from "../stores/chat-runtime-store";
/**
* Drives the deferred ("Load on selection" off) staging flow for a GGUF:
* download the file if needed (HF repo) or read it in place (native drag-drop /
* picked file), then read its header context length so the settings sheet can
* show the real context slider before the single GPU load. The staged context
* lands on `pendingSelection.contextLength` (scoped to the staged model, never
* the loaded model's `ggufContextLength`). Returns the live download job so the
* sheet can render progress / cancel. Mount once on the chat page.
*/
export function useStagedModelPreparation(): DownloadJob {
const pendingId = useChatRuntimeStore((s) => s.pendingSelection?.id ?? null);
const pendingVariant = useChatRuntimeStore(
(s) => s.pendingSelection?.ggufVariant ?? null,
);
const pendingNativeToken = useChatRuntimeStore(
(s) => s.pendingSelection?.nativePathToken ?? null,
);
// Only GGUF picks (HF variant or native file) have a header worth reading.
const pendingIsGguf = useChatRuntimeStore((s) =>
isPendingGguf(s.pendingSelection),
);
const pendingDownloaded = useChatRuntimeStore(
(s) => s.pendingSelection?.isDownloaded ?? false,
);
const pendingHasContext = useChatRuntimeStore(
(s) => s.pendingSelection?.contextLength != null,
);
const setPendingSelection = useChatRuntimeStore((s) => s.setPendingSelection);
const fetchContextMetadata = useCallback(async () => {
const current = useChatRuntimeStore.getState().pendingSelection;
if (!current?.id || !isPendingGguf(current)) return;
const { id, ggufVariant, nativePathToken } = current;
try {
const contextLength = await fetchGgufContextLength({
model_path: id,
gguf_variant: ggufVariant,
hf_token: useChatRuntimeStore.getState().hfToken || null,
nativePathToken,
});
// Apply only if the same model is still staged (the user may have switched
// picks or loaded/cancelled while the request was in flight). Native ids
// are display labels, not paths, so two files can share an id -- compare
// the path token too, or a stale response could land on the wrong pick.
const latest = useChatRuntimeStore.getState().pendingSelection;
if (
latest?.id === id &&
(latest.ggufVariant ?? null) === (ggufVariant ?? null) &&
(latest.nativePathToken ?? null) === (nativePathToken ?? null) &&
contextLength != null
) {
setPendingSelection({ ...latest, contextLength });
}
} catch {
// Leave contextLength null: the context slider stays hidden and the user
// can still load (context fills in from the load response afterwards).
}
}, [setPendingSelection]);
const job = useRepoDownload({
kind: "model",
// useRepoDownload must be called unconditionally; an idle repo id keeps it
// inert until something is staged.
repoId: pendingId ?? "__staged_idle__",
activeVariant: pendingVariant,
onComplete: () => {
void fetchContextMetadata();
},
});
// job.requestStartDownload's identity changes per render; hold it in a ref so
// the staging effect re-runs only when the staged model itself changes.
const startDownloadRef = useLatestRef(job.requestStartDownload);
const fetchMetadataRef = useLatestRef(fetchContextMetadata);
useEffect(() => {
if (!pendingId || !pendingIsGguf || pendingHasContext) return;
// Native files and already-downloaded HF files are local: read the header
// now. Otherwise download first; onComplete then reads it.
if (pendingNativeToken || pendingDownloaded) {
void fetchMetadataRef.current();
} else {
const expectedBytes =
useChatRuntimeStore.getState().pendingSelection?.expectedBytes ?? 0;
void startDownloadRef.current(pendingVariant, expectedBytes);
}
}, [
pendingId,
pendingVariant,
pendingNativeToken,
pendingIsGguf,
pendingDownloaded,
pendingHasContext,
startDownloadRef,
fetchMetadataRef,
]);
return job;
}

View file

@ -297,6 +297,7 @@ export function mergeBackendRecommendedInference({
export function resolveLoadMaxSeqLength({
modelId,
ggufVariant,
isGguf,
customContextLength,
ggufContextLength,
currentCheckpoint,
@ -306,6 +307,7 @@ export function resolveLoadMaxSeqLength({
}: {
modelId: string;
ggufVariant?: string | null;
isGguf?: boolean | null;
customContextLength: number | null;
ggufContextLength: number | null;
currentCheckpoint: string;
@ -314,7 +316,7 @@ export function resolveLoadMaxSeqLength({
presetSource: ChatPresetSource;
}): number {
const isDirectGgufFile = modelId.toLowerCase().endsWith(".gguf");
const isGgufLoad = ggufVariant != null || isDirectGgufFile;
const isGgufLoad = isGguf === true || ggufVariant != null || isDirectGgufFile;
const isReloadingCurrentGguf =
isGgufLoad &&
currentCheckpoint === modelId &&

View file

@ -3,6 +3,7 @@
import { toast } from "@/lib/toast";
import { create } from "zustand";
import { cancelStagedModelDownload } from "@/features/hub";
import {
type ChatPresetSource,
type Preset,
@ -35,6 +36,7 @@ export const CHAT_ALLOW_ARTIFACT_NETWORK_ACCESS_KEY =
"unsloth_chat_allow_artifact_network_access";
export const CHAT_MCP_ENABLED_KEY = "unsloth_chat_mcp_enabled";
export const CHAT_CONFIRM_TOOL_CALLS_KEY = "unsloth_chat_confirm_tool_calls";
export const CHAT_LOAD_ON_SELECTION_KEY = "unsloth_chat_load_on_selection";
export const CHAT_BYPASS_PERMISSIONS_KEY = "unsloth_chat_bypass_permissions";
export const CHAT_WEB_FETCH_TOOLS_ENABLED_KEY =
"unsloth_chat_web_fetch_tools_enabled";
@ -86,6 +88,7 @@ function saveRagSource(value: RagSource): void {
try {
window.localStorage.setItem(CHAT_RAG_SOURCE_KEY, JSON.stringify(value));
} catch {
// Ignore storage failures; the default RAG source still works for this session.
}
}
@ -408,6 +411,42 @@ function notifyHfTokenChanged(value: string): void {
}
}
/** A local model staged for a deferred load (see `pendingSelection`). Shape is
* a subset of the load hook's `SelectedModelInput`, structurally assignable. */
export type PendingModelSelection = {
id: string;
isLora?: boolean;
ggufVariant?: string;
isDownloaded?: boolean;
expectedBytes?: number;
/** Native (drag-drop / picked-from-disk) GGUF: the path token used to read
* the header and to load. Absent for HF-repo models. */
nativePathToken?: string;
/** Direct local .gguf file (custom folder / LM Studio): a GGUF source even
* though it carries neither an HF variant nor a native path token. */
isGguf?: boolean;
/** Native context length read from the GGUF header once the file is local.
* Scoped here (not the shared `ggufContextLength`) so a staged model's
* metadata never pollutes the currently-loaded model's context display. */
contextLength?: number | null;
};
/** A pick is a GGUF (HF variant, native file, or a direct local .gguf) and so
* has pre-load options worth staging. Works on a selection or a staged pick. */
export function hasGgufSource(x: {
ggufVariant?: string;
nativePathToken?: string;
isGguf?: boolean;
}): boolean {
return (
x.ggufVariant != null || x.nativePathToken != null || x.isGguf === true
);
}
export function isPendingGguf(pending: PendingModelSelection | null): boolean {
return pending != null && hasGgufSource(pending);
}
type ChatRuntimeStore = {
settingsHydrated: boolean;
params: InferenceParams;
@ -538,6 +577,13 @@ type ChatRuntimeStore = {
tensorParallel: boolean;
/** Backend-reported tensor-parallel state; null until first hydrated. */
loadedTensorParallel: boolean | null;
/** Persisted: when false, picking a local model stages it as
* `pendingSelection` (and opens settings) instead of loading immediately,
* so load settings can be set before the single load. */
loadOnSelection: boolean;
/** A local model picked while `loadOnSelection` is off: staged, not loaded.
* The settings sheet shows its load knobs and a Load button. */
pendingSelection: PendingModelSelection | null;
loadedIsMultimodal: boolean;
/** Active model is a block-diffusion model (DiffusionGemma): drives the
* denoising-canvas artifact auto-render. */
@ -638,7 +684,20 @@ type ChatRuntimeStore = {
setKvCacheDtype: (dtype: string | null) => void;
setSpeculativeType: (type: string | null) => void;
setSpecDraftNMax: (value: number | null) => void;
/** Revert the editable load knobs to the loaded model's baseline (or defaults
* when nothing is loaded). Used by the settings-sheet Reset button and to
* start each deferred-staging session clean so one staged pick's settings
* don't leak onto the next. */
resetModelSettingsToLoaded: () => void;
setTensorParallel: (value: boolean) => void;
setLoadOnSelection: (value: boolean) => void;
setPendingSelection: (selection: PendingModelSelection | null) => void;
/** Stage a pick for a deferred load: revert knobs to the loaded baseline,
* record the selection, and open the settings sheet. */
stageModel: (selection: PendingModelSelection) => void;
/** Abandon a staged pick without loading: revert the knobs to the loaded
* baseline and clear the pending selection. */
abandonStagedModel: () => void;
setCustomContextLength: (v: number | null) => void;
setChatTemplateOverride: (template: string | null) => void;
setPendingAudio: (base64: string, name: string) => void;
@ -839,6 +898,23 @@ function setScalarSettingVersion<K extends ScalarSettingKey>(
saveSettingsPatch({ [key]: value });
}
/** The "revert to the loaded model" baseline for the editable load knobs.
* Shared by resetModelSettingsToLoaded (full revert) and stageModel (which
* overrides speculative to start a fresh pick from the standing default). */
function loadedBaselineSettings(s: ChatRuntimeStore) {
const hasLoadedModel = Boolean(s.params.checkpoint);
return {
customContextLength: null,
kvCacheDtype: s.loadedKvCacheDtype,
tensorParallel: s.loadedTensorParallel ?? false,
speculativeType: hasLoadedModel
? s.loadedSpeculativeType
: readPersistedSpeculativeType(),
specDraftNMax: hasLoadedModel ? s.loadedSpecDraftNMax : null,
chatTemplateOverride: s.loadedChatTemplateOverride,
};
}
export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
settingsHydrated: false,
// Hydrate the last external checkpoint so the external picker survives a
@ -924,6 +1000,8 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
loadedSpecDraftNMax: null,
tensorParallel: false,
loadedTensorParallel: null,
loadOnSelection: loadBool(CHAT_LOAD_ON_SELECTION_KEY, true),
pendingSelection: null,
loadedIsMultimodal: false,
loadedIsDiffusion: false,
customContextLength: null,
@ -1061,6 +1139,11 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
// Clear stale per-turn usage on model change; the relaxed external-provider
// render gate would otherwise show old counters until the next completion.
const checkpointChanged = state.params.checkpoint !== modelId;
const pendingToClear =
checkpointChanged && state.params.checkpoint ? state.pendingSelection : null;
if (pendingToClear) {
cancelStagedModelDownload(pendingToClear);
}
// Clamp maxTokens to the new model's cap when switching into an external
// model so a value carried over from a local session doesn't exceed the
// slider's max.
@ -1088,6 +1171,14 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
},
activeGgufVariant: ggufVariant ?? null,
...(checkpointChanged ? { contextUsage: null } : {}),
// Switching away from a loaded model (e.g. picking an external provider)
// abandons any staged pick, so its Load button and edited knobs don't
// linger over the newly active model. Same revert as abandonStagedModel.
// Guarded on a non-empty current checkpoint: an establishing set from a
// background status sync (empty -> active) must not wipe a fresh stage.
...(pendingToClear
? { ...loadedBaselineSettings(state), pendingSelection: null }
: {}),
};
}),
setActiveThreadId: (activeThreadId) =>
@ -1100,6 +1191,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
// clear any stored external selection so the next refresh doesn't snap
// back to a model the user intentionally cleared.
saveLastExternalCheckpoint(null);
cancelStagedModelDownload(get().pendingSelection);
return set((state) => ({
params: {
...state.params,
@ -1107,6 +1199,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
},
activeGgufVariant: null,
activeNativePathToken: null,
pendingSelection: null,
ggufContextLength: null,
ggufMaxContextLength: null,
ggufNativeContextLength: null,
@ -1339,6 +1432,43 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
setSpeculativeType: (speculativeType) => set({ speculativeType }),
setSpecDraftNMax: (specDraftNMax) => set({ specDraftNMax }),
setTensorParallel: (tensorParallel) => set({ tensorParallel }),
resetModelSettingsToLoaded: () => set((s) => loadedBaselineSettings(s)),
setLoadOnSelection: (loadOnSelection) => {
saveBool(CHAT_LOAD_ON_SELECTION_KEY, loadOnSelection);
set({ loadOnSelection });
},
setPendingSelection: (pendingSelection) => set({ pendingSelection }),
stageModel: (selection) =>
set((s) => {
if (
s.pendingSelection &&
(s.pendingSelection.id !== selection.id ||
(s.pendingSelection.ggufVariant ?? null) !==
(selection.ggufVariant ?? null))
) {
cancelStagedModelDownload(s.pendingSelection);
}
return {
...loadedBaselineSettings(s),
pendingSelection: selection,
settingsPanelOpen: true,
// Speculative starts from the standing default, not the loaded model's
// mode, so a fresh pick doesn't inherit (and then carry, via the staged
// Load's keepSpeculative) a forced MTP mode onto a model that may lack it.
speculativeType: readPersistedSpeculativeType(),
specDraftNMax: null,
};
}),
abandonStagedModel: () => {
const { pendingSelection } = get();
if (!pendingSelection) return;
// Cancel the staged pick's in-flight download so it doesn't keep running
// after the staging UI is gone. Centralized here so every abandon path
// (sheet close, thread switch, route exit, new chat) cancels it, including
// root-level callers that have no access to the useRepoDownload hook.
cancelStagedModelDownload(pendingSelection);
set((s) => ({ ...loadedBaselineSettings(s), pendingSelection: null }));
},
setCustomContextLength: (customContextLength) => set({ customContextLength }),
setChatTemplateOverride: (chatTemplateOverride) =>
set({ chatTemplateOverride }),

View file

@ -72,6 +72,8 @@ export interface ValidateModelResponse {
is_lora?: boolean;
is_vision?: boolean;
requires_trust_remote_code?: boolean;
/** Native context length from the local GGUF header; null until downloaded. */
context_length?: number | null;
}
export interface GgufVariantDetail {

View file

@ -1,10 +1,14 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { DOWNLOAD_KIND } from "./constants";
import {
createDownloadManagerInitialState,
jobKeyOf,
removeJob,
selectActiveJob,
setState,
useDownloadManagerStore,
} from "./download-manager-state";
import { resetDownloadApiAdapterState } from "./download-api-adapter";
import {
@ -65,6 +69,26 @@ export const downloadManager: DownloadManagerController = {
dismiss: removeJob,
};
/** Cancel the in-flight download for a staged model pick. No-op when nothing is
* downloading (e.g. a native/local file that was never fetched). Lets non-React
* callers (the chat store's abandon paths) stop a staged transfer without the
* useRepoDownload hook. */
export function cancelStagedModelDownload(
pending: { id: string; ggufVariant?: string | null } | null,
): void {
if (!pending) return;
const variant = pending.ggufVariant ?? null;
const activeJob = selectActiveJob(
useDownloadManagerStore.getState(),
DOWNLOAD_KIND.MODEL,
pending.id,
variant,
);
void downloadManager.cancel(
activeJob?.key ?? jobKeyOf(DOWNLOAD_KIND.MODEL, pending.id, variant),
);
}
if (import.meta.hot) {
import.meta.hot.dispose(() => {
__resetDownloadManagerForTests();

View file

@ -20,6 +20,7 @@ export {
} from "./constants";
export {
__resetDownloadManagerForTests,
cancelStagedModelDownload,
clearCompletedInventoryHint,
downloadManager,
hydrateDownloadManager,

View file

@ -1004,6 +1004,23 @@ export function ModelsPage() {
(opts: ModelLoadOptions, isDownloaded: boolean) => {
if (!selectedModel) return;
const runId = selectedModel.resource.runId;
// "Load on selection" off: stage GGUF picks instead of loading, so the
// chat page's staging flow can read the header and show the load options.
// Non-GGUF models have nothing to configure pre-load, so they load now.
if (
!useChatRuntimeStore.getState().loadOnSelection &&
(opts.ggufVariant != null || selectedModel.isGguf)
) {
useChatRuntimeStore.getState().stageModel({
id: runId,
ggufVariant: opts.ggufVariant,
isGguf: selectedModel.isGguf,
isDownloaded,
expectedBytes: opts.expectedBytes,
});
openNewChat();
return;
}
void selectModel({
id: runId,
ggufVariant: opts.ggufVariant,
@ -1012,6 +1029,7 @@ export function ModelsPage() {
throwOnError: true,
})
.then(() => {
// Read fresh: the load is async, so the checkpoint may have changed.
const store = useChatRuntimeStore.getState();
if (!modelIdsMatch(store.params.checkpoint, runId)) {
store.setCheckpoint(runId, opts.ggufVariant ?? null);

View file

@ -0,0 +1,4 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
export { cancelStagedModelDownload } from "./download-manager";