a5d6e6928d* Studio: surface the llama.cpp update affordance when MTP is disabled When a model asks for MTP (auto on an MTP model, or forced mtp / mtp+ngram) but it gets disabled, the load already degrades gracefully and serves without speculative decoding. Until now the UI gave no hint why, or that an update would fix it. Record why MTP was dropped on the backend (spec_fallback_reason): the probe found no mtp token (binary_no_mtp), the spawn aborted with an outdated-arch / context-build error such as a prebuilt that predates the Gemma drafter (binary_outdated), or the current build could not run it, e.g. a CUDA kernel limit (runtime_error). Expose it in the inference status. In the chat Speculative Decoding section, show a short note and, for the two update-fixable reasons, an inline Update llama.cpp button that reuses the existing update flow. A runtime_error gets the note without an update push, since a newer build may not fix it. Backend tests cover the reason being set / cleared. Frontend typechecks. * Address review: tighten the update hint to genuinely outdated binaries Reserve binary_outdated (which surfaces the Update llama.cpp affordance) for an unknown-architecture abort, which proves the prebuilt predates the model; classify the generic memory/context build failures as runtime_error, where an update may not help. Frontend: only append the "Update llama.cpp to enable it" sentence when an update is actually available, so the text never points at an action the UI is not offering.
1237 lines
47 KiB
TypeScript
1237 lines
47 KiB
TypeScript
// SPDX-License-Identifier: AGPL-3.0-only
|
|
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
|
|
|
import { createElement, useCallback, useRef, useState } from "react";
|
|
import { toast } from "@/lib/toast";
|
|
import { consumeNativePathToken } from "@/features/native-intents/api";
|
|
import {
|
|
notifyNative,
|
|
primeNativeNotificationPermission,
|
|
safeNotificationLabel,
|
|
sanitizeNotificationBody,
|
|
} from "@/lib/native-notifications";
|
|
import { ModelLoadDescription } from "../components/model-load-status";
|
|
import {
|
|
getDownloadProgress,
|
|
getGgufDownloadProgress,
|
|
getInferenceStatus,
|
|
getLoadProgress,
|
|
listLoras,
|
|
listModels,
|
|
loadModel,
|
|
unloadModel,
|
|
validateModel,
|
|
} from "../api/chat-api";
|
|
import { formatEta, formatRate } from "../utils/format-transfer";
|
|
import {
|
|
CHAT_REASONING_ENABLED_KEY,
|
|
loadOptionalBool,
|
|
type ReasoningEffort,
|
|
resolveToolsEnabledOnLoad,
|
|
useChatRuntimeStore,
|
|
} from "../stores/chat-runtime-store";
|
|
import {
|
|
mergeBackendRecommendedInference,
|
|
resolveLoadMaxSeqLength,
|
|
} from "../presets/preset-policy";
|
|
import {
|
|
isMultimodalResponse,
|
|
} from "../types/api";
|
|
import { isExternalModelId } from "../external-providers";
|
|
import type {
|
|
ChatLoraSummary,
|
|
ChatModelSummary,
|
|
} from "../types/runtime";
|
|
|
|
type SelectedModelInput = {
|
|
id: string;
|
|
isLora?: boolean;
|
|
ggufVariant?: string;
|
|
loadingDescription?: string;
|
|
isDownloaded?: boolean;
|
|
expectedBytes?: number;
|
|
forceReload?: boolean;
|
|
nativePathToken?: string;
|
|
throwOnError?: boolean;
|
|
};
|
|
|
|
const MODEL_LOAD_TOAST_CLASSNAMES = {
|
|
toast: "chat-model-load-toast items-center gap-2.5",
|
|
content: "gap-0.5 flex-1 min-w-0",
|
|
title: "leading-5",
|
|
description: "mt-0 w-full",
|
|
cancelButton:
|
|
"!h-auto !rounded-none !border-0 !bg-transparent !px-1 !text-[11px] !font-normal !text-muted-foreground hover:!bg-transparent hover:!text-destructive focus-visible:!text-destructive",
|
|
} as const;
|
|
|
|
const MODEL_LOADED_TOAST_CLASSNAMES = {
|
|
toast: "chat-model-loaded-toast items-center gap-2.5",
|
|
} as const;
|
|
|
|
const LORA_SUFFIX_RE = /_(\d{9,})$/;
|
|
|
|
function parseTrailingEpoch(input: string): number | undefined {
|
|
const match = input.match(LORA_SUFFIX_RE);
|
|
if (!match) {
|
|
return undefined;
|
|
}
|
|
const parsed = Number.parseInt(match[1], 10);
|
|
return Number.isFinite(parsed) ? parsed : undefined;
|
|
}
|
|
|
|
function stripTrailingEpoch(input: string): string {
|
|
const cleaned = input.replace(LORA_SUFFIX_RE, "").replace(/[_-]+$/, "").trim();
|
|
return cleaned || input;
|
|
}
|
|
|
|
function shortModelLabel(idOrName: string): string {
|
|
const slash = idOrName.lastIndexOf("/");
|
|
const label = slash >= 0 ? idOrName.slice(slash + 1) : idOrName;
|
|
return label || idOrName;
|
|
}
|
|
|
|
function describeModel(model: {
|
|
is_lora?: boolean;
|
|
is_vision?: boolean;
|
|
is_gguf?: boolean;
|
|
is_mlx?: boolean;
|
|
is_audio?: boolean;
|
|
has_audio_input?: boolean;
|
|
}): string | undefined {
|
|
const tags: string[] = [];
|
|
if (model.is_gguf) tags.push("GGUF");
|
|
if (model.is_mlx) tags.push("MLX");
|
|
if (model.is_lora) tags.push("LoRA");
|
|
if (model.is_vision) tags.push("Vision");
|
|
if (model.is_audio) tags.push("Audio");
|
|
if (model.has_audio_input) tags.push("Audio Input");
|
|
if (
|
|
!model.is_lora &&
|
|
!model.is_vision &&
|
|
!model.is_gguf &&
|
|
!model.is_mlx &&
|
|
!model.is_audio &&
|
|
!model.has_audio_input
|
|
)
|
|
tags.push("Base");
|
|
return tags.join(" · ");
|
|
}
|
|
|
|
function toChatModelSummary(model: {
|
|
id: string;
|
|
name?: string | null;
|
|
is_lora?: boolean;
|
|
is_vision?: boolean;
|
|
is_gguf?: boolean;
|
|
is_mlx?: boolean;
|
|
is_audio?: boolean;
|
|
audio_type?: string | null;
|
|
has_audio_input?: boolean;
|
|
}): ChatModelSummary {
|
|
return {
|
|
id: model.id,
|
|
name: model.name || model.id,
|
|
description: describeModel(model),
|
|
isLora: Boolean(model.is_lora),
|
|
isVision: Boolean(model.is_vision),
|
|
isGguf: Boolean(model.is_gguf),
|
|
isMlx: Boolean(model.is_mlx),
|
|
isAudio: Boolean(model.is_audio),
|
|
audioType: model.audio_type ?? null,
|
|
hasAudioInput: Boolean(model.has_audio_input),
|
|
};
|
|
}
|
|
|
|
// Merge capability flags from a load/status response into the matching
|
|
// models[] entry. /api/models/list omits audio capability for default and
|
|
// active-GGUF entries, so the attach gates (`activeModel?.hasAudioInput`)
|
|
// would otherwise stay false. Mirrors the compare composer's sync.
|
|
// Exported for tests.
|
|
export function syncModelCapabilities(
|
|
modelId: string,
|
|
resp: {
|
|
display_name?: string | null;
|
|
is_vision?: boolean;
|
|
is_lora?: boolean;
|
|
is_gguf?: boolean;
|
|
is_audio?: boolean;
|
|
audio_type?: string | null;
|
|
has_audio_input?: boolean;
|
|
},
|
|
): void {
|
|
const store = useChatRuntimeStore.getState();
|
|
const models = store.models;
|
|
const synced = {
|
|
isVision: Boolean(resp.is_vision),
|
|
isGguf: Boolean(resp.is_gguf),
|
|
isAudio: Boolean(resp.is_audio),
|
|
audioType: resp.audio_type ?? null,
|
|
hasAudioInput: Boolean(resp.has_audio_input),
|
|
};
|
|
const idx = models.findIndex((m) => m.id === modelId);
|
|
if (idx === -1) {
|
|
store.setModels([
|
|
...models,
|
|
{
|
|
id: modelId,
|
|
name: resp.display_name || modelId,
|
|
isLora: Boolean(resp.is_lora),
|
|
...synced,
|
|
},
|
|
]);
|
|
} else {
|
|
const next = [...models];
|
|
next[idx] = { ...next[idx], ...synced };
|
|
store.setModels(next);
|
|
}
|
|
}
|
|
|
|
function toLoraSummary(lora: {
|
|
display_name: string;
|
|
adapter_path: string;
|
|
base_model?: string | null;
|
|
source?: "training" | "exported" | null;
|
|
export_type?: "lora" | "merged" | "gguf" | null;
|
|
}): ChatLoraSummary {
|
|
const idTail = lora.adapter_path.split("/").filter(Boolean).at(-1) ?? "";
|
|
const updatedAt =
|
|
parseTrailingEpoch(lora.display_name) ?? parseTrailingEpoch(idTail);
|
|
|
|
return {
|
|
id: lora.adapter_path,
|
|
name: stripTrailingEpoch(lora.display_name),
|
|
baseModel: lora.base_model || "Unknown base model",
|
|
updatedAt,
|
|
source: lora.source ?? undefined,
|
|
exportType: lora.export_type ?? undefined,
|
|
};
|
|
}
|
|
|
|
function getTrustRemoteCodeRequiredMessage(modelName: string): string {
|
|
return `${modelName} needs custom code enabled to load. Turn on "Enable custom code" in Chat Settings, then try again.`;
|
|
}
|
|
|
|
// Canonicalises any backend/persisted value onto the Speculative Decoding
|
|
// dropdown's modes ("auto"/"mtp"/"ngram"/"mtp+ngram"/"off"/null). Mirrors
|
|
// backend _canonicalize_spec_mode so legacy persisted values round-trip.
|
|
function normalizeSpeculativeType(v: string | null | undefined): string | null {
|
|
if (v == null) return null;
|
|
const s = String(v).trim().toLowerCase();
|
|
if (!s) return null;
|
|
if (s === "auto" || s === "default") return "auto";
|
|
if (s === "off") return "off";
|
|
if (s === "ngram-simple") return "ngram-simple";
|
|
if (s === "mtp" || s === "draft-mtp") return "mtp";
|
|
if (s === "ngram" || s === "ngram-mod") return "ngram";
|
|
if (s === "mtp+ngram") return "mtp+ngram";
|
|
// Comma-chained legacy values (e.g. from older persisted state).
|
|
const parts = s.split(",").map((p) => p.trim()).filter(Boolean);
|
|
const hasMtp = parts.some((p) => p === "mtp" || p === "draft-mtp");
|
|
const hasNgram = parts.some((p) => p === "ngram" || p === "ngram-mod");
|
|
if (hasMtp && hasNgram) return "mtp+ngram";
|
|
if (hasMtp) return "mtp";
|
|
if (hasNgram) return "ngram";
|
|
// Unknown -> safe fallback to Auto so the dropdown stays controlled.
|
|
return "auto";
|
|
}
|
|
|
|
type LocalReasoningEffort = Extract<ReasoningEffort, "low" | "medium" | "high">;
|
|
|
|
function clampLocalReasoningEffort(value: ReasoningEffort): LocalReasoningEffort {
|
|
if (value === "low" || value === "medium" || value === "high") {
|
|
return value;
|
|
}
|
|
return "low";
|
|
}
|
|
|
|
export function useChatModelRuntime() {
|
|
const params = useChatRuntimeStore((state) => state.params);
|
|
const models = useChatRuntimeStore((state) => state.models);
|
|
const loras = useChatRuntimeStore((state) => state.loras);
|
|
const setModels = useChatRuntimeStore((state) => state.setModels);
|
|
const setLoras = useChatRuntimeStore((state) => state.setLoras);
|
|
const setParams = useChatRuntimeStore((state) => state.setParams);
|
|
const setModelsError = useChatRuntimeStore((state) => state.setModelsError);
|
|
const setCheckpoint = useChatRuntimeStore((state) => state.setCheckpoint);
|
|
const clearCheckpoint = useChatRuntimeStore((state) => state.clearCheckpoint);
|
|
|
|
const [loadingModel, setLoadingModel] = useState<{
|
|
id: string;
|
|
displayName: string;
|
|
isDownloaded?: boolean;
|
|
isCachedLora?: boolean;
|
|
nativePathToken?: string | null;
|
|
} | null>(null);
|
|
const [loadToastDismissed, setLoadToastDismissed] = useState(false);
|
|
const [loadProgress, setLoadProgress] = useState<{
|
|
percent: number | null;
|
|
label: string | null;
|
|
phase: "downloading" | "starting";
|
|
} | null>(null);
|
|
const loadAbortRef = useRef<AbortController | null>(null);
|
|
const loadingModelRef = useRef<typeof loadingModel>(null);
|
|
const loadToastIdRef = useRef<string | number | null>(null);
|
|
const loadAttemptRef = useRef(0);
|
|
const loadToastDismissedRef = useRef(false);
|
|
const cancelUnloadPendingRef = useRef(false);
|
|
|
|
const setLoadToastDismissedState = useCallback((dismissed: boolean) => {
|
|
loadToastDismissedRef.current = dismissed;
|
|
setLoadToastDismissed(dismissed);
|
|
}, []);
|
|
|
|
const resetLoadingUi = useCallback(() => {
|
|
setLoadingModel(null);
|
|
setLoadProgress(null);
|
|
loadingModelRef.current = null;
|
|
loadAbortRef.current = null;
|
|
loadToastIdRef.current = null;
|
|
setLoadToastDismissedState(false);
|
|
if (!cancelUnloadPendingRef.current) {
|
|
useChatRuntimeStore.getState().setModelLoading(false);
|
|
}
|
|
}, [setLoadToastDismissedState]);
|
|
|
|
const renderLoadDescription = useCallback(
|
|
(
|
|
title: string,
|
|
message: string,
|
|
progressPercent?: number | null,
|
|
progressLabel?: string | null,
|
|
) =>
|
|
createElement(ModelLoadDescription, {
|
|
title,
|
|
message,
|
|
progressPercent,
|
|
progressLabel,
|
|
}),
|
|
[],
|
|
);
|
|
|
|
const refresh = useCallback(async (options?: { signal?: AbortSignal }) => {
|
|
const signal = options?.signal;
|
|
setModelsError(null);
|
|
try {
|
|
const [listRes, statusRes, lorasRes] = await Promise.all([
|
|
listModels(),
|
|
getInferenceStatus(),
|
|
listLoras(),
|
|
]);
|
|
|
|
// Cancellation can land while the requests above are in flight. Bail
|
|
// before writing backend state back -- cancelLoading already cleared it.
|
|
if (signal?.aborted) return;
|
|
|
|
setModels(listRes.models.map(toChatModelSummary));
|
|
setLoras(lorasRes.loras.map(toLoraSummary));
|
|
|
|
const selectedCheckpoint = useChatRuntimeStore.getState().params.checkpoint;
|
|
const isExternalSelectionActive = isExternalModelId(selectedCheckpoint);
|
|
if (statusRes.active_model && !isExternalSelectionActive) {
|
|
setCheckpoint(statusRes.active_model, statusRes.gguf_variant);
|
|
|
|
// Apply inference defaults on reconnect (page refresh with model already loaded)
|
|
if (statusRes.inference) {
|
|
const currentParams = useChatRuntimeStore.getState().params;
|
|
setParams(
|
|
mergeBackendRecommendedInference({
|
|
current: currentParams,
|
|
response: statusRes,
|
|
modelId: statusRes.active_model,
|
|
presetSource: useChatRuntimeStore.getState().activePresetSource,
|
|
}),
|
|
);
|
|
}
|
|
|
|
// Restore reasoning/tools support flags and context length
|
|
const hydratingExistingModel =
|
|
selectedCheckpoint !== statusRes.active_model ||
|
|
useChatRuntimeStore.getState().activeGgufVariant !==
|
|
(statusRes.gguf_variant ?? null);
|
|
const supportsReasoning = statusRes.supports_reasoning ?? false;
|
|
const reasoningAlwaysOn = statusRes.reasoning_always_on ?? false;
|
|
const reasoningStyle = statusRes.reasoning_style ?? "enable_thinking";
|
|
const reasoningEffortLevels =
|
|
reasoningStyle === "reasoning_effort"
|
|
? (["low", "medium", "high"] as const)
|
|
: (["low", "medium", "high"] as const);
|
|
const supportsPreserveThinking = statusRes.supports_preserve_thinking ?? false;
|
|
const supportsTools = statusRes.supports_tools ?? false;
|
|
const storedReasoningEnabled = loadOptionalBool(
|
|
CHAT_REASONING_ENABLED_KEY,
|
|
);
|
|
const currentGgufContextLength = statusRes.is_gguf
|
|
? (statusRes.context_length ?? null)
|
|
: null;
|
|
const ggufMaxContextLength = statusRes.is_gguf
|
|
? (statusRes.max_context_length ?? null)
|
|
: null;
|
|
const ggufNativeContextLength = statusRes.is_gguf
|
|
? (statusRes.native_context_length ?? null)
|
|
: null;
|
|
const currentSpecType = normalizeSpeculativeType(
|
|
statusRes.speculative_type,
|
|
);
|
|
// Refresh runs on F5 (needs hydration) and right after a load (store
|
|
// already set). For user-configurable params, only hydrate when the
|
|
// shadow `loaded*` field is null ("not yet hydrated"); otherwise we'd
|
|
// clobber what the load path just applied and revert the user.
|
|
const prevState = useChatRuntimeStore.getState();
|
|
const clampedReasoningEffort = clampLocalReasoningEffort(
|
|
prevState.reasoningEffort,
|
|
);
|
|
const nextDefaultChatTemplate =
|
|
statusRes.chat_template === undefined
|
|
? prevState.defaultChatTemplate
|
|
: statusRes.chat_template;
|
|
useChatRuntimeStore.setState({
|
|
supportsReasoning,
|
|
reasoningAlwaysOn,
|
|
reasoningStyle,
|
|
supportsReasoningOff: reasoningStyle !== "reasoning_effort",
|
|
reasoningEffortLevels,
|
|
reasoningEffort: clampedReasoningEffort,
|
|
supportsPreserveThinking,
|
|
supportsTools,
|
|
// Reset per-turn reasoning flag so:
|
|
// 1. non-reasoning models don't inherit a stale off state, and
|
|
// 2. local reasoning-effort models (Off hidden via
|
|
// supportsReasoningOff=false) don't carry reasoningEnabled=false
|
|
// from an external model where Off was selected -- the composer
|
|
// would still show "Think: <level>" but the adapter would omit
|
|
// the kwarg, so Harmony falls back to its default effort.
|
|
reasoningEnabled: supportsReasoning
|
|
? reasoningStyle === "reasoning_effort"
|
|
? true
|
|
: useChatRuntimeStore.getState().reasoningEnabled
|
|
: true,
|
|
ggufContextLength: currentGgufContextLength,
|
|
ggufMaxContextLength,
|
|
ggufNativeContextLength,
|
|
modelRequiresTrustRemoteCode:
|
|
statusRes.requires_trust_remote_code ?? false,
|
|
defaultChatTemplate: nextDefaultChatTemplate,
|
|
loadedIsMultimodal: isMultimodalResponse(statusRes),
|
|
specFallbackReason: statusRes.spec_fallback_reason ?? null,
|
|
...(prevState.loadedSpeculativeType === null && {
|
|
speculativeType: currentSpecType,
|
|
loadedSpeculativeType: currentSpecType,
|
|
}),
|
|
...(statusRes.spec_draft_n_max !== undefined &&
|
|
prevState.loadedSpecDraftNMax === null &&
|
|
prevState.specDraftNMax === null && {
|
|
specDraftNMax: statusRes.spec_draft_n_max ?? null,
|
|
loadedSpecDraftNMax: statusRes.spec_draft_n_max ?? null,
|
|
}),
|
|
...(statusRes.cache_type_kv !== undefined &&
|
|
prevState.loadedKvCacheDtype === null && {
|
|
kvCacheDtype: statusRes.cache_type_kv,
|
|
loadedKvCacheDtype: statusRes.cache_type_kv,
|
|
}),
|
|
...(statusRes.chat_template_override !== undefined &&
|
|
prevState.loadedChatTemplateOverride === null &&
|
|
prevState.chatTemplateOverride === null && {
|
|
chatTemplateOverride: statusRes.chat_template_override,
|
|
loadedChatTemplateOverride: statusRes.chat_template_override,
|
|
}),
|
|
});
|
|
// setModels(listRes...) above used catalog data, which omits audio
|
|
// capability. Re-apply live status so attach gates survive a refresh.
|
|
syncModelCapabilities(statusRes.active_model, statusRes);
|
|
|
|
// Set reasoning default for Qwen3.5/3.6 small models
|
|
if (
|
|
supportsReasoning &&
|
|
hydratingExistingModel &&
|
|
storedReasoningEnabled === null
|
|
) {
|
|
let reasoningDefault = true;
|
|
const mid = statusRes.active_model.toLowerCase();
|
|
if (mid.includes("qwen3.5") || mid.includes("qwen3.6")) {
|
|
const sizeMatch = mid.match(/(\d+\.?\d*)\s*b/);
|
|
if (sizeMatch && parseFloat(sizeMatch[1]) < 9) {
|
|
reasoningDefault = false;
|
|
}
|
|
}
|
|
useChatRuntimeStore.setState({ reasoningEnabled: reasoningDefault });
|
|
}
|
|
} else if (!statusRes.active_model && !isExternalSelectionActive) {
|
|
useChatRuntimeStore.setState({
|
|
modelRequiresTrustRemoteCode: false,
|
|
loadedIsMultimodal: false,
|
|
});
|
|
}
|
|
} catch (error) {
|
|
if (signal?.aborted) return;
|
|
const message =
|
|
error instanceof Error ? error.message : "Failed to load models";
|
|
setModelsError(message);
|
|
toast.error("Failed to refresh models", {
|
|
description: message,
|
|
});
|
|
}
|
|
}, [setCheckpoint, setLoras, setModels, setModelsError, setParams]);
|
|
|
|
const cancelLoading = useCallback(() => {
|
|
const model = loadingModelRef.current;
|
|
if (!model) return;
|
|
loadAbortRef.current?.abort();
|
|
loadAbortRef.current = null;
|
|
loadingModelRef.current = null;
|
|
const tid = loadToastIdRef.current;
|
|
loadToastIdRef.current = null;
|
|
setLoadingModel(null);
|
|
setLoadProgress(null);
|
|
setLoadToastDismissedState(false);
|
|
clearCheckpoint();
|
|
if (tid != null) toast.dismiss(tid);
|
|
const isCachedOrLocal = model.isDownloaded || model.isCachedLora;
|
|
toast.info("Stopped loading model", {
|
|
description: isCachedOrLocal
|
|
? undefined
|
|
: "The current download may still finish in the background.",
|
|
});
|
|
cancelUnloadPendingRef.current = true;
|
|
useChatRuntimeStore.getState().setModelLoading(true);
|
|
void (async () => {
|
|
try {
|
|
await unloadModel({ model_path: model.id }).catch(() => {});
|
|
} finally {
|
|
cancelUnloadPendingRef.current = false;
|
|
if (!loadingModelRef.current) {
|
|
useChatRuntimeStore.getState().setModelLoading(false);
|
|
}
|
|
}
|
|
})();
|
|
}, [clearCheckpoint, setLoadToastDismissedState]);
|
|
|
|
const selectModel = useCallback(
|
|
async (selection: string | SelectedModelInput) => {
|
|
const modelId = typeof selection === "string" ? selection : selection.id;
|
|
const ggufVariant =
|
|
typeof selection === "string" ? undefined : selection.ggufVariant;
|
|
const forceReload =
|
|
typeof selection === "string" ? false : selection.forceReload ?? false;
|
|
const nativePathToken =
|
|
typeof selection === "string" ? undefined : selection.nativePathToken;
|
|
const throwOnError =
|
|
typeof selection === "string" ? false : selection.throwOnError ?? false;
|
|
const currentVariant = useChatRuntimeStore.getState().activeGgufVariant;
|
|
if (!forceReload && (!modelId || (params.checkpoint === modelId && (ggufVariant ?? null) === (currentVariant ?? null)))) {
|
|
return;
|
|
}
|
|
// Prevent duplicate loads if already loading this model
|
|
if (
|
|
loadingModelRef.current?.id === modelId &&
|
|
(loadingModelRef.current?.nativePathToken ?? null) === (nativePathToken ?? null)
|
|
)
|
|
return;
|
|
|
|
const explicitIsLora =
|
|
typeof selection === "string" ? undefined : selection.isLora;
|
|
const extraLoadingDescription =
|
|
typeof selection === "string" ? undefined : selection.loadingDescription;
|
|
const isDownloaded =
|
|
typeof selection === "string" ? false : selection.isDownloaded ?? false;
|
|
const model = models.find((entry) => entry.id === modelId);
|
|
const lora = loras.find((entry) => entry.id === modelId);
|
|
const loraIsAdapter = lora?.exportType === "lora";
|
|
const isLora =
|
|
explicitIsLora ?? model?.isLora ?? loraIsAdapter ?? false;
|
|
const displayName = model?.name || lora?.name || modelId;
|
|
const toastDisplayName = shortModelLabel(displayName);
|
|
const loadAttemptId = ++loadAttemptRef.current;
|
|
primeNativeNotificationPermission().catch(() => undefined);
|
|
const notificationModelKey = `${modelId}:${ggufVariant ?? ""}:${loadAttemptId}`;
|
|
const safeModelName = safeNotificationLabel(toastDisplayName, "The model");
|
|
const currentCheckpoint =
|
|
useChatRuntimeStore.getState().params.checkpoint;
|
|
const previousCheckpoint = currentCheckpoint;
|
|
const previousVariant =
|
|
useChatRuntimeStore.getState().activeGgufVariant ?? null;
|
|
const reloadingSameModel =
|
|
previousCheckpoint === modelId &&
|
|
(ggufVariant ?? null) === (previousVariant ?? null);
|
|
const previousModel = previousCheckpoint
|
|
? models.find((entry) => entry.id === previousCheckpoint)
|
|
: undefined;
|
|
const previousLora = previousCheckpoint
|
|
? loras.find((entry) => entry.id === previousCheckpoint)
|
|
: undefined;
|
|
const previousIsLora =
|
|
previousModel?.isLora ?? (previousLora?.exportType === "lora");
|
|
// Covers Unix absolute (/), relative (./ ../), tilde (~/), Windows drive (C:\), UNC (\\server)
|
|
const isLocal = /^(\/|\.{1,2}[\\/]|~[\\/]|[A-Za-z]:[\\/]|\\\\)/.test(modelId);
|
|
const isCachedLora = isLora && isLocal;
|
|
const loadingDescription = [
|
|
currentCheckpoint ? "Switching models." : null,
|
|
extraLoadingDescription ?? null,
|
|
isDownloaded ? "Loading cached model into memory." : null,
|
|
!isDownloaded && isCachedLora ? "Loading trained model into memory." : null,
|
|
]
|
|
.filter(Boolean)
|
|
.join(" ");
|
|
setModelsError(null);
|
|
setLoadToastDismissedState(false);
|
|
const loadInfo = {
|
|
id: modelId,
|
|
displayName,
|
|
isDownloaded,
|
|
isCachedLora,
|
|
nativePathToken: nativePathToken ?? null,
|
|
};
|
|
setLoadingModel(loadInfo);
|
|
useChatRuntimeStore.getState().setModelLoading(true);
|
|
setLoadProgress(
|
|
isDownloaded || isCachedLora
|
|
? { percent: null, label: null, phase: "starting" }
|
|
: { percent: 0, label: "Preparing download", phase: "downloading" },
|
|
);
|
|
loadingModelRef.current = loadInfo;
|
|
const abortCtrl = new AbortController();
|
|
loadAbortRef.current = abortCtrl;
|
|
try {
|
|
async function performLoad(): Promise<void> {
|
|
if (abortCtrl.signal.aborted) throw new Error("Cancelled");
|
|
let previousWasUnloaded = false;
|
|
const currentCheckpoint =
|
|
useChatRuntimeStore.getState().params.checkpoint;
|
|
const stateBeforeUnload = useChatRuntimeStore.getState();
|
|
const trustRemoteCode = stateBeforeUnload.params.trustRemoteCode ?? false;
|
|
const maxSeqLength = stateBeforeUnload.params.maxSeqLength;
|
|
const previousIsGguf =
|
|
previousModel?.isGguf === true
|
|
|| previousVariant != null
|
|
|| (previousCheckpoint?.toLowerCase().endsWith(".gguf") ?? false);
|
|
const rollbackMaxSeqLength = previousIsGguf
|
|
? (stateBeforeUnload.ggufContextLength ?? 0)
|
|
: maxSeqLength;
|
|
const hfToken = stateBeforeUnload.hfToken || null;
|
|
const previousModelRequiresTrustRemoteCode =
|
|
stateBeforeUnload.modelRequiresTrustRemoteCode;
|
|
const previousActiveNativePathToken =
|
|
stateBeforeUnload.activeNativePathToken;
|
|
try {
|
|
// Lightweight pre-flight validation: avoid unloading a working model
|
|
// if the new identifier is clearly invalid (e.g. bad HF id / path).
|
|
const validateNativePathLease = nativePathToken
|
|
? (await consumeNativePathToken(nativePathToken, "validate-model")).nativePathLease
|
|
: undefined;
|
|
const validation = await validateModel({
|
|
model_path: modelId,
|
|
nativePathLease: validateNativePathLease,
|
|
hf_token: hfToken,
|
|
max_seq_length: maxSeqLength,
|
|
load_in_4bit: true,
|
|
is_lora: isLora,
|
|
gguf_variant: ggufVariant ?? null,
|
|
});
|
|
if (validation.requires_trust_remote_code && !trustRemoteCode) {
|
|
throw new Error(getTrustRemoteCodeRequiredMessage(displayName));
|
|
}
|
|
if (abortCtrl.signal.aborted) throw new Error("Cancelled");
|
|
const loadNativePathLease = nativePathToken
|
|
? (await consumeNativePathToken(nativePathToken, "load-model")).nativePathLease
|
|
: undefined;
|
|
|
|
if (currentCheckpoint) {
|
|
await unloadModel({ model_path: currentCheckpoint });
|
|
previousWasUnloaded = true;
|
|
}
|
|
if (abortCtrl.signal.aborted) throw new Error("Cancelled");
|
|
|
|
// Reset Speculative Decoding to Auto on model switch: spec
|
|
// strategy is per-model, so a sub-3B non-MTP GGUF's "Off" must
|
|
// not carry into a 27B MTP GGUF where Auto auto-promotes to
|
|
// draft-mtp. Clears the stale prior choice so the backend's
|
|
// platform-aware path runs by default; same for spec_draft_n_max
|
|
// (MTP-only). The user can still force a mode on the new model.
|
|
if (currentCheckpoint && currentCheckpoint !== modelId) {
|
|
useChatRuntimeStore.setState({
|
|
speculativeType: null,
|
|
loadedSpeculativeType: null,
|
|
specDraftNMax: null,
|
|
loadedSpecDraftNMax: null,
|
|
});
|
|
}
|
|
|
|
const {
|
|
chatTemplateOverride,
|
|
kvCacheDtype,
|
|
customContextLength,
|
|
ggufContextLength,
|
|
speculativeType,
|
|
specDraftNMax,
|
|
activePresetSource,
|
|
activeGgufVariant,
|
|
} = useChatRuntimeStore.getState();
|
|
const effectiveMaxSeqLength = resolveLoadMaxSeqLength({
|
|
modelId,
|
|
ggufVariant,
|
|
customContextLength,
|
|
ggufContextLength,
|
|
currentCheckpoint,
|
|
activeGgufVariant,
|
|
maxSeqLength,
|
|
presetSource: activePresetSource,
|
|
});
|
|
const effectiveChatTemplateOverride =
|
|
chatTemplateOverride?.trim() ? chatTemplateOverride : null;
|
|
const loadResponse = await loadModel({
|
|
model_path: modelId,
|
|
nativePathLease: loadNativePathLease,
|
|
hf_token: hfToken,
|
|
max_seq_length: effectiveMaxSeqLength,
|
|
load_in_4bit: true,
|
|
is_lora: isLora,
|
|
gguf_variant: ggufVariant ?? null,
|
|
trust_remote_code: trustRemoteCode,
|
|
chat_template_override: effectiveChatTemplateOverride,
|
|
cache_type_kv: kvCacheDtype,
|
|
speculative_type: speculativeType,
|
|
spec_draft_n_max: specDraftNMax,
|
|
});
|
|
|
|
// If cancelled while loading, don't update UI to show
|
|
// the model as active -- it's being unloaded.
|
|
if (abortCtrl.signal.aborted) throw new Error("Cancelled");
|
|
|
|
const currentParams = useChatRuntimeStore.getState().params;
|
|
setParams(
|
|
mergeBackendRecommendedInference({
|
|
current: currentParams,
|
|
response: loadResponse,
|
|
modelId,
|
|
presetSource: useChatRuntimeStore.getState().activePresetSource,
|
|
}),
|
|
);
|
|
// Qwen3.5/3.6 small models (0.8B, 2B, 4B, 9B) disable thinking by default
|
|
let reasoningDefault = loadResponse.supports_reasoning ?? false;
|
|
if (reasoningDefault) {
|
|
const mid = modelId.toLowerCase();
|
|
if (mid.includes("qwen3.5") || mid.includes("qwen3.6")) {
|
|
const sizeMatch = mid.match(/(\d+\.?\d*)\s*b/);
|
|
if (sizeMatch && parseFloat(sizeMatch[1]) < 9) {
|
|
reasoningDefault = false;
|
|
}
|
|
}
|
|
}
|
|
const loadedKv = loadResponse.cache_type_kv ?? null;
|
|
const loadedSpec = normalizeSpeculativeType(
|
|
loadResponse.speculative_type,
|
|
);
|
|
const nativeCtx = loadResponse.is_gguf
|
|
? (loadResponse.context_length ?? 131072)
|
|
: null;
|
|
const reportedMaxCtx = loadResponse.is_gguf
|
|
? (loadResponse.max_context_length ?? null)
|
|
: null;
|
|
const reportedNativeCtx = loadResponse.is_gguf
|
|
? (loadResponse.native_context_length ?? null)
|
|
: null;
|
|
// A successful reload has applied settings, so clear pending custom
|
|
// context state and display the backend-reported effective context.
|
|
const keepCustomCtx = null;
|
|
const reasoningAlwaysOn = loadResponse.reasoning_always_on ?? false;
|
|
const reasoningStyle = loadResponse.reasoning_style ?? "enable_thinking";
|
|
const supportsReasoning = loadResponse.supports_reasoning ?? false;
|
|
const supportsTools = loadResponse.supports_tools ?? false;
|
|
const reasoningEffortLevels =
|
|
reasoningStyle === "reasoning_effort"
|
|
? (["low", "medium", "high"] as const)
|
|
: (["low", "medium", "high"] as const);
|
|
const existingReasoningEffort = useChatRuntimeStore.getState().reasoningEffort;
|
|
const clampedReasoningEffort = clampLocalReasoningEffort(
|
|
existingReasoningEffort,
|
|
);
|
|
const ggufMaxContextLength = reportedMaxCtx;
|
|
const nextReasoningEnabled = reasoningAlwaysOn
|
|
? true
|
|
: reloadingSameModel && supportsReasoning
|
|
? stateBeforeUnload.reasoningEnabled
|
|
: reasoningDefault;
|
|
useChatRuntimeStore.setState({
|
|
ggufContextLength: nativeCtx,
|
|
ggufMaxContextLength,
|
|
ggufNativeContextLength: reportedNativeCtx,
|
|
modelRequiresTrustRemoteCode:
|
|
loadResponse.requires_trust_remote_code ?? false,
|
|
supportsReasoning,
|
|
reasoningAlwaysOn,
|
|
reasoningEnabled: nextReasoningEnabled,
|
|
reasoningStyle,
|
|
supportsReasoningOff: reasoningStyle !== "reasoning_effort",
|
|
reasoningEffortLevels,
|
|
reasoningEffort: clampedReasoningEffort,
|
|
supportsPreserveThinking: loadResponse.supports_preserve_thinking ?? false,
|
|
supportsTools,
|
|
...(reloadingSameModel && supportsTools
|
|
? {
|
|
toolsEnabled: stateBeforeUnload.toolsEnabled,
|
|
codeToolsEnabled: stateBeforeUnload.codeToolsEnabled,
|
|
}
|
|
: resolveToolsEnabledOnLoad(supportsTools)),
|
|
kvCacheDtype: loadedKv,
|
|
loadedKvCacheDtype: loadedKv,
|
|
speculativeType: loadedSpec,
|
|
loadedSpeculativeType: loadedSpec,
|
|
specDraftNMax: loadResponse.spec_draft_n_max ?? null,
|
|
loadedSpecDraftNMax: loadResponse.spec_draft_n_max ?? null,
|
|
customContextLength: keepCustomCtx,
|
|
defaultChatTemplate: loadResponse.chat_template ?? null,
|
|
chatTemplateOverride: effectiveChatTemplateOverride,
|
|
loadedChatTemplateOverride: effectiveChatTemplateOverride,
|
|
loadedIsMultimodal: isMultimodalResponse(loadResponse),
|
|
activeNativePathToken: nativePathToken ?? null,
|
|
});
|
|
// Unlock attach menus for capabilities the catalog entry lacked.
|
|
syncModelCapabilities(modelId, loadResponse);
|
|
// Qwen3/3.5/3.6: apply thinking-mode-specific params after load
|
|
if (
|
|
modelId.toLowerCase().includes("qwen3") &&
|
|
(loadResponse.supports_reasoning ?? false)
|
|
) {
|
|
const store = useChatRuntimeStore.getState();
|
|
if (store.activePresetSource === "builtin-default") {
|
|
const mid = modelId.toLowerCase();
|
|
const needsPresencePenalty =
|
|
mid.includes("qwen3.5") || mid.includes("qwen3.6");
|
|
const p = nextReasoningEnabled
|
|
? {
|
|
temperature: 0.6,
|
|
topP: 0.95,
|
|
topK: 20,
|
|
minP: 0.0,
|
|
...(needsPresencePenalty
|
|
? { presencePenalty: 1.5 }
|
|
: {}),
|
|
}
|
|
: {
|
|
temperature: 0.7,
|
|
topP: 0.8,
|
|
topK: 20,
|
|
minP: 0.0,
|
|
...(needsPresencePenalty
|
|
? { presencePenalty: 1.5 }
|
|
: {}),
|
|
};
|
|
store.setParams({ ...store.params, ...p });
|
|
}
|
|
}
|
|
await refresh({ signal: abortCtrl.signal });
|
|
} catch (error) {
|
|
// Skip rollback if user cancelled -- model is already being unloaded.
|
|
if (abortCtrl.signal.aborted) throw error;
|
|
// If we unloaded a previous model and the new load failed, attempt a rollback.
|
|
if (previousWasUnloaded && previousCheckpoint) {
|
|
let rollbackNativePathLease: string | undefined;
|
|
if (previousActiveNativePathToken) {
|
|
try {
|
|
rollbackNativePathLease = (
|
|
await consumeNativePathToken(previousActiveNativePathToken, "load-model")
|
|
).nativePathLease;
|
|
} catch {
|
|
throw new Error(
|
|
"Could not reload the previous local model: please re-select the file.",
|
|
);
|
|
}
|
|
}
|
|
try {
|
|
await loadModel({
|
|
model_path: previousCheckpoint,
|
|
nativePathLease: rollbackNativePathLease,
|
|
hf_token: hfToken,
|
|
max_seq_length: rollbackMaxSeqLength,
|
|
load_in_4bit: true,
|
|
is_lora: previousIsLora,
|
|
gguf_variant: previousVariant,
|
|
trust_remote_code:
|
|
previousModelRequiresTrustRemoteCode || trustRemoteCode,
|
|
});
|
|
useChatRuntimeStore.setState({
|
|
activeNativePathToken: previousActiveNativePathToken ?? null,
|
|
});
|
|
await refresh();
|
|
} catch {
|
|
// Rollback also failed; surface the original load error below.
|
|
}
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
const isCachedLoad = isDownloaded || isCachedLora;
|
|
const toastTitle = isCachedLoad ? "Starting model…" : "Downloading model…";
|
|
const modelLoadToastOptions = (description: ReturnType<typeof renderLoadDescription>) => ({
|
|
description,
|
|
duration: Infinity,
|
|
closeButton: true,
|
|
cancel: {
|
|
label: "Cancel",
|
|
onClick: cancelLoading,
|
|
},
|
|
classNames: MODEL_LOAD_TOAST_CLASSNAMES,
|
|
onDismiss: (dismissedToast: { id: string | number }) => {
|
|
if (loadToastIdRef.current !== dismissedToast.id) {
|
|
return;
|
|
}
|
|
setLoadToastDismissedState(true);
|
|
},
|
|
});
|
|
const toastId = toast(
|
|
null,
|
|
modelLoadToastOptions(
|
|
renderLoadDescription(
|
|
toastTitle,
|
|
loadingDescription,
|
|
isCachedLoad ? null : 0,
|
|
isCachedLoad ? null : "Preparing download",
|
|
),
|
|
),
|
|
);
|
|
loadToastIdRef.current = toastId;
|
|
|
|
// Poll download progress for non-cached models, then (after download
|
|
// or for cached models) poll the llama-server mmap phase so "Starting
|
|
// model..." doesn't look frozen for minutes on large MoE models.
|
|
let progressInterval: ReturnType<typeof setInterval> | null = null;
|
|
const expectedBytes =
|
|
typeof selection !== "string" ? selection.expectedBytes ?? 0 : 0;
|
|
|
|
// Rolling window of byte samples for rate/ETA estimation, shared
|
|
// across download + mmap phases so it survives phase flips.
|
|
type Sample = { t: number; b: number };
|
|
const MIN_SAMPLES = 3;
|
|
const MIN_WINDOW = 3_000; // ms
|
|
const MAX_WINDOW = 15_000; // ms
|
|
const dlSamples: Sample[] = [];
|
|
const mmapSamples: Sample[] = [];
|
|
|
|
function estimate(
|
|
samples: Sample[],
|
|
bytes: number,
|
|
total: number,
|
|
): { rate: number; eta: number; stable: boolean } {
|
|
const now = Date.now();
|
|
// Drop samples if the counter reset (e.g. phase flipped).
|
|
if (samples.length > 0 && bytes < samples[samples.length - 1].b) {
|
|
samples.length = 0;
|
|
}
|
|
samples.push({ t: now, b: bytes });
|
|
const cutoff = now - MAX_WINDOW;
|
|
while (samples.length > 2 && samples[0].t < cutoff) {
|
|
samples.shift();
|
|
}
|
|
if (samples.length < MIN_SAMPLES) {
|
|
return { rate: 0, eta: 0, stable: false };
|
|
}
|
|
const first = samples[0];
|
|
const last = samples[samples.length - 1];
|
|
const dt = (last.t - first.t) / 1000;
|
|
const db = last.b - first.b;
|
|
if (dt * 1000 < MIN_WINDOW || db <= 0) {
|
|
return { rate: 0, eta: 0, stable: false };
|
|
}
|
|
const rate = db / dt;
|
|
const eta =
|
|
total > 0 && bytes < total && rate > 0 ? (total - bytes) / rate : 0;
|
|
return { rate, eta, stable: true };
|
|
}
|
|
|
|
function composeProgressLabel(
|
|
dlGb: number,
|
|
totalGb: number,
|
|
bytes: number,
|
|
total: number,
|
|
samples: Sample[],
|
|
): string {
|
|
const base =
|
|
totalGb > 0
|
|
? `${dlGb.toFixed(1)} of ${totalGb.toFixed(1)} GB`
|
|
: `${dlGb.toFixed(1)} GB downloaded`;
|
|
const est = estimate(samples, bytes, total);
|
|
if (!est.stable) return base;
|
|
const rateStr = formatRate(est.rate);
|
|
const etaStr = total > 0 ? formatEta(est.eta) : "";
|
|
return etaStr && etaStr !== "--"
|
|
? `${base} • ${rateStr} • ${etaStr} left`
|
|
: `${base} • ${rateStr}`;
|
|
}
|
|
|
|
let downloadComplete = isDownloaded || isCachedLora;
|
|
|
|
const pollDownload = async () => {
|
|
if (abortCtrl.signal.aborted || !loadingModelRef.current) {
|
|
if (progressInterval) clearInterval(progressInterval);
|
|
return;
|
|
}
|
|
try {
|
|
const prog =
|
|
ggufVariant && expectedBytes > 0
|
|
? await getGgufDownloadProgress(modelId, ggufVariant, expectedBytes)
|
|
: await getDownloadProgress(modelId);
|
|
if (!loadingModelRef.current) return;
|
|
|
|
if (prog.progress > 0 && prog.progress < 1) {
|
|
hasShownProgress = true;
|
|
const dlGb = prog.downloaded_bytes / (1024 ** 3);
|
|
const totalGb = prog.expected_bytes / (1024 ** 3);
|
|
const pct = Math.round(prog.progress * 100);
|
|
const progressLabel = composeProgressLabel(
|
|
dlGb,
|
|
totalGb,
|
|
prog.downloaded_bytes,
|
|
prog.expected_bytes,
|
|
dlSamples,
|
|
);
|
|
setLoadProgress({
|
|
percent: pct,
|
|
label: progressLabel,
|
|
phase: "downloading",
|
|
});
|
|
if (loadToastDismissedRef.current) return;
|
|
toast(null, {
|
|
id: toastId,
|
|
...modelLoadToastOptions(
|
|
renderLoadDescription(
|
|
"Downloading model…",
|
|
loadingDescription,
|
|
pct,
|
|
progressLabel,
|
|
),
|
|
),
|
|
});
|
|
} else if (
|
|
prog.downloaded_bytes > 0 &&
|
|
prog.expected_bytes === 0 &&
|
|
prog.progress === 0
|
|
) {
|
|
hasShownProgress = true;
|
|
const dlGb = prog.downloaded_bytes / (1024 ** 3);
|
|
const est = estimate(dlSamples, prog.downloaded_bytes, 0);
|
|
const rateSuffix =
|
|
est.stable ? ` • ${formatRate(est.rate)}` : "";
|
|
setLoadProgress({
|
|
percent: null,
|
|
label: `${dlGb.toFixed(1)} GB downloaded${rateSuffix}`,
|
|
phase: "downloading",
|
|
});
|
|
} else if (prog.progress >= 1 && hasShownProgress) {
|
|
downloadComplete = true;
|
|
setLoadProgress({
|
|
percent: 100,
|
|
label: "Download complete",
|
|
phase: "starting",
|
|
});
|
|
if (!loadToastDismissedRef.current) {
|
|
toast(null, {
|
|
id: toastId,
|
|
...modelLoadToastOptions(
|
|
renderLoadDescription(
|
|
"Starting model…",
|
|
"Download complete. Loading the model into memory.",
|
|
100,
|
|
"Download complete",
|
|
),
|
|
),
|
|
});
|
|
}
|
|
notifyNative({
|
|
key: `model-downloaded:${notificationModelKey}`,
|
|
title: "Model downloaded",
|
|
body: `${safeModelName} finished downloading and is loading into memory.`,
|
|
requestPermission: false,
|
|
}).catch(() => undefined);
|
|
// Keep polling: the mmap branch below takes over from here.
|
|
}
|
|
} catch {
|
|
// Ignore polling errors; keep polling.
|
|
}
|
|
};
|
|
|
|
const pollLoad = async () => {
|
|
if (abortCtrl.signal.aborted || !loadingModelRef.current) {
|
|
if (progressInterval) clearInterval(progressInterval);
|
|
return;
|
|
}
|
|
try {
|
|
const prog = await getLoadProgress();
|
|
if (!loadingModelRef.current) return;
|
|
if (!prog || prog.phase == null) return;
|
|
if (prog.phase === "ready") {
|
|
// Loaded. The chat flow will flip loadingModelRef shortly;
|
|
// just stop polling.
|
|
if (progressInterval) clearInterval(progressInterval);
|
|
return;
|
|
}
|
|
if (prog.bytes_total <= 0) return; // nothing useful to render
|
|
const loadedGb = prog.bytes_loaded / (1024 ** 3);
|
|
const totalGb = prog.bytes_total / (1024 ** 3);
|
|
const pct = Math.min(99, Math.round(prog.fraction * 100));
|
|
const est = estimate(mmapSamples, prog.bytes_loaded, prog.bytes_total);
|
|
const base = `${loadedGb.toFixed(1)} of ${totalGb.toFixed(1)} GB in memory`;
|
|
const label = est.stable
|
|
? `${base} • ${formatRate(est.rate)}${
|
|
formatEta(est.eta) !== "--" ? ` • ${formatEta(est.eta)} left` : ""
|
|
}`
|
|
: base;
|
|
setLoadProgress({
|
|
percent: pct,
|
|
label,
|
|
phase: "starting",
|
|
});
|
|
if (loadToastDismissedRef.current) return;
|
|
toast(null, {
|
|
id: toastId,
|
|
...modelLoadToastOptions(
|
|
renderLoadDescription(
|
|
"Starting model…",
|
|
"Paging weights into memory.",
|
|
pct,
|
|
label,
|
|
),
|
|
),
|
|
});
|
|
} catch {
|
|
// Ignore polling errors.
|
|
}
|
|
};
|
|
|
|
const pollProgress = async () => {
|
|
if (!downloadComplete) {
|
|
await pollDownload();
|
|
} else {
|
|
await pollLoad();
|
|
}
|
|
};
|
|
|
|
let hasShownProgress = false;
|
|
setTimeout(pollProgress, 500);
|
|
progressInterval = setInterval(pollProgress, 2000);
|
|
|
|
try {
|
|
await performLoad();
|
|
// User cancelled mid-refresh; cancelLoading handles teardown.
|
|
if (abortCtrl.signal.aborted) return;
|
|
if (loadToastDismissedRef.current) {
|
|
toast.success(`${toastDisplayName} loaded`, {
|
|
classNames: MODEL_LOADED_TOAST_CLASSNAMES,
|
|
closeButton: true,
|
|
duration: 8000,
|
|
});
|
|
} else {
|
|
toast.success(`${toastDisplayName} loaded`, {
|
|
id: toastId,
|
|
description: undefined,
|
|
cancel: undefined,
|
|
classNames: MODEL_LOADED_TOAST_CLASSNAMES,
|
|
closeButton: true,
|
|
duration: 8000,
|
|
onDismiss: undefined,
|
|
});
|
|
}
|
|
notifyNative({
|
|
key: `model-loaded:${notificationModelKey}`,
|
|
title: "Model ready",
|
|
body: `${safeModelName} is loaded and ready to chat.`,
|
|
requestPermission: false,
|
|
}).catch(() => undefined);
|
|
} catch (err) {
|
|
if (!abortCtrl.signal.aborted) {
|
|
const message =
|
|
err instanceof Error ? err.message : "Failed to load model";
|
|
if (loadToastDismissedRef.current) {
|
|
toast.error(message);
|
|
} else {
|
|
toast.error(message, {
|
|
id: toastId,
|
|
description: undefined,
|
|
cancel: undefined,
|
|
classNames: undefined,
|
|
closeButton: true,
|
|
duration: 8000,
|
|
onDismiss: undefined,
|
|
});
|
|
}
|
|
notifyNative({
|
|
key: `model-load-failed:${notificationModelKey}`,
|
|
title: "Model failed to load",
|
|
body: sanitizeNotificationBody(message, "The model failed to load."),
|
|
requestPermission: false,
|
|
}).catch(() => undefined);
|
|
}
|
|
throw err;
|
|
} finally {
|
|
if (progressInterval) clearInterval(progressInterval);
|
|
resetLoadingUi();
|
|
}
|
|
} catch (error) {
|
|
if (abortCtrl.signal.aborted) return; // User cancelled, nothing to report
|
|
resetLoadingUi();
|
|
const message =
|
|
error instanceof Error ? error.message : "Failed to load model";
|
|
setModelsError(message);
|
|
if (throwOnError) {
|
|
throw error instanceof Error ? error : new Error(message);
|
|
}
|
|
}
|
|
},
|
|
[
|
|
cancelLoading,
|
|
loras,
|
|
models,
|
|
params.checkpoint,
|
|
refresh,
|
|
renderLoadDescription,
|
|
resetLoadingUi,
|
|
setLoadToastDismissedState,
|
|
setModelsError,
|
|
setParams,
|
|
],
|
|
);
|
|
|
|
const ejectModel = useCallback(async (): Promise<boolean> => {
|
|
if (!params.checkpoint) {
|
|
return false;
|
|
}
|
|
setModelsError(null);
|
|
if (isExternalModelId(params.checkpoint)) {
|
|
clearCheckpoint();
|
|
await refresh();
|
|
return true;
|
|
}
|
|
try {
|
|
async function performUnload(): Promise<void> {
|
|
await unloadModel({ model_path: params.checkpoint });
|
|
clearCheckpoint();
|
|
await refresh();
|
|
}
|
|
|
|
const unloadPromise = performUnload();
|
|
toast.promise(unloadPromise, {
|
|
loading: "Unloading model",
|
|
success: { message: "Model unloaded", duration: 1200 },
|
|
error: (err) =>
|
|
err instanceof Error ? err.message : "Failed to unload model",
|
|
description: "Releases VRAM and resets inference state.",
|
|
});
|
|
await unloadPromise;
|
|
return true;
|
|
} catch (error) {
|
|
const message =
|
|
error instanceof Error ? error.message : "Failed to unload model";
|
|
setModelsError(message);
|
|
return false;
|
|
}
|
|
}, [clearCheckpoint, params.checkpoint, refresh, setModelsError]);
|
|
|
|
return {
|
|
refresh,
|
|
selectModel,
|
|
ejectModel,
|
|
cancelLoading,
|
|
loadingModel,
|
|
loadProgress,
|
|
loadToastDismissed,
|
|
};
|
|
}
|