From 1611514ebe9f8fe0729e2ca76ea90230f7cb0b7f Mon Sep 17 00:00:00 2001 From: sneakr Date: Thu, 9 Jul 2026 19:24:12 +0200 Subject: [PATCH] Fix GGUF context auto-fit and gated model config token Stop forcing a 32768 context when a GGUF native context is unknown so the backend auto-fits to VRAM again, while still honoring an explicit context edit. Send the HF token as a query param so gated safetensors models resolve their max position embeddings. Derive model default state during render to drop the set-state-in-effect calls. --- .../model-picker/api/model-metadata.ts | 8 +- .../components/model-config-page.tsx | 8 +- .../model-picker/hooks/use-model-defaults.ts | 127 ++++++++++-------- 3 files changed, 77 insertions(+), 66 deletions(-) diff --git a/studio/frontend/src/features/model-picker/api/model-metadata.ts b/studio/frontend/src/features/model-picker/api/model-metadata.ts index 3b3ad876c0..843e5439fc 100644 --- a/studio/frontend/src/features/model-picker/api/model-metadata.ts +++ b/studio/frontend/src/features/model-picker/api/model-metadata.ts @@ -2,7 +2,6 @@ // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import { authFetch } from "@/features/auth"; -import { hubTokenHeader } from "@/features/hub/lib/hub-token-header"; import { readFastApiError } from "@/lib/format-fastapi-error"; async function parseJsonOrThrow(response: Response): Promise { @@ -17,9 +16,12 @@ export async function fetchModelMaxPositionEmbeddings( hfToken?: string | null, signal?: AbortSignal, ): Promise { + const query = hfToken?.trim() + ? `?hf_token=${encodeURIComponent(hfToken.trim())}` + : ""; const response = await authFetch( - `/api/models/config/${encodeURIComponent(modelName)}`, - { headers: hubTokenHeader(hfToken), signal }, + `/api/models/config/${encodeURIComponent(modelName)}${query}`, + { signal }, ); const data = await parseJsonOrThrow<{ max_position_embeddings?: unknown }>( response, diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx index 815d48c0bf..712158b97b 100644 --- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx +++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx @@ -458,10 +458,10 @@ export function ModelConfigPage({ const runtimeConfig = target.isGguf ? { ...config, - customContextLength: resolveCustomContextLength( - contextValue, - contextBaseline, - ), + customContextLength: + contextBaseline == null && config.customContextLength == null + ? null + : resolveCustomContextLength(contextValue, contextBaseline), } : { ...config, diff --git a/studio/frontend/src/features/model-picker/hooks/use-model-defaults.ts b/studio/frontend/src/features/model-picker/hooks/use-model-defaults.ts index c6769cb08d..57dab863cc 100644 --- a/studio/frontend/src/features/model-picker/hooks/use-model-defaults.ts +++ b/studio/frontend/src/features/model-picker/hooks/use-model-defaults.ts @@ -53,28 +53,18 @@ export function useDefaultChatTemplate( enabled: boolean, ): DefaultChatTemplateState { const token = useHfTokenStore((s) => s.token); - const [state, setState] = useState({ - template: null, - loading: false, - error: null, - }); + const cacheKey = + enabled && modelId ? `${modelId}::${ggufVariant ?? ""}::${token}` : null; + const [fetched, setFetched] = useState<{ + key: string; + state: DefaultChatTemplateState; + } | null>(null); useEffect(() => { - if (!(enabled && modelId)) { + if (cacheKey == null || !modelId || templateCache.has(cacheKey)) { return; } - const cacheKey = `${modelId}::${ggufVariant ?? ""}::${token}`; - if (templateCache.has(cacheKey)) { - setState({ - template: templateCache.get(cacheKey) ?? null, - loading: false, - error: null, - }); - return; - } - const controller = new AbortController(); - setState({ template: null, loading: true, error: null }); fetchDefaultChatTemplate(modelId, ggufVariant, token, controller.signal) .then((template) => { if (controller.signal.aborted) { @@ -83,23 +73,43 @@ export function useDefaultChatTemplate( if (!(template === null && looksLikeLocalPath(modelId))) { cacheTemplate(cacheKey, template); } - setState({ template, loading: false, error: null }); + setFetched({ + key: cacheKey, + state: { template, loading: false, error: null }, + }); }) .catch((err: unknown) => { if (controller.signal.aborted) { return; } - setState({ - template: null, - loading: false, - error: err instanceof Error ? err.message : "Failed to load template", + setFetched({ + key: cacheKey, + state: { + template: null, + loading: false, + error: + err instanceof Error ? err.message : "Failed to load template", + }, }); }); return () => controller.abort(); - }, [modelId, ggufVariant, enabled, token]); + }, [cacheKey, modelId, ggufVariant, token]); - return state; + if (cacheKey == null) { + return { template: null, loading: false, error: null }; + } + if (templateCache.has(cacheKey)) { + return { + template: templateCache.get(cacheKey) ?? null, + loading: false, + error: null, + }; + } + if (fetched?.key === cacheKey) { + return fetched.state; + } + return { template: null, loading: true, error: null }; } export function useModelMaxPositionEmbeddings( @@ -107,61 +117,60 @@ export function useModelMaxPositionEmbeddings( enabled: boolean, ): ModelMaxPositionState { const token = useHfTokenStore((s) => s.token); - const [state, setState] = useState({ - maxPositionEmbeddings: null, - loading: false, - error: null, - }); + const cacheKey = enabled && modelId ? `${modelId}::${token}` : null; + const [fetched, setFetched] = useState<{ + key: string; + state: ModelMaxPositionState; + } | null>(null); useEffect(() => { - if (!(enabled && modelId)) { - setState({ - maxPositionEmbeddings: null, - loading: false, - error: null, - }); + if (cacheKey == null || !modelId || maxPositionCache.has(cacheKey)) { return; } - const cacheKey = `${modelId}::${token}`; - if (maxPositionCache.has(cacheKey)) { - setState({ - maxPositionEmbeddings: maxPositionCache.get(cacheKey) ?? null, - loading: false, - error: null, - }); - return; - } - const controller = new AbortController(); - setState({ maxPositionEmbeddings: null, loading: true, error: null }); fetchModelMaxPositionEmbeddings(modelId, token, controller.signal) .then((maxPositionEmbeddings) => { if (controller.signal.aborted) { return; } cacheMaxPosition(cacheKey, maxPositionEmbeddings); - setState({ - maxPositionEmbeddings, - loading: false, - error: null, + setFetched({ + key: cacheKey, + state: { maxPositionEmbeddings, loading: false, error: null }, }); }) .catch((err: unknown) => { if (controller.signal.aborted) { return; } - setState({ - maxPositionEmbeddings: null, - loading: false, - error: - err instanceof Error - ? err.message - : "Failed to load model metadata", + setFetched({ + key: cacheKey, + state: { + maxPositionEmbeddings: null, + loading: false, + error: + err instanceof Error + ? err.message + : "Failed to load model metadata", + }, }); }); return () => controller.abort(); - }, [modelId, enabled, token]); + }, [cacheKey, modelId, token]); - return state; + if (cacheKey == null) { + return { maxPositionEmbeddings: null, loading: false, error: null }; + } + if (maxPositionCache.has(cacheKey)) { + return { + maxPositionEmbeddings: maxPositionCache.get(cacheKey) ?? null, + loading: false, + error: null, + }; + } + if (fetched?.key === cacheKey) { + return fetched.state; + } + return { maxPositionEmbeddings: null, loading: true, error: null }; }