Fix GGUF context auto-fit and gated model config token
Stop forcing a 32768 context when a GGUF native context is unknown so the backend auto-fits to VRAM again, while still honoring an explicit context edit. Send the HF token as a query param so gated safetensors models resolve their max position embeddings. Derive model default state during render to drop the set-state-in-effect calls.
This commit is contained in:
parent
c0a0c4d0bf
commit
1611514ebe
3 changed files with 77 additions and 66 deletions
|
|
@ -2,7 +2,6 @@
|
|||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { authFetch } from "@/features/auth";
|
||||
import { hubTokenHeader } from "@/features/hub/lib/hub-token-header";
|
||||
import { readFastApiError } from "@/lib/format-fastapi-error";
|
||||
|
||||
async function parseJsonOrThrow<T>(response: Response): Promise<T> {
|
||||
|
|
@ -17,9 +16,12 @@ export async function fetchModelMaxPositionEmbeddings(
|
|||
hfToken?: string | null,
|
||||
signal?: AbortSignal,
|
||||
): Promise<number | null> {
|
||||
const query = hfToken?.trim()
|
||||
? `?hf_token=${encodeURIComponent(hfToken.trim())}`
|
||||
: "";
|
||||
const response = await authFetch(
|
||||
`/api/models/config/${encodeURIComponent(modelName)}`,
|
||||
{ headers: hubTokenHeader(hfToken), signal },
|
||||
`/api/models/config/${encodeURIComponent(modelName)}${query}`,
|
||||
{ signal },
|
||||
);
|
||||
const data = await parseJsonOrThrow<{ max_position_embeddings?: unknown }>(
|
||||
response,
|
||||
|
|
|
|||
|
|
@ -458,10 +458,10 @@ export function ModelConfigPage({
|
|||
const runtimeConfig = target.isGguf
|
||||
? {
|
||||
...config,
|
||||
customContextLength: resolveCustomContextLength(
|
||||
contextValue,
|
||||
contextBaseline,
|
||||
),
|
||||
customContextLength:
|
||||
contextBaseline == null && config.customContextLength == null
|
||||
? null
|
||||
: resolveCustomContextLength(contextValue, contextBaseline),
|
||||
}
|
||||
: {
|
||||
...config,
|
||||
|
|
|
|||
|
|
@ -53,28 +53,18 @@ export function useDefaultChatTemplate(
|
|||
enabled: boolean,
|
||||
): DefaultChatTemplateState {
|
||||
const token = useHfTokenStore((s) => s.token);
|
||||
const [state, setState] = useState<DefaultChatTemplateState>({
|
||||
template: null,
|
||||
loading: false,
|
||||
error: null,
|
||||
});
|
||||
const cacheKey =
|
||||
enabled && modelId ? `${modelId}::${ggufVariant ?? ""}::${token}` : null;
|
||||
const [fetched, setFetched] = useState<{
|
||||
key: string;
|
||||
state: DefaultChatTemplateState;
|
||||
} | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (!(enabled && modelId)) {
|
||||
if (cacheKey == null || !modelId || templateCache.has(cacheKey)) {
|
||||
return;
|
||||
}
|
||||
const cacheKey = `${modelId}::${ggufVariant ?? ""}::${token}`;
|
||||
if (templateCache.has(cacheKey)) {
|
||||
setState({
|
||||
template: templateCache.get(cacheKey) ?? null,
|
||||
loading: false,
|
||||
error: null,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
setState({ template: null, loading: true, error: null });
|
||||
fetchDefaultChatTemplate(modelId, ggufVariant, token, controller.signal)
|
||||
.then((template) => {
|
||||
if (controller.signal.aborted) {
|
||||
|
|
@ -83,23 +73,43 @@ export function useDefaultChatTemplate(
|
|||
if (!(template === null && looksLikeLocalPath(modelId))) {
|
||||
cacheTemplate(cacheKey, template);
|
||||
}
|
||||
setState({ template, loading: false, error: null });
|
||||
setFetched({
|
||||
key: cacheKey,
|
||||
state: { template, loading: false, error: null },
|
||||
});
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
if (controller.signal.aborted) {
|
||||
return;
|
||||
}
|
||||
setState({
|
||||
template: null,
|
||||
loading: false,
|
||||
error: err instanceof Error ? err.message : "Failed to load template",
|
||||
setFetched({
|
||||
key: cacheKey,
|
||||
state: {
|
||||
template: null,
|
||||
loading: false,
|
||||
error:
|
||||
err instanceof Error ? err.message : "Failed to load template",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
return () => controller.abort();
|
||||
}, [modelId, ggufVariant, enabled, token]);
|
||||
}, [cacheKey, modelId, ggufVariant, token]);
|
||||
|
||||
return state;
|
||||
if (cacheKey == null) {
|
||||
return { template: null, loading: false, error: null };
|
||||
}
|
||||
if (templateCache.has(cacheKey)) {
|
||||
return {
|
||||
template: templateCache.get(cacheKey) ?? null,
|
||||
loading: false,
|
||||
error: null,
|
||||
};
|
||||
}
|
||||
if (fetched?.key === cacheKey) {
|
||||
return fetched.state;
|
||||
}
|
||||
return { template: null, loading: true, error: null };
|
||||
}
|
||||
|
||||
export function useModelMaxPositionEmbeddings(
|
||||
|
|
@ -107,61 +117,60 @@ export function useModelMaxPositionEmbeddings(
|
|||
enabled: boolean,
|
||||
): ModelMaxPositionState {
|
||||
const token = useHfTokenStore((s) => s.token);
|
||||
const [state, setState] = useState<ModelMaxPositionState>({
|
||||
maxPositionEmbeddings: null,
|
||||
loading: false,
|
||||
error: null,
|
||||
});
|
||||
const cacheKey = enabled && modelId ? `${modelId}::${token}` : null;
|
||||
const [fetched, setFetched] = useState<{
|
||||
key: string;
|
||||
state: ModelMaxPositionState;
|
||||
} | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (!(enabled && modelId)) {
|
||||
setState({
|
||||
maxPositionEmbeddings: null,
|
||||
loading: false,
|
||||
error: null,
|
||||
});
|
||||
if (cacheKey == null || !modelId || maxPositionCache.has(cacheKey)) {
|
||||
return;
|
||||
}
|
||||
const cacheKey = `${modelId}::${token}`;
|
||||
if (maxPositionCache.has(cacheKey)) {
|
||||
setState({
|
||||
maxPositionEmbeddings: maxPositionCache.get(cacheKey) ?? null,
|
||||
loading: false,
|
||||
error: null,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
setState({ maxPositionEmbeddings: null, loading: true, error: null });
|
||||
fetchModelMaxPositionEmbeddings(modelId, token, controller.signal)
|
||||
.then((maxPositionEmbeddings) => {
|
||||
if (controller.signal.aborted) {
|
||||
return;
|
||||
}
|
||||
cacheMaxPosition(cacheKey, maxPositionEmbeddings);
|
||||
setState({
|
||||
maxPositionEmbeddings,
|
||||
loading: false,
|
||||
error: null,
|
||||
setFetched({
|
||||
key: cacheKey,
|
||||
state: { maxPositionEmbeddings, loading: false, error: null },
|
||||
});
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
if (controller.signal.aborted) {
|
||||
return;
|
||||
}
|
||||
setState({
|
||||
maxPositionEmbeddings: null,
|
||||
loading: false,
|
||||
error:
|
||||
err instanceof Error
|
||||
? err.message
|
||||
: "Failed to load model metadata",
|
||||
setFetched({
|
||||
key: cacheKey,
|
||||
state: {
|
||||
maxPositionEmbeddings: null,
|
||||
loading: false,
|
||||
error:
|
||||
err instanceof Error
|
||||
? err.message
|
||||
: "Failed to load model metadata",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
return () => controller.abort();
|
||||
}, [modelId, enabled, token]);
|
||||
}, [cacheKey, modelId, token]);
|
||||
|
||||
return state;
|
||||
if (cacheKey == null) {
|
||||
return { maxPositionEmbeddings: null, loading: false, error: null };
|
||||
}
|
||||
if (maxPositionCache.has(cacheKey)) {
|
||||
return {
|
||||
maxPositionEmbeddings: maxPositionCache.get(cacheKey) ?? null,
|
||||
loading: false,
|
||||
error: null,
|
||||
};
|
||||
}
|
||||
if (fetched?.key === cacheKey) {
|
||||
return fetched.state;
|
||||
}
|
||||
return { maxPositionEmbeddings: null, loading: true, error: null };
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue