Fix model picker per-model config regressions
- Stop reloading the already loaded model on re-pick - Hide infra models from the chat picker - Detect vision support on cached GGUF repos - Honor saved maxSeqLength on auto load - Restore default chat template for local GGUFs - Warn on save failure and revert config on cancel - Refetch picker inventory on open - Persist read only per model config safely
This commit is contained in:
parent
a615a40284
commit
f4838782cb
16 changed files with 213 additions and 135 deletions
|
|
@ -31,6 +31,7 @@ from hub.services.models.common import (
|
||||||
_is_checkpoint_weight_name,
|
_is_checkpoint_weight_name,
|
||||||
_is_gguf_filename,
|
_is_gguf_filename,
|
||||||
_is_main_gguf_filename,
|
_is_main_gguf_filename,
|
||||||
|
_is_mmproj_filename,
|
||||||
_is_transformers_safetensors_weight_name,
|
_is_transformers_safetensors_weight_name,
|
||||||
_local_inventory_id,
|
_local_inventory_id,
|
||||||
_prefer_complete_larger,
|
_prefer_complete_larger,
|
||||||
|
|
@ -143,6 +144,14 @@ def _repo_gguf_last_modified(repo_info) -> float:
|
||||||
return latest
|
return latest
|
||||||
|
|
||||||
|
|
||||||
|
def _repo_has_mmproj(repo_info) -> bool:
|
||||||
|
return any(
|
||||||
|
_is_mmproj_filename(f.file_name)
|
||||||
|
for revision in repo_info.revisions
|
||||||
|
for f in revision.files
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _cached_repo_file_name(file_obj) -> str:
|
def _cached_repo_file_name(file_obj) -> str:
|
||||||
file_path = getattr(file_obj, "file_path", None)
|
file_path = getattr(file_obj, "file_path", None)
|
||||||
if file_path:
|
if file_path:
|
||||||
|
|
@ -270,6 +279,8 @@ def _scan_cached_gguf() -> list[dict]:
|
||||||
requires_variant = True,
|
requires_variant = True,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
if _repo_has_mmproj(repo_info):
|
||||||
|
row["capabilities"]["supports_vision"] = True
|
||||||
if _prefer_cache_row(row, existing):
|
if _prefer_cache_row(row, existing):
|
||||||
seen_lower[key] = row
|
seen_lower[key] = row
|
||||||
elif last_modified > existing.get("last_modified", 0.0):
|
elif last_modified > existing.get("last_modified", 0.0):
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,7 @@ from hub.services.models.folder_browser import (
|
||||||
_build_browse_allowlist,
|
_build_browse_allowlist,
|
||||||
_is_path_inside_allowlist,
|
_is_path_inside_allowlist,
|
||||||
)
|
)
|
||||||
|
from hub.utils.gguf import iter_hf_cache_snapshots
|
||||||
from utils.models.gguf_metadata import read_gguf_chat_template
|
from utils.models.gguf_metadata import read_gguf_chat_template
|
||||||
from utils.models.model_config import (
|
from utils.models.model_config import (
|
||||||
_extract_quant_label,
|
_extract_quant_label,
|
||||||
|
|
@ -25,7 +26,6 @@ from utils.models.model_config import (
|
||||||
_is_mtp_drafter,
|
_is_mtp_drafter,
|
||||||
)
|
)
|
||||||
from utils.paths.path_utils import (
|
from utils.paths.path_utils import (
|
||||||
get_cache_path,
|
|
||||||
is_local_path,
|
is_local_path,
|
||||||
normalize_path,
|
normalize_path,
|
||||||
resolve_cached_repo_id_case,
|
resolve_cached_repo_id_case,
|
||||||
|
|
@ -210,18 +210,6 @@ def _chat_template_from_dir(dir_path: Path, gguf_variant: Optional[str] = None)
|
||||||
return _chat_template_from_tokenizer_dir(dir_path) or from_gguf()
|
return _chat_template_from_tokenizer_dir(dir_path) or from_gguf()
|
||||||
|
|
||||||
|
|
||||||
def _snapshots_newest_first(snapshots_dir: Path) -> list[Path]:
|
|
||||||
dirs_with_mtime: list[tuple[float, Path]] = []
|
|
||||||
for entry in snapshots_dir.iterdir():
|
|
||||||
try:
|
|
||||||
if entry.is_dir():
|
|
||||||
dirs_with_mtime.append((entry.stat().st_mtime, entry))
|
|
||||||
except OSError:
|
|
||||||
continue
|
|
||||||
dirs_with_mtime.sort(key = lambda item: item[0], reverse = True)
|
|
||||||
return [entry for _, entry in dirs_with_mtime]
|
|
||||||
|
|
||||||
|
|
||||||
def read_default_chat_template(
|
def read_default_chat_template(
|
||||||
model_name: str,
|
model_name: str,
|
||||||
hf_token: Optional[str] = None,
|
hf_token: Optional[str] = None,
|
||||||
|
|
@ -250,14 +238,10 @@ def read_default_chat_template(
|
||||||
resolved = resolve_cached_repo_id_case(name)
|
resolved = resolve_cached_repo_id_case(name)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
repo_dir = get_cache_path(resolved)
|
for snapshot in iter_hf_cache_snapshots(resolved):
|
||||||
if repo_dir is not None and repo_dir.exists():
|
template = _chat_template_from_dir(snapshot, gguf_variant)
|
||||||
snapshots_dir = repo_dir / "snapshots"
|
if template:
|
||||||
if snapshots_dir.exists():
|
return template
|
||||||
for snapshot in _snapshots_newest_first(snapshots_dir):
|
|
||||||
template = _chat_template_from_dir(snapshot, gguf_variant)
|
|
||||||
if template:
|
|
||||||
return template
|
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.debug("Could not read cached chat template for %s: %s", resolved, exc)
|
logger.debug("Could not read cached chat template for %s: %s", resolved, exc)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,16 @@
|
||||||
# SPDX-License-Identifier: AGPL-3.0-only
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||||
|
|
||||||
from picker.service import _find_gguf_in_dir, _iter_ggufs
|
import json
|
||||||
|
|
||||||
|
from picker.service import (
|
||||||
|
_chat_template_from_dir,
|
||||||
|
_chat_template_from_tokenizer_config,
|
||||||
|
_chat_template_from_tokenizer_dir,
|
||||||
|
_find_gguf_in_dir,
|
||||||
|
_iter_ggufs,
|
||||||
|
validate_chat_template,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_iter_ggufs_skips_gguf_companions(tmp_path):
|
def test_iter_ggufs_skips_gguf_companions(tmp_path):
|
||||||
|
|
@ -46,3 +55,68 @@ def test_find_gguf_in_dir_matches_bpw_variant_base_label(tmp_path):
|
||||||
assert _find_gguf_in_dir(tmp_path, "IQ4_XS") == target
|
assert _find_gguf_in_dir(tmp_path, "IQ4_XS") == target
|
||||||
assert _find_gguf_in_dir(tmp_path, "IQ4_XS-3.53bpw") == target
|
assert _find_gguf_in_dir(tmp_path, "IQ4_XS-3.53bpw") == target
|
||||||
assert _find_gguf_in_dir(tmp_path, "Q4_K") is None
|
assert _find_gguf_in_dir(tmp_path, "Q4_K") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_chat_template_accepts_valid_and_empty():
|
||||||
|
assert validate_chat_template("{{ messages[0].content }}").valid is True
|
||||||
|
assert validate_chat_template("").valid is True
|
||||||
|
assert validate_chat_template(" ").valid is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_chat_template_reports_syntax_error_with_line():
|
||||||
|
result = validate_chat_template("{% if %}{% endif %}")
|
||||||
|
assert result.valid is False
|
||||||
|
assert result.error is not None
|
||||||
|
assert result.error.startswith("Line ")
|
||||||
|
|
||||||
|
|
||||||
|
def test_chat_template_from_tokenizer_config_reads_string():
|
||||||
|
assert _chat_template_from_tokenizer_config({"chat_template": "HELLO"}) == "HELLO"
|
||||||
|
assert _chat_template_from_tokenizer_config({"chat_template": " "}) is None
|
||||||
|
assert _chat_template_from_tokenizer_config({}) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_chat_template_from_tokenizer_config_prefers_named_default():
|
||||||
|
config = {
|
||||||
|
"chat_template": [
|
||||||
|
{"name": "tool_use", "template": "TOOL"},
|
||||||
|
{"name": "default", "template": "DEFAULT"},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
assert _chat_template_from_tokenizer_config(config) == "DEFAULT"
|
||||||
|
|
||||||
|
|
||||||
|
def test_chat_template_from_tokenizer_config_falls_back_to_first_entry():
|
||||||
|
config = {
|
||||||
|
"chat_template": [
|
||||||
|
{"name": "tool_use", "template": "TOOL"},
|
||||||
|
{"name": "other", "template": "OTHER"},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
assert _chat_template_from_tokenizer_config(config) == "TOOL"
|
||||||
|
|
||||||
|
|
||||||
|
def test_chat_template_from_tokenizer_dir_prefers_jinja_file(tmp_path):
|
||||||
|
(tmp_path / "chat_template.jinja").write_text("FROM_JINJA", encoding="utf-8")
|
||||||
|
(tmp_path / "tokenizer_config.json").write_text(
|
||||||
|
json.dumps({"chat_template": "FROM_CONFIG"}), encoding="utf-8"
|
||||||
|
)
|
||||||
|
assert _chat_template_from_tokenizer_dir(tmp_path) == "FROM_JINJA"
|
||||||
|
|
||||||
|
|
||||||
|
def test_chat_template_from_tokenizer_dir_reads_tokenizer_config(tmp_path):
|
||||||
|
(tmp_path / "tokenizer_config.json").write_text(
|
||||||
|
json.dumps({"chat_template": "FROM_CONFIG"}), encoding="utf-8"
|
||||||
|
)
|
||||||
|
assert _chat_template_from_tokenizer_dir(tmp_path) == "FROM_CONFIG"
|
||||||
|
|
||||||
|
|
||||||
|
def test_chat_template_from_dir_without_variant_prefers_tokenizer(tmp_path):
|
||||||
|
(tmp_path / "tokenizer_config.json").write_text(
|
||||||
|
json.dumps({"chat_template": "FROM_CONFIG"}), encoding="utf-8"
|
||||||
|
)
|
||||||
|
assert _chat_template_from_dir(tmp_path) == "FROM_CONFIG"
|
||||||
|
|
||||||
|
|
||||||
|
def test_chat_template_from_dir_returns_none_when_absent(tmp_path):
|
||||||
|
assert _chat_template_from_dir(tmp_path) is None
|
||||||
|
|
|
||||||
|
|
@ -1441,7 +1441,7 @@ async function autoLoadSmallestModel(): Promise<{
|
||||||
ggufContextLength: null,
|
ggufContextLength: null,
|
||||||
currentCheckpoint: currentStore.params.checkpoint,
|
currentCheckpoint: currentStore.params.checkpoint,
|
||||||
activeGgufVariant: currentStore.activeGgufVariant,
|
activeGgufVariant: currentStore.activeGgufVariant,
|
||||||
maxSeqLength: candidate.maxSeqLength,
|
maxSeqLength: config.maxSeqLength ?? candidate.maxSeqLength,
|
||||||
presetSource: currentStore.activePresetSource,
|
presetSource: currentStore.activePresetSource,
|
||||||
});
|
});
|
||||||
const effectiveSpeculativeType =
|
const effectiveSpeculativeType =
|
||||||
|
|
@ -1489,6 +1489,9 @@ async function autoLoadSmallestModel(): Promise<{
|
||||||
);
|
);
|
||||||
store.setParams({
|
store.setParams({
|
||||||
...store.params,
|
...store.params,
|
||||||
|
...(candidate.kind === "gguf"
|
||||||
|
? {}
|
||||||
|
: { maxSeqLength: effectiveMaxSeqLength }),
|
||||||
maxTokens:
|
maxTokens:
|
||||||
candidate.kind === "gguf"
|
candidate.kind === "gguf"
|
||||||
? loadResp.context_length ?? 131072
|
? loadResp.context_length ?? 131072
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,6 @@ import {
|
||||||
ModelSelector,
|
ModelSelector,
|
||||||
type ModelSelectorChangeMeta,
|
type ModelSelectorChangeMeta,
|
||||||
type PerModelConfig,
|
type PerModelConfig,
|
||||||
perModelConfigsEqual,
|
|
||||||
resolveInitialConfig,
|
resolveInitialConfig,
|
||||||
SidebarModelConfig,
|
SidebarModelConfig,
|
||||||
} from "@/features/model-picker";
|
} from "@/features/model-picker";
|
||||||
|
|
@ -1887,20 +1886,7 @@ export function ChatPage({
|
||||||
const isSameLoadedModel =
|
const isSameLoadedModel =
|
||||||
value === currentCheckpoint &&
|
value === currentCheckpoint &&
|
||||||
(meta?.ggufVariant ?? null) === (currentVariant ?? null);
|
(meta?.ggufVariant ?? null) === (currentVariant ?? null);
|
||||||
const metaIsGguf =
|
if (isSameLoadedModel && !meta?.forceReload) {
|
||||||
meta?.isGguf === true ||
|
|
||||||
meta?.ggufVariant != null ||
|
|
||||||
value.toLowerCase().endsWith(".gguf");
|
|
||||||
if (
|
|
||||||
isSameLoadedModel &&
|
|
||||||
(!meta?.config ||
|
|
||||||
perModelConfigsEqual(
|
|
||||||
meta.config,
|
|
||||||
currentRuntimePerModelConfig({
|
|
||||||
includeMaxSeqLength: !metaIsGguf,
|
|
||||||
}),
|
|
||||||
))
|
|
||||||
) {
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (meta?.source === "external" || isExternalModelId(value)) {
|
if (meta?.source === "external" || isExternalModelId(value)) {
|
||||||
|
|
@ -2103,6 +2089,7 @@ export function ChatPage({
|
||||||
isGguf: activeModelIsGguf,
|
isGguf: activeModelIsGguf,
|
||||||
isDownloaded: true,
|
isDownloaded: true,
|
||||||
config,
|
config,
|
||||||
|
forceReload: true,
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
[
|
[
|
||||||
|
|
@ -2731,7 +2718,7 @@ export function ChatPage({
|
||||||
params={inferenceParams}
|
params={inferenceParams}
|
||||||
onParamsChange={setInferenceParams}
|
onParamsChange={setInferenceParams}
|
||||||
modelConfig={
|
modelConfig={
|
||||||
view.mode !== "compare" && activeModelConfig ? (
|
view.mode !== "compare" && activeModelConfig && !modelLoading ? (
|
||||||
<SidebarModelConfig
|
<SidebarModelConfig
|
||||||
modelId={inferenceParams.checkpoint}
|
modelId={inferenceParams.checkpoint}
|
||||||
ggufVariant={activeGgufVariant ?? null}
|
ggufVariant={activeGgufVariant ?? null}
|
||||||
|
|
|
||||||
|
|
@ -1179,10 +1179,10 @@ export function useChatModelRuntime() {
|
||||||
resetLoadingUi();
|
resetLoadingUi();
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (abortCtrl.signal.aborted) return; // User cancelled, nothing to report
|
|
||||||
if (typeof selection !== "string" && selection.previousConfig) {
|
if (typeof selection !== "string" && selection.previousConfig) {
|
||||||
applyPerModelConfigToRuntime(selection.previousConfig);
|
applyPerModelConfigToRuntime(selection.previousConfig);
|
||||||
}
|
}
|
||||||
|
if (abortCtrl.signal.aborted) return; // User cancelled, nothing to report
|
||||||
resetLoadingUi();
|
resetLoadingUi();
|
||||||
const message =
|
const message =
|
||||||
error instanceof Error ? error.message : "Failed to load model";
|
error instanceof Error ? error.message : "Failed to load model";
|
||||||
|
|
|
||||||
|
|
@ -1,32 +1,19 @@
|
||||||
// SPDX-License-Identifier: AGPL-3.0-only
|
// SPDX-License-Identifier: AGPL-3.0-only
|
||||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||||
|
|
||||||
import { authFetch } from "@/features/auth";
|
import { getModelConfig } from "@/features/training";
|
||||||
import { readFastApiError } from "@/lib/format-fastapi-error";
|
|
||||||
|
|
||||||
async function parseJsonOrThrow<T>(response: Response): Promise<T> {
|
|
||||||
if (!response.ok) {
|
|
||||||
throw new Error(await readFastApiError(response));
|
|
||||||
}
|
|
||||||
return response.json();
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function fetchModelMaxPositionEmbeddings(
|
export async function fetchModelMaxPositionEmbeddings(
|
||||||
modelName: string,
|
modelName: string,
|
||||||
hfToken?: string | null,
|
hfToken?: string | null,
|
||||||
signal?: AbortSignal,
|
signal?: AbortSignal,
|
||||||
): Promise<number | null> {
|
): Promise<number | null> {
|
||||||
const query = hfToken?.trim()
|
const config = await getModelConfig(
|
||||||
? `?hf_token=${encodeURIComponent(hfToken.trim())}`
|
modelName,
|
||||||
: "";
|
signal,
|
||||||
const response = await authFetch(
|
hfToken?.trim() || undefined,
|
||||||
`/api/models/config/${encodeURIComponent(modelName)}${query}`,
|
|
||||||
{ signal },
|
|
||||||
);
|
);
|
||||||
const data = await parseJsonOrThrow<{ max_position_embeddings?: unknown }>(
|
const value = config.max_position_embeddings;
|
||||||
response,
|
|
||||||
);
|
|
||||||
const value = data.max_position_embeddings;
|
|
||||||
return typeof value === "number" && Number.isFinite(value) && value > 0
|
return typeof value === "number" && Number.isFinite(value) && value > 0
|
||||||
? Math.floor(value)
|
? Math.floor(value)
|
||||||
: null;
|
: null;
|
||||||
|
|
|
||||||
|
|
@ -30,11 +30,13 @@ import {
|
||||||
import { perModelConfigsEqual } from "../model-config/apply-per-model-config";
|
import { perModelConfigsEqual } from "../model-config/apply-per-model-config";
|
||||||
import {
|
import {
|
||||||
DEFAULT_PER_MODEL_CONFIG,
|
DEFAULT_PER_MODEL_CONFIG,
|
||||||
|
KV_CACHE_DTYPES,
|
||||||
MAX_SEQ_LENGTH_MAX,
|
MAX_SEQ_LENGTH_MAX,
|
||||||
MAX_SEQ_LENGTH_MIN,
|
MAX_SEQ_LENGTH_MIN,
|
||||||
MAX_SEQ_LENGTH_STEP,
|
MAX_SEQ_LENGTH_STEP,
|
||||||
MTP_SPECULATIVE_TYPES,
|
MTP_SPECULATIVE_TYPES,
|
||||||
type PerModelConfig,
|
type PerModelConfig,
|
||||||
|
SPECULATIVE_TYPES,
|
||||||
deletePerModelConfig,
|
deletePerModelConfig,
|
||||||
isDefaultConfig,
|
isDefaultConfig,
|
||||||
normalizeMaxSeqLength,
|
normalizeMaxSeqLength,
|
||||||
|
|
@ -53,6 +55,16 @@ const CONTROL_SURFACE =
|
||||||
const SELECT_TRIGGER_CLASS = `grid h-8 min-w-0 grid-cols-[minmax(0,1fr)_auto] items-center gap-1 ${CONTROL_SURFACE} pl-3 pr-2 py-0 text-[13px]! font-medium text-nav-fg focus-visible:ring-0 focus-visible:border-transparent [&_[data-slot=select-value]]:min-w-0 [&_[data-slot=select-value]]:truncate [&>svg]:shrink-0`;
|
const SELECT_TRIGGER_CLASS = `grid h-8 min-w-0 grid-cols-[minmax(0,1fr)_auto] items-center gap-1 ${CONTROL_SURFACE} pl-3 pr-2 py-0 text-[13px]! font-medium text-nav-fg focus-visible:ring-0 focus-visible:border-transparent [&_[data-slot=select-value]]:min-w-0 [&_[data-slot=select-value]]:truncate [&>svg]:shrink-0`;
|
||||||
const NUMBER_INPUT_CLASS = `h-8 w-[92px] ${CONTROL_SURFACE} pl-3 pr-2 py-0 text-right text-[13px] font-medium text-nav-fg outline-none focus-visible:ring-0`;
|
const NUMBER_INPUT_CLASS = `h-8 w-[92px] ${CONTROL_SURFACE} pl-3 pr-2 py-0 text-right text-[13px] font-medium text-nav-fg outline-none focus-visible:ring-0`;
|
||||||
|
|
||||||
|
const KV_CACHE_DTYPE_DEFAULT = "f16";
|
||||||
|
const SPECULATIVE_TYPE_LABELS: Record<(typeof SPECULATIVE_TYPES)[number], string> =
|
||||||
|
{
|
||||||
|
auto: "Auto",
|
||||||
|
mtp: "MTP",
|
||||||
|
ngram: "Ngram",
|
||||||
|
"mtp+ngram": "MTP+Ngram",
|
||||||
|
off: "Off",
|
||||||
|
};
|
||||||
|
|
||||||
function hasNonDefaultAdvanced(config: PerModelConfig): boolean {
|
function hasNonDefaultAdvanced(config: PerModelConfig): boolean {
|
||||||
return (
|
return (
|
||||||
config.kvCacheDtype != null ||
|
config.kvCacheDtype != null ||
|
||||||
|
|
@ -182,9 +194,9 @@ function GgufAdvancedSettings({
|
||||||
</InfoHint>
|
</InfoHint>
|
||||||
</div>
|
</div>
|
||||||
<Select
|
<Select
|
||||||
value={config.kvCacheDtype ?? "f16"}
|
value={config.kvCacheDtype ?? KV_CACHE_DTYPE_DEFAULT}
|
||||||
onValueChange={(v) =>
|
onValueChange={(v) =>
|
||||||
update({ kvCacheDtype: v === "f16" ? null : v })
|
update({ kvCacheDtype: v === KV_CACHE_DTYPE_DEFAULT ? null : v })
|
||||||
}
|
}
|
||||||
>
|
>
|
||||||
<SelectTrigger
|
<SelectTrigger
|
||||||
|
|
@ -196,11 +208,14 @@ function GgufAdvancedSettings({
|
||||||
<SelectValue />
|
<SelectValue />
|
||||||
</SelectTrigger>
|
</SelectTrigger>
|
||||||
<SelectContent className="menu-soft-surface ring-0 border-0 rounded-lg">
|
<SelectContent className="menu-soft-surface ring-0 border-0 rounded-lg">
|
||||||
<SelectItem value="f16">f16</SelectItem>
|
<SelectItem value={KV_CACHE_DTYPE_DEFAULT}>
|
||||||
<SelectItem value="bf16">bf16</SelectItem>
|
{KV_CACHE_DTYPE_DEFAULT}
|
||||||
<SelectItem value="q8_0">q8_0</SelectItem>
|
</SelectItem>
|
||||||
<SelectItem value="q5_1">q5_1</SelectItem>
|
{KV_CACHE_DTYPES.map((dtype) => (
|
||||||
<SelectItem value="q4_1">q4_1</SelectItem>
|
<SelectItem key={dtype} value={dtype}>
|
||||||
|
{dtype}
|
||||||
|
</SelectItem>
|
||||||
|
))}
|
||||||
</SelectContent>
|
</SelectContent>
|
||||||
</Select>
|
</Select>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -232,11 +247,11 @@ function GgufAdvancedSettings({
|
||||||
<SelectValue />
|
<SelectValue />
|
||||||
</SelectTrigger>
|
</SelectTrigger>
|
||||||
<SelectContent className="menu-soft-surface ring-0 border-0 rounded-lg">
|
<SelectContent className="menu-soft-surface ring-0 border-0 rounded-lg">
|
||||||
<SelectItem value="auto">Auto</SelectItem>
|
{SPECULATIVE_TYPES.map((type) => (
|
||||||
<SelectItem value="mtp">MTP</SelectItem>
|
<SelectItem key={type} value={type}>
|
||||||
<SelectItem value="ngram">Ngram</SelectItem>
|
{SPECULATIVE_TYPE_LABELS[type]}
|
||||||
<SelectItem value="mtp+ngram">MTP+Ngram</SelectItem>
|
</SelectItem>
|
||||||
<SelectItem value="off">Off</SelectItem>
|
))}
|
||||||
</SelectContent>
|
</SelectContent>
|
||||||
</Select>
|
</Select>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -317,6 +332,12 @@ export function ModelConfigPage({
|
||||||
const isActiveModel = loadedConfig != null;
|
const isActiveModel = loadedConfig != null;
|
||||||
const runtimeMaxSeqLength = useChatRuntimeStore((s) => s.params.maxSeqLength);
|
const runtimeMaxSeqLength = useChatRuntimeStore((s) => s.params.maxSeqLength);
|
||||||
const hfToken = useChatRuntimeStore((s) => s.hfToken);
|
const hfToken = useChatRuntimeStore((s) => s.hfToken);
|
||||||
|
const loadedDefaultChatTemplate = useChatRuntimeStore(
|
||||||
|
(s) => s.defaultChatTemplate,
|
||||||
|
);
|
||||||
|
const loadedMaxContextLength = useChatRuntimeStore(
|
||||||
|
(s) => s.ggufMaxContextLength,
|
||||||
|
);
|
||||||
const [initialMaxSeqLength] = useState(
|
const [initialMaxSeqLength] = useState(
|
||||||
() => normalizeMaxSeqLength(runtimeMaxSeqLength) ?? 4096,
|
() => normalizeMaxSeqLength(runtimeMaxSeqLength) ?? 4096,
|
||||||
);
|
);
|
||||||
|
|
@ -353,6 +374,14 @@ export function ModelConfigPage({
|
||||||
target.id,
|
target.id,
|
||||||
!target.isGguf,
|
!target.isGguf,
|
||||||
);
|
);
|
||||||
|
const hasLoadedDefaultTemplate =
|
||||||
|
isActiveModel && loadedDefaultChatTemplate != null;
|
||||||
|
const resolvedDefaultTemplate = hasLoadedDefaultTemplate
|
||||||
|
? loadedDefaultChatTemplate
|
||||||
|
: templateDefaults.template;
|
||||||
|
const resolvedDefaultLoading = hasLoadedDefaultTemplate
|
||||||
|
? false
|
||||||
|
: templateDefaults.loading;
|
||||||
|
|
||||||
const update = (patch: Partial<PerModelConfig>) =>
|
const update = (patch: Partial<PerModelConfig>) =>
|
||||||
setConfig((current) => ({ ...current, ...patch }));
|
setConfig((current) => ({ ...current, ...patch }));
|
||||||
|
|
@ -454,7 +483,7 @@ export function ModelConfigPage({
|
||||||
customContextLength:
|
customContextLength:
|
||||||
contextBaseline == null && config.customContextLength == null
|
contextBaseline == null && config.customContextLength == null
|
||||||
? null
|
? null
|
||||||
: resolveCustomContextLength(contextValue, nativeContextLength),
|
: resolveCustomContextLength(contextValue, contextBaseline),
|
||||||
}
|
}
|
||||||
: {
|
: {
|
||||||
...config,
|
...config,
|
||||||
|
|
@ -472,20 +501,21 @@ export function ModelConfigPage({
|
||||||
|
|
||||||
const handleRun = () => {
|
const handleRun = () => {
|
||||||
const defaultConfig = isDefaultConfig(runtimeConfig);
|
const defaultConfig = isDefaultConfig(runtimeConfig);
|
||||||
|
let saveFailed = false;
|
||||||
if (remember) {
|
if (remember) {
|
||||||
const saved = savePerModelConfig(
|
saveFailed = !savePerModelConfig(
|
||||||
target.id,
|
target.id,
|
||||||
target.ggufVariant,
|
target.ggufVariant,
|
||||||
runtimeConfig,
|
runtimeConfig,
|
||||||
);
|
);
|
||||||
if (!saved) {
|
|
||||||
toast.error("Couldn't save settings for this model.");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
deletePerModelConfig(target.id, target.ggufVariant);
|
deletePerModelConfig(target.id, target.ggufVariant);
|
||||||
}
|
}
|
||||||
if (persistenceOnly) {
|
if (persistenceOnly) {
|
||||||
|
if (saveFailed) {
|
||||||
|
toast.error("Couldn't save settings for this model.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
const nextRemember = remember && !defaultConfig;
|
const nextRemember = remember && !defaultConfig;
|
||||||
setSavedRemember(nextRemember);
|
setSavedRemember(nextRemember);
|
||||||
setRemember(nextRemember);
|
setRemember(nextRemember);
|
||||||
|
|
@ -498,6 +528,9 @@ export function ModelConfigPage({
|
||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (saveFailed) {
|
||||||
|
toast.error("Couldn't save these settings, loading with them anyway.");
|
||||||
|
}
|
||||||
onRun(runtimeConfig);
|
onRun(runtimeConfig);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -566,6 +599,15 @@ export function ModelConfigPage({
|
||||||
aria-label="Context Length"
|
aria-label="Context Length"
|
||||||
/>
|
/>
|
||||||
) : null}
|
) : null}
|
||||||
|
{isActiveModel &&
|
||||||
|
loadedMaxContextLength != null &&
|
||||||
|
contextValue > loadedMaxContextLength && (
|
||||||
|
<p className="text-[11px] text-amber-500">
|
||||||
|
Exceeds estimated VRAM capacity (
|
||||||
|
{loadedMaxContextLength.toLocaleString()} tokens). The model
|
||||||
|
may use system RAM.
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{showAdvanced && (
|
{showAdvanced && (
|
||||||
|
|
@ -679,8 +721,8 @@ export function ModelConfigPage({
|
||||||
open={templateOpen}
|
open={templateOpen}
|
||||||
onOpenChange={setTemplateOpen}
|
onOpenChange={setTemplateOpen}
|
||||||
value={config.chatTemplateOverride}
|
value={config.chatTemplateOverride}
|
||||||
defaultTemplate={templateDefaults.template}
|
defaultTemplate={resolvedDefaultTemplate}
|
||||||
defaultLoading={templateDefaults.loading}
|
defaultLoading={resolvedDefaultLoading}
|
||||||
readOnly={!target.isGguf}
|
readOnly={!target.isGguf}
|
||||||
onSave={(override) => update({ chatTemplateOverride: override })}
|
onSave={(override) => update({ chatTemplateOverride: override })}
|
||||||
/>
|
/>
|
||||||
|
|
|
||||||
|
|
@ -542,6 +542,7 @@ function ModelSelectorContent({
|
||||||
onSelect(visibleConfigTarget.id, {
|
onSelect(visibleConfigTarget.id, {
|
||||||
...visibleConfigTarget.meta,
|
...visibleConfigTarget.meta,
|
||||||
config,
|
config,
|
||||||
|
forceReload: true,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
loadedConfig={
|
loadedConfig={
|
||||||
|
|
|
||||||
|
|
@ -1477,7 +1477,8 @@ export function HubModelPicker({
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const pickerInventory = useChatPickerInventory({ enabled: true });
|
const pickerInventory = useChatPickerInventory({ enabled: true });
|
||||||
const { cachedGguf, cachedModels, cachedReady } = pickerInventory;
|
const { cachedGguf, cachedModels, cachedReady, refreshInventory } =
|
||||||
|
pickerInventory;
|
||||||
const lmStudioModels = useMemo(
|
const lmStudioModels = useMemo(
|
||||||
() =>
|
() =>
|
||||||
sortLmStudio(
|
sortLmStudio(
|
||||||
|
|
@ -1658,6 +1659,10 @@ export function HubModelPicker({
|
||||||
.catch(() => {});
|
.catch(() => {});
|
||||||
}, [refreshScanFolders]);
|
}, [refreshScanFolders]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
void refreshInventory();
|
||||||
|
}, [refreshInventory]);
|
||||||
|
|
||||||
// Hide downloaded models from the recommended list. Case-insensitive
|
// Hide downloaded models from the recommended list. Case-insensitive
|
||||||
// since the HF cache lowercases repo IDs.
|
// since the HF cache lowercases repo IDs.
|
||||||
const downloadedSet = useMemo(() => {
|
const downloadedSet = useMemo(() => {
|
||||||
|
|
|
||||||
|
|
@ -38,6 +38,7 @@ export interface ModelSelectorChangeMeta {
|
||||||
* Studio). Marks it as a GGUF source for the deferred-load staging flow. */
|
* Studio). Marks it as a GGUF source for the deferred-load staging flow. */
|
||||||
isGguf?: boolean;
|
isGguf?: boolean;
|
||||||
config?: PerModelConfig;
|
config?: PerModelConfig;
|
||||||
|
forceReload?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ModelPickTarget {
|
export interface ModelPickTarget {
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
// SPDX-License-Identifier: AGPL-3.0-only
|
// SPDX-License-Identifier: AGPL-3.0-only
|
||||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||||
|
|
||||||
import { looksLikeLocalPath, useHfTokenStore } from "@/features/hub";
|
import { useHfTokenStore } from "@/features/hub";
|
||||||
import { useEffect, useState } from "react";
|
import { useEffect, useState } from "react";
|
||||||
import { fetchModelMaxPositionEmbeddings } from "../api/model-metadata";
|
import { fetchModelMaxPositionEmbeddings } from "../api/model-metadata";
|
||||||
import { fetchDefaultChatTemplate } from "../api/templates";
|
import { fetchDefaultChatTemplate } from "../api/templates";
|
||||||
|
|
@ -69,7 +69,7 @@ export function useDefaultChatTemplate(
|
||||||
if (controller.signal.aborted) {
|
if (controller.signal.aborted) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (!(template === null && looksLikeLocalPath(modelId))) {
|
if (template !== null) {
|
||||||
cacheTemplate(cacheKey, template);
|
cacheTemplate(cacheKey, template);
|
||||||
}
|
}
|
||||||
setFetched({
|
setFetched({
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ import {
|
||||||
type CachedInventoryRow,
|
type CachedInventoryRow,
|
||||||
type LocalInventoryRow,
|
type LocalInventoryRow,
|
||||||
type LocalSource,
|
type LocalSource,
|
||||||
|
isHiddenModelId,
|
||||||
useHubInventory,
|
useHubInventory,
|
||||||
} from "@/features/hub";
|
} from "@/features/hub";
|
||||||
import { useMemo } from "react";
|
import { useMemo } from "react";
|
||||||
|
|
@ -74,21 +75,35 @@ export function useChatPickerInventory(
|
||||||
const cachedGguf = useMemo(
|
const cachedGguf = useMemo(
|
||||||
() =>
|
() =>
|
||||||
inventory.cachedRows
|
inventory.cachedRows
|
||||||
.filter((row) => row.modelFormat === "gguf" && isCompleteCachedRow(row))
|
.filter(
|
||||||
|
(row) =>
|
||||||
|
row.modelFormat === "gguf" &&
|
||||||
|
isCompleteCachedRow(row) &&
|
||||||
|
!isHiddenModelId(row.repoId),
|
||||||
|
)
|
||||||
.map(toCachedGgufRepo),
|
.map(toCachedGgufRepo),
|
||||||
[inventory.cachedRows],
|
[inventory.cachedRows],
|
||||||
);
|
);
|
||||||
const cachedModels = useMemo(
|
const cachedModels = useMemo(
|
||||||
() =>
|
() =>
|
||||||
inventory.cachedRows
|
inventory.cachedRows
|
||||||
.filter((row) => row.modelFormat !== "gguf" && isCompleteCachedRow(row))
|
.filter(
|
||||||
|
(row) =>
|
||||||
|
row.modelFormat !== "gguf" &&
|
||||||
|
isCompleteCachedRow(row) &&
|
||||||
|
!isHiddenModelId(row.repoId),
|
||||||
|
)
|
||||||
.map(toCachedModelRepo),
|
.map(toCachedModelRepo),
|
||||||
[inventory.cachedRows],
|
[inventory.cachedRows],
|
||||||
);
|
);
|
||||||
const localModels = useMemo(
|
const localModels = useMemo(
|
||||||
() =>
|
() =>
|
||||||
inventory.localRows
|
inventory.localRows
|
||||||
.filter((row) => PICKER_LOCAL_SOURCES.has(row.source))
|
.filter(
|
||||||
|
(row) =>
|
||||||
|
PICKER_LOCAL_SOURCES.has(row.source) &&
|
||||||
|
!isHiddenModelId(row.modelId, row.repoId, row.path),
|
||||||
|
)
|
||||||
.map(toLocalModelInfo),
|
.map(toLocalModelInfo),
|
||||||
[inventory.localRows],
|
[inventory.localRows],
|
||||||
);
|
);
|
||||||
|
|
|
||||||
|
|
@ -18,10 +18,11 @@ function cleanTemplate(value: string | null | undefined): string | null {
|
||||||
|
|
||||||
export function applyPerModelConfigToRuntime(config: PerModelConfig): void {
|
export function applyPerModelConfigToRuntime(config: PerModelConfig): void {
|
||||||
const maxSeqLength = normalizeMaxSeqLength(config.maxSeqLength);
|
const maxSeqLength = normalizeMaxSeqLength(config.maxSeqLength);
|
||||||
useChatRuntimeStore.setState((state) => ({
|
const store = useChatRuntimeStore.getState();
|
||||||
...(maxSeqLength == null
|
if (maxSeqLength != null && maxSeqLength !== store.params.maxSeqLength) {
|
||||||
? {}
|
store.setParams({ ...store.params, maxSeqLength });
|
||||||
: { params: { ...state.params, maxSeqLength } }),
|
}
|
||||||
|
useChatRuntimeStore.setState({
|
||||||
customContextLength: config.customContextLength ?? null,
|
customContextLength: config.customContextLength ?? null,
|
||||||
kvCacheDtype: config.kvCacheDtype ?? null,
|
kvCacheDtype: config.kvCacheDtype ?? null,
|
||||||
speculativeType:
|
speculativeType:
|
||||||
|
|
@ -30,7 +31,7 @@ export function applyPerModelConfigToRuntime(config: PerModelConfig): void {
|
||||||
specDraftNMax: config.specDraftNMax ?? null,
|
specDraftNMax: config.specDraftNMax ?? null,
|
||||||
tensorParallel: config.tensorParallel ?? false,
|
tensorParallel: config.tensorParallel ?? false,
|
||||||
chatTemplateOverride: cleanTemplate(config.chatTemplateOverride),
|
chatTemplateOverride: cleanTemplate(config.chatTemplateOverride),
|
||||||
}));
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
export function applyModelLoadConfigToRuntime(
|
export function applyModelLoadConfigToRuntime(
|
||||||
|
|
|
||||||
|
|
@ -53,7 +53,7 @@ const LEGACY_STORAGE_KEY = "unsloth_load_settings";
|
||||||
const LEGACY_MIGRATION_FLAG = "unsloth_model_configs_migrated";
|
const LEGACY_MIGRATION_FLAG = "unsloth_model_configs_migrated";
|
||||||
const STORAGE_SCHEMA_VERSION = 1;
|
const STORAGE_SCHEMA_VERSION = 1;
|
||||||
const MAX_ENTRIES = 500;
|
const MAX_ENTRIES = 500;
|
||||||
export const MAX_PER_MODEL_CONFIG_STORAGE_BYTES = 1024 * 1024;
|
const MAX_PER_MODEL_CONFIG_STORAGE_BYTES = 1024 * 1024;
|
||||||
export const MAX_CHAT_TEMPLATE_BYTES = 65_536;
|
export const MAX_CHAT_TEMPLATE_BYTES = 65_536;
|
||||||
|
|
||||||
type StoredPerModelConfig = PerModelConfig & {
|
type StoredPerModelConfig = PerModelConfig & {
|
||||||
|
|
@ -149,17 +149,6 @@ function deleteOldestEvictableEntry(
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
function isMostRecentEntry(map: StoredMap, key: string): boolean {
|
|
||||||
const keys = Object.keys(map);
|
|
||||||
return keys.length > 0 && keys[keys.length - 1] === key;
|
|
||||||
}
|
|
||||||
|
|
||||||
function touchEntry(map: StoredMap, key: string): void {
|
|
||||||
const value = map[key];
|
|
||||||
delete map[key];
|
|
||||||
map[key] = value;
|
|
||||||
}
|
|
||||||
|
|
||||||
function enforceStorageBudget(map: StoredMap, protectedKey?: string): boolean {
|
function enforceStorageBudget(map: StoredMap, protectedKey?: string): boolean {
|
||||||
let entryCount = Object.keys(map).length;
|
let entryCount = Object.keys(map).length;
|
||||||
while (entryCount > MAX_ENTRIES) {
|
while (entryCount > MAX_ENTRIES) {
|
||||||
|
|
@ -255,9 +244,12 @@ function migrateLegacyLoadSettingsOnce(): void {
|
||||||
if (localStorage.getItem(LEGACY_MIGRATION_FLAG)) {
|
if (localStorage.getItem(LEGACY_MIGRATION_FLAG)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const legacy = JSON.parse(
|
let legacy: unknown = null;
|
||||||
localStorage.getItem(LEGACY_STORAGE_KEY) ?? "null",
|
try {
|
||||||
);
|
legacy = JSON.parse(localStorage.getItem(LEGACY_STORAGE_KEY) ?? "null");
|
||||||
|
} catch {
|
||||||
|
legacy = null;
|
||||||
|
}
|
||||||
if (!legacy || typeof legacy !== "object" || Array.isArray(legacy)) {
|
if (!legacy || typeof legacy !== "object" || Array.isArray(legacy)) {
|
||||||
localStorage.setItem(LEGACY_MIGRATION_FLAG, "1");
|
localStorage.setItem(LEGACY_MIGRATION_FLAG, "1");
|
||||||
return;
|
return;
|
||||||
|
|
@ -270,8 +262,6 @@ function migrateLegacyLoadSettingsOnce(): void {
|
||||||
enforceStorageBudget(map);
|
enforceStorageBudget(map);
|
||||||
if (writeMap(map)) {
|
if (writeMap(map)) {
|
||||||
localStorage.setItem(LEGACY_MIGRATION_FLAG, "1");
|
localStorage.setItem(LEGACY_MIGRATION_FLAG, "1");
|
||||||
} else {
|
|
||||||
legacyMigrationChecked = false;
|
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.warn("Failed to migrate legacy load settings:", err);
|
console.warn("Failed to migrate legacy load settings:", err);
|
||||||
|
|
@ -472,36 +462,13 @@ function deleteConfigEntriesForModelVariant(
|
||||||
return changed;
|
return changed;
|
||||||
}
|
}
|
||||||
|
|
||||||
function loadPerModelConfigInternal(
|
function loadPerModelConfig(
|
||||||
modelId: string,
|
modelId: string,
|
||||||
ggufVariant: string | null | undefined,
|
ggufVariant?: string | null,
|
||||||
touch: boolean,
|
|
||||||
): PerModelConfig | null {
|
): PerModelConfig | null {
|
||||||
const map = readMap();
|
const map = readMap();
|
||||||
const key = findConfigKeyForModelVariant(map, modelId, ggufVariant);
|
const key = findConfigKeyForModelVariant(map, modelId, ggufVariant);
|
||||||
if (!key) {
|
return key ? normalize(map[key]) : null;
|
||||||
return null;
|
|
||||||
}
|
|
||||||
const config = normalize(map[key]);
|
|
||||||
if (touch && !isMostRecentEntry(map, key)) {
|
|
||||||
touchEntry(map, key);
|
|
||||||
writeMap(map);
|
|
||||||
}
|
|
||||||
return config;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function loadPerModelConfig(
|
|
||||||
modelId: string,
|
|
||||||
ggufVariant?: string | null,
|
|
||||||
): PerModelConfig | null {
|
|
||||||
return loadPerModelConfigInternal(modelId, ggufVariant, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function hasPerModelConfig(
|
|
||||||
modelId: string,
|
|
||||||
ggufVariant?: string | null,
|
|
||||||
): boolean {
|
|
||||||
return loadPerModelConfigInternal(modelId, ggufVariant, false) != null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export function isDefaultConfig(config: PerModelConfig): boolean {
|
export function isDefaultConfig(config: PerModelConfig): boolean {
|
||||||
|
|
|
||||||
|
|
@ -24,8 +24,8 @@ export { HfDatasetSubsetSplitSelectors } from "./components/hf-dataset-subset-sp
|
||||||
export { useDatasetPreviewDialogStore } from "./stores/dataset-preview-dialog-store";
|
export { useDatasetPreviewDialogStore } from "./stores/dataset-preview-dialog-store";
|
||||||
export { listLocalDatasets, uploadTrainingDataset } from "./api/datasets-api";
|
export { listLocalDatasets, uploadTrainingDataset } from "./api/datasets-api";
|
||||||
export type { LocalDatasetInfo } from "./types/datasets";
|
export type { LocalDatasetInfo } from "./types/datasets";
|
||||||
export { listLocalModels } from "./api/models-api";
|
export { getModelConfig, listLocalModels } from "./api/models-api";
|
||||||
export type { LocalModelInfo } from "./api/models-api";
|
export type { LocalModelInfo, ModelConfigResponse } from "./api/models-api";
|
||||||
export type {
|
export type {
|
||||||
TrainingPhase,
|
TrainingPhase,
|
||||||
TrainingViewData,
|
TrainingViewData,
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue