From 562e54fc6e78409bf0ef30c6eb30c1f59b496635 Mon Sep 17 00:00:00 2001 From: Roland Tannous <115670425+rolandtannous@users.noreply.github.com> Date: Fri, 27 Mar 2026 17:59:27 +0400 Subject: [PATCH] Fix HF cache default and show LM Studio models in chat/inference (#4653) * fix: default HF cache to standard platform path instead of legacy Unsloth cache * feat: show LM Studio and local models in chat Fine-tuned tab * feat: show LM Studio models in Hub models tab * fix: fetch local models after auth refresh completes * Revert "fix: fetch local models after auth refresh completes" This reverts commit cfd61f0ac76a6f578f14bcd0c668bb011b0ff330. * fix: increase llama-server health check timeout to 600s for large models * feat: expandable GGUF variant picker for LM Studio local models * fix: show GGUF variant label for locally loaded LM Studio models * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix: show publisher name in LM Studio model labels * fix: set model_id for loose GGUF files in LM Studio publisher dirs * fix: show publisher prefix in Fine-tuned tab LM Studio models * fix: only use model_id for lmstudio source models * fix: only show LM Studio models in Hub tab on Mac/chat-only mode * fix: respect XDG_CACHE_HOME, handle Windows paths in isLocalPath, refresh LM Studio on remount - _setup_cache_env now reads XDG_CACHE_HOME (falls back to ~/.cache) instead of hard-coding ~/.cache/huggingface. This follows the standard HF cache resolution chain and respects distro/container overrides. - isLocalPath in GgufVariantExpander uses a regex that covers Windows drive letters (C:\, D:/), UNC paths (\\server\share), relative paths (./, ../), and tilde (~/) -- not just startsWith("/"). - HubModelPicker.useEffect now calls listLocalModels() before the alreadyCached early-return gate so LM Studio models are always refreshed on remount. Also seeds useState from _lmStudioCache for instant display on re-open. * fix: add comment explaining isLocalPath regex for Windows/cross-platform paths * fix: prioritize unsloth publisher in LM Studio model list * fix: scope unsloth-first sort to LM Studio models on all platforms * fix: add missing _lmStudioCache module-level declaration * fix: prioritize unsloth publisher before timestamp sort in LM Studio group --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Daniel Han --- studio/backend/core/inference/llama_cpp.py | 15 +- studio/backend/routes/models.py | 30 +++- studio/backend/utils/models/model_config.py | 72 ++++++++- studio/backend/utils/paths/storage_roots.py | 23 +-- .../assistant-ui/model-selector/pickers.tsx | 153 ++++++++++++++---- .../assistant-ui/model-selector/types.ts | 4 +- .../src/features/chat/api/chat-api.ts | 21 +++ .../frontend/src/features/chat/chat-page.tsx | 39 +++-- 8 files changed, 297 insertions(+), 60 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 7909af8a23..05e038dbb7 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -1292,7 +1292,18 @@ class LlamaCppBackend: self._gguf_path = gguf_path self._hf_repo = hf_repo - self._hf_variant = hf_variant + # For local GGUF files, extract variant from filename if not provided + if hf_variant: + self._hf_variant = hf_variant + elif gguf_path: + try: + from utils.models.model_config import _extract_quant_label + + self._hf_variant = _extract_quant_label(gguf_path) + except Exception: + self._hf_variant = None + else: + self._hf_variant = None self._is_vision = is_vision self._model_identifier = model_identifier @@ -1304,7 +1315,7 @@ class LlamaCppBackend: ) # Wait for llama-server to become healthy - if not self._wait_for_health(timeout = 120.0): + if not self._wait_for_health(timeout = 600.0): self._kill_process() raise RuntimeError( "llama-server failed to start. " diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py index f76034c95b..348ffbf6ea 100644 --- a/studio/backend/routes/models.py +++ b/studio/backend/routes/models.py @@ -271,6 +271,7 @@ def _scan_lmstudio_dir(lm_dir: Path) -> List[LocalModelInfo]: found.append( LocalModelInfo( id = str(model_dir), + model_id = f"{child.name}/{model_dir.stem}", display_name = model_dir.stem, path = str(model_dir), source = "lmstudio", @@ -725,13 +726,40 @@ async def get_gguf_variants( current_subject: str = Depends(get_current_subject), ): """ - List available GGUF quantization variants for a HuggingFace repo. + List available GGUF quantization variants for a HuggingFace repo + or a local directory (e.g. LM Studio model folder). Returns all available quantization variants (Q4_K_M, Q8_0, BF16, etc.) with file sizes, whether the model supports vision, and the recommended default variant. """ try: + from utils.models.model_config import is_local_path, list_local_gguf_variants + + # Local directory path (e.g. LM Studio models) — scan filesystem + if is_local_path(repo_id): + variants, has_vision = list_local_gguf_variants(repo_id) + + filenames = [v.filename for v in variants] + best = _pick_best_gguf(filenames) + default_variant = _extract_quant_label(best) if best else None + + return GgufVariantsResponse( + repo_id = repo_id, + variants = [ + GgufVariantDetail( + filename = v.filename, + quant = v.quant, + size_bytes = v.size_bytes, + downloaded = True, # all local variants are downloaded + ) + for v in variants + ], + has_vision = has_vision, + default_variant = default_variant, + ) + + # Remote HuggingFace repo — query HF API variants, has_vision = list_gguf_variants(repo_id, hf_token = hf_token) # Determine default variant diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 13f1b5febf..5de3fd2cf9 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -973,6 +973,73 @@ def list_gguf_variants( return variants, has_vision +def list_local_gguf_variants( + directory: str, +) -> tuple[list[GgufVariantInfo], bool]: + """List GGUF quantization variants in a local directory. + + Mirrors :func:`list_gguf_variants` but reads from the filesystem + instead of the HuggingFace API. Aggregates shard sizes by quant + label so that split GGUFs appear as a single variant. + + Returns: + (variants, has_vision): list of non-mmproj GGUF variants + vision flag. + """ + p = Path(directory) + if not p.is_dir(): + return [], False + + quant_totals: dict[str, int] = {} + quant_first_file: dict[str, str] = {} + has_vision = False + + for f in sorted(p.glob("*.gguf")): + if _is_mmproj(f.name): + has_vision = True + continue + try: + size = f.stat().st_size + except OSError: + size = 0 + quant = _extract_quant_label(f.name) + quant_totals[quant] = quant_totals.get(quant, 0) + size + if quant not in quant_first_file: + quant_first_file[quant] = f.name + + variants = [ + GgufVariantInfo( + filename = quant_first_file[q], + quant = q, + size_bytes = s, + ) + for q, s in quant_totals.items() + ] + variants.sort(key = lambda v: -v.size_bytes) + return variants, has_vision + + +def _find_local_gguf_by_variant(directory: str, variant: str) -> Optional[str]: + """Find the GGUF file in *directory* matching a quantization *variant*. + + For sharded GGUFs (multiple files with the same quant label), returns + the first shard (sorted by name) which is what ``llama-server -m`` expects. + + Returns the resolved absolute path, or ``None`` if no match. + """ + p = Path(directory) + if not p.is_dir(): + return None + + matches = sorted( + f + for f in p.glob("*.gguf") + if not _is_mmproj(f.name) and _extract_quant_label(f.name) == variant + ) + if matches: + return str(matches[0].resolve()) + return None + + def detect_gguf_model_remote( repo_id: str, hf_token: Optional[str] = None, @@ -1530,7 +1597,10 @@ class ModelConfig: # Auto-detect GGUF models (check before LoRA/vision detection) if is_local: - gguf_file = detect_gguf_model(path) + if gguf_variant: + gguf_file = _find_local_gguf_by_variant(path, gguf_variant) + else: + gguf_file = detect_gguf_model(path) if gguf_file: display_name = Path(gguf_file).stem logger.info(f"Detected local GGUF model: {gguf_file}") diff --git a/studio/backend/utils/paths/storage_roots.py b/studio/backend/utils/paths/storage_roots.py index 9bcf3758ad..4841c5d0a3 100644 --- a/studio/backend/utils/paths/storage_roots.py +++ b/studio/backend/utils/paths/storage_roots.py @@ -133,27 +133,28 @@ def lmstudio_model_dirs() -> list[Path]: def _setup_cache_env() -> None: """Set cache environment variables for HuggingFace, uv, and vLLM. - HuggingFace cache variables are only set when the legacy Unsloth HF - cache already exists, preserving existing model locations. New - installations leave HF at its own defaults. + Respects the standard HF cache resolution chain: explicit ``HF_HOME`` + / ``HF_HUB_CACHE`` env vars take priority, then ``XDG_CACHE_HOME``, + then the platform default (``~/.cache/huggingface``). The legacy + Unsloth cache is still *scanned* for models but is never set as the + active download target. Only sets variables that are not already set by the user, so explicit overrides (e.g. HF_HOME=/data/hf) are respected. Works on Linux, macOS, and Windows. """ root = cache_root() - hf_dir = root / "huggingface" + xdg_cache = Path( + os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache") + ).expanduser() + hf_default = xdg_cache / "huggingface" defaults: dict[str, str] = { + "HF_HOME": str(hf_default), + "HF_HUB_CACHE": str(hf_default / "hub"), + "HF_XET_CACHE": str(hf_default / "xet"), "UV_CACHE_DIR": str(root / "uv"), "VLLM_CACHE_ROOT": str(root / "vllm"), } - # Preserve legacy HF cache for existing installations - legacy_hub = hf_dir / "hub" - if legacy_hub.is_dir() and any(legacy_hub.iterdir()): - defaults["HF_HOME"] = str(hf_dir) - defaults["HF_HUB_CACHE"] = str(legacy_hub) - defaults["HF_XET_CACHE"] = str(hf_dir / "xet") - for key, value in defaults.items(): if key not in os.environ: os.environ[key] = value diff --git a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx index 8d4b6ae0d6..3ac3416df4 100644 --- a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx +++ b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx @@ -18,8 +18,8 @@ import { TooltipContent, TooltipTrigger, } from "@/components/ui/tooltip"; -import { deleteCachedModel, listCachedGguf, listCachedModels, listGgufVariants } from "@/features/chat/api/chat-api"; -import type { CachedGgufRepo, CachedModelRepo } from "@/features/chat/api/chat-api"; +import { deleteCachedModel, listCachedGguf, listCachedModels, listGgufVariants, listLocalModels } from "@/features/chat/api/chat-api"; +import type { CachedGgufRepo, CachedModelRepo, LocalModelInfo } from "@/features/chat/api/chat-api"; import type { GgufVariantDetail } from "@/features/chat/types/api"; import { usePlatformStore } from "@/config/env"; import { @@ -203,17 +203,20 @@ function GgufVariantExpander({ }; }, [repoId]); + // Covers Unix absolute (/), Windows drive (C:\, D:/), UNC (\\server), relative (./, ../), tilde (~/) + const isLocalPath = /^(\/|\.{1,2}[\\\/]|~[\\\/]|[A-Za-z]:[\\\/]|\\\\)/.test(repoId); + const handleVariantClick = useCallback( (quant: string, downloaded?: boolean, sizeBytes?: number) => { onSelect(repoId, { - source: "hub", + source: isLocalPath ? "local" : "hub", isLora: false, ggufVariant: quant, - isDownloaded: downloaded, + isDownloaded: isLocalPath ? true : downloaded, expectedBytes: sizeBytes, }); }, - [repoId, onSelect], + [repoId, isLocalPath, onSelect], ); // GGUF fit classification matching llama-server's _select_gpus logic: @@ -380,6 +383,17 @@ function extractParamLabel(id: string): string | undefined { // Module-level caches so re-mounting the popover shows results instantly let _cachedGgufCache: CachedGgufRepo[] = []; let _cachedModelsCache: CachedModelRepo[] = []; +let _lmStudioCache: LocalModelInfo[] = []; + +/** Sort LM Studio models with unsloth publisher first. */ +function sortLmStudio(models: LocalModelInfo[]): LocalModelInfo[] { + return [...models].sort((a, b) => { + const aUnsloth = (a.model_id ?? "").startsWith("unsloth/") ? 0 : 1; + const bUnsloth = (b.model_id ?? "").startsWith("unsloth/") ? 0 : 1; + if (aUnsloth !== bUnsloth) return aUnsloth - bUnsloth; + return (a.model_id ?? a.display_name).localeCompare(b.model_id ?? b.display_name); + }); +} // ── Hub Model Picker ────────────────────────────────────────── @@ -413,12 +427,28 @@ export function HubModelPicker({ const alreadyCached = _cachedGgufCache.length > 0 || _cachedModelsCache.length > 0; const [cachedReady, setCachedReady] = useState(alreadyCached); + // LM Studio local models -- module-level cache so re-mounting the + // popover does not flash an empty section (same pattern as GGUF/models). + const [lmStudioModels, setLmStudioModels] = useState(_lmStudioCache); + const refreshCachedLists = useCallback(() => { listCachedGguf().then((v) => { _cachedGgufCache = v; setCachedGguf(v); }).catch(() => {}); listCachedModels().then((v) => { _cachedModelsCache = v; setCachedModels(v); }).catch(() => {}); + listLocalModels().then((res) => { + const next = sortLmStudio(res.models.filter((m) => m.source === "lmstudio")); + _lmStudioCache = next; + setLmStudioModels(next); + }).catch(() => {}); }, []); useEffect(() => { + // Always refresh LM Studio models (not gated by alreadyCached) + listLocalModels().then((res) => { + const next = sortLmStudio(res.models.filter((m) => m.source === "lmstudio")); + _lmStudioCache = next; + setLmStudioModels(next); + }).catch(() => {}); + if (alreadyCached) return; let done = 0; const check = () => { if (++done >= 2) setCachedReady(true); }; @@ -686,6 +716,40 @@ export function HubModelPicker({ ) : null} + {!showHfSection && chatOnly && lmStudioModels.length > 0 ? ( + <> + LM Studio + {lmStudioModels.map((m) => { + const isGguf = isGgufRepo(m.id) || isGgufRepo(m.display_name); + return ( +
+ { + if (isGguf) { + setExpandedGguf((prev) => (prev === m.id ? null : m.id)); + } else { + onSelect(m.id, { source: "local", isLora: false, isDownloaded: true }); + } + }} + vramStatus={null} + /> + {expandedGguf === m.id && ( + + )} +
+ ); + })} + + ) : null} + {!showHfSection && cachedReady ? ( <> {"\uD83E\uDDA5"} Recommended @@ -837,6 +901,8 @@ export function LoraModelPicker({ onSelect: (id: string, meta: ModelSelectorChangeMeta) => void; }) { const [query, setQuery] = useState(""); + const [expandedGguf, setExpandedGguf] = useState(null); + const gpu = useGpuInfo(); const normalized = useMemo( () => @@ -846,11 +912,17 @@ export function LoraModelPicker({ baseModel: model.baseModel || model.description || "Unknown base model", })) .sort((a, b) => { + const baseCmp = a.baseModel.localeCompare(b.baseModel); + if (baseCmp !== 0) return baseCmp; + // Prioritize unsloth publisher within LM Studio group + if (a.baseModel === "LM Studio" && b.baseModel === "LM Studio") { + const aUnsloth = a.name.startsWith("unsloth/") ? 0 : 1; + const bUnsloth = b.name.startsWith("unsloth/") ? 0 : 1; + if (aUnsloth !== bUnsloth) return aUnsloth - bUnsloth; + } const aTime = a.updatedAt ?? -1; const bTime = b.updatedAt ?? -1; if (aTime !== bTime) return bTime - aTime; - const baseCmp = a.baseModel.localeCompare(b.baseModel); - if (baseCmp !== 0) return baseCmp; return a.name.localeCompare(b.name); }), [loraModels], @@ -905,34 +977,53 @@ export function LoraModelPicker({ {index > 0 ?
: null} {baseModel} {adapters.map((adapter) => { + const isLocal = adapter.source === "local"; const isExported = adapter.source === "exported"; const isMerged = adapter.exportType === "merged"; const isGguf = adapter.exportType === "gguf"; - const tag = isGguf - ? "GGUF" - : isExported - ? isMerged ? "Merged" : "LoRA" - : "LoRA"; - const meta = isExported ? `${tag} · Exported` : tag; + const isLocalGgufDir = isLocal && (isGgufRepo(adapter.id) || isGgufRepo(adapter.name)); + const tag = isLocal + ? isLocalGgufDir ? "GGUF" : "Local" + : isGguf + ? "GGUF" + : isExported + ? isMerged ? "Merged" : "LoRA" + : "LoRA"; + const meta = isLocal ? (isLocalGgufDir ? "GGUF" : "Local") : isExported ? `${tag} · Exported` : tag; return ( - onSelect(adapter.id, { - source: isExported ? "exported" : "lora", - isLora: !isMerged && !isGguf, - })} - tooltipText={ - <> - {adapter.name} - - {adapter.id} - - - } - /> +
+ { + if (isLocalGgufDir) { + setExpandedGguf((prev) => (prev === adapter.id ? null : adapter.id)); + } else { + onSelect(adapter.id, { + source: isLocal ? "local" : isExported ? "exported" : "lora", + isLora: !isLocal && !isMerged && !isGguf, + }); + } + }} + tooltipText={ + <> + {adapter.name} + + {adapter.id} + + + } + /> + {expandedGguf === adapter.id && ( + + )} +
); })}
diff --git a/studio/frontend/src/components/assistant-ui/model-selector/types.ts b/studio/frontend/src/components/assistant-ui/model-selector/types.ts index 9da7fd975f..f70cfc3b01 100644 --- a/studio/frontend/src/components/assistant-ui/model-selector/types.ts +++ b/studio/frontend/src/components/assistant-ui/model-selector/types.ts @@ -13,12 +13,12 @@ export interface ModelOption { export interface LoraModelOption extends ModelOption { baseModel?: string; updatedAt?: number; - source?: "training" | "exported"; + source?: "training" | "exported" | "local"; exportType?: "lora" | "merged" | "gguf"; } export interface ModelSelectorChangeMeta { - source: "hub" | "lora" | "exported"; + source: "hub" | "lora" | "exported" | "local"; isLora: boolean; ggufVariant?: string; isDownloaded?: boolean; diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts index 57cbcccf66..bb603b90c4 100644 --- a/studio/frontend/src/features/chat/api/chat-api.ts +++ b/studio/frontend/src/features/chat/api/chat-api.ts @@ -125,6 +125,27 @@ export async function getDownloadProgress( return parseJsonOrThrow(response); } +export interface LocalModelInfo { + id: string; + display_name: string; + path: string; + source: "models_dir" | "hf_cache" | "lmstudio"; + model_id?: string | null; + updated_at?: number | null; +} + +interface LocalModelListResponse { + models_dir: string; + hf_cache_dir?: string | null; + lmstudio_dirs: string[]; + models: LocalModelInfo[]; +} + +export async function listLocalModels(): Promise { + const response = await authFetch("/api/models/local"); + return parseJsonOrThrow(response); +} + export async function listCachedGguf(): Promise { const response = await authFetch("/api/models/cached-gguf"); const data = await parseJsonOrThrow<{ cached: CachedGgufRepo[] }>(response); diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx index c04cfbc89c..07b52ebc30 100644 --- a/studio/frontend/src/features/chat/chat-page.tsx +++ b/studio/frontend/src/features/chat/chat-page.tsx @@ -37,6 +37,7 @@ import { } from "react"; import { toast } from "sonner"; import { GuidedTour, useGuidedTourController } from "@/features/tour"; +import { listLocalModels } from "./api/chat-api"; import { ChatSettingsPanel } from "./chat-settings-sheet"; import { ContextUsageBar } from "./components/context-usage-bar"; import { ModelLoadInlineStatus } from "./components/model-load-status"; @@ -578,22 +579,36 @@ export function ChatPage(): ReactElement { [modelsFromStore], ); - const loraModels = useMemo( - () => - lorasFromStore.map((lora) => ({ - id: lora.id, - name: lora.name, - baseModel: lora.baseModel, - updatedAt: lora.updatedAt, - source: lora.source, - exportType: lora.exportType, - })), - [lorasFromStore], - ); + const [localModels, setLocalModels] = useState([]); + + const loraModels = useMemo(() => { + const fromLoras = lorasFromStore.map((lora) => ({ + id: lora.id, + name: lora.name, + baseModel: lora.baseModel, + updatedAt: lora.updatedAt, + source: lora.source, + exportType: lora.exportType, + })); + return [...fromLoras, ...localModels]; + }, [lorasFromStore, localModels]); useEffect(() => { if (getTrainingCompareHandoff()) return; void refresh(); + void listLocalModels().then((res) => { + setLocalModels( + res.models + .filter((m) => m.source === "lmstudio" || m.source === "models_dir") + .map((m) => ({ + id: m.id, + name: m.source === "lmstudio" && m.model_id ? m.model_id : m.display_name, + baseModel: m.source === "lmstudio" ? "LM Studio" : "Local models", + updatedAt: m.updated_at ?? undefined, + source: "local" as const, + })), + ); + }).catch(() => {}); }, [refresh]); useEffect(() => {