diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index d1c943ee16..fafefbb7a7 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -48,6 +48,7 @@ class LlamaCppBackend: self._lock = threading.Lock() self._stdout_lines: list[str] = [] self._stdout_thread: Optional[threading.Thread] = None + self._cancel_event = threading.Event() self._kill_orphaned_servers() atexit.register(self._cleanup) @@ -399,6 +400,7 @@ class LlamaCppBackend: Returns True if server started and health check passed. """ + self._cancel_event.clear() with self._lock: self._kill_process() @@ -539,6 +541,8 @@ class LlamaCppBackend: ) ) try: + if self._cancel_event.is_set(): + raise RuntimeError("Cancelled") local_path = hf_hub_download( repo_id = hf_repo, filename = gguf_filename, @@ -547,12 +551,20 @@ class LlamaCppBackend: # Download remaining shards for split GGUFs — llama-server # auto-discovers them when they are in the same directory. for shard in gguf_extra_shards: + if self._cancel_event.is_set(): + raise RuntimeError("Cancelled") logger.info(f"Downloading GGUF shard: {shard}") hf_hub_download( repo_id = hf_repo, filename = shard, token = hf_token, ) + except RuntimeError as e: + if "Cancelled" in str(e): + raise + raise RuntimeError( + f"Failed to download GGUF file '{gguf_filename}' from {hf_repo}: {e}" + ) except Exception as e: raise RuntimeError( f"Failed to download GGUF file '{gguf_filename}' from {hf_repo}: {e}" @@ -680,7 +692,8 @@ class LlamaCppBackend: return True def unload_model(self) -> bool: - """Terminate the llama-server subprocess and clean up state.""" + """Terminate the llama-server subprocess and cancel any in-flight download.""" + self._cancel_event.set() with self._lock: self._kill_process() logger.info(f"Unloaded GGUF model: {self._model_identifier}") diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py index 99b42b491e..5ffd3d955b 100644 --- a/studio/backend/routes/models.py +++ b/studio/backend/routes/models.py @@ -577,6 +577,49 @@ async def get_gguf_variants( ) +@router.get("/cached-gguf") +async def list_cached_gguf( + current_subject: str = Depends(get_current_subject), +): + """List GGUF repos that have already been downloaded to the HF cache.""" + try: + from huggingface_hub import constants as hf_constants + + cache_dir = Path(hf_constants.HF_HUB_CACHE) + cached = [] + if cache_dir.is_dir(): + for entry in sorted(cache_dir.iterdir()): + if not entry.name.startswith("models--"): + continue + # models--unsloth--Qwen3-8B-GGUF -> unsloth/Qwen3-8B-GGUF + parts = entry.name.split("--", 1) + if len(parts) < 2: + continue + repo_id = parts[1].replace("--", "/") + if not repo_id.lower().endswith("-gguf"): + continue + # Check if there are actual .gguf files in snapshots + snapshots = entry / "snapshots" + if not snapshots.is_dir(): + continue + total_size = 0 + has_gguf = False + for snap in snapshots.iterdir(): + for f in snap.rglob("*.gguf"): + has_gguf = True + total_size += f.stat().st_size + if has_gguf: + cached.append({ + "repo_id": repo_id, + "size_bytes": total_size, + "cache_path": str(entry), + }) + return {"cached": cached} + except Exception as e: + logger.error(f"Error listing cached GGUF repos: {e}", exc_info = True) + return {"cached": []} + + @router.get("/checkpoints", response_model = CheckpointListResponse) async def list_checkpoints( outputs_dir: str = Query( diff --git a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx index f8a8ec4abb..54ccd18d3b 100644 --- a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx +++ b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx @@ -8,7 +8,8 @@ import { TooltipContent, TooltipTrigger, } from "@/components/ui/tooltip"; -import { listGgufVariants } from "@/features/chat/api/chat-api"; +import { listCachedGguf, listGgufVariants } from "@/features/chat/api/chat-api"; +import type { CachedGgufRepo } from "@/features/chat/api/chat-api"; import type { GgufVariantDetail } from "@/features/chat/types/api"; import { useDebouncedValue, @@ -348,6 +349,12 @@ export function HubModelPicker({ // Track which GGUF repo is expanded for variant selection const [expandedGguf, setExpandedGguf] = useState(null); + // Cached (already downloaded) GGUF repos + const [cachedGguf, setCachedGguf] = useState([]); + useEffect(() => { + listCachedGguf().then(setCachedGguf).catch(() => {}); + }, []); + const recommendedIds = useMemo( () => dedupe([...models.map((model) => model.id), value ?? ""]), [models, value], @@ -450,6 +457,26 @@ export function HubModelPicker({
+ {!showHfSection && cachedGguf.length > 0 ? ( + <> + Downloaded + {cachedGguf.map((c) => ( +
+ handleModelClick(c.repo_id)} + vramStatus={null} + /> + {expandedGguf === c.repo_id && ( + + )} +
+ ))} + + ) : null} + {!showHfSection ? ( <> Recommended diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts index 90f6198629..a526366782 100644 --- a/studio/frontend/src/features/chat/api/chat-api.ts +++ b/studio/frontend/src/features/chat/api/chat-api.ts @@ -97,6 +97,18 @@ export async function unloadModel(payload: UnloadModelRequest): Promise { await parseJsonOrThrow(response); } +export interface CachedGgufRepo { + repo_id: string; + size_bytes: number; + cache_path: string; +} + +export async function listCachedGguf(): Promise { + const response = await authFetch("/api/models/cached-gguf"); + const data = await parseJsonOrThrow<{ cached: CachedGgufRepo[] }>(response); + return data.cached; +} + export async function listGgufVariants( repoId: string, hfToken?: string,