diff --git a/studio/backend/auth/authentication.py b/studio/backend/auth/authentication.py index dfb8fc513e..2481cd13e6 100644 --- a/studio/backend/auth/authentication.py +++ b/studio/backend/auth/authentication.py @@ -164,6 +164,22 @@ async def get_current_subject_allow_password_change( ) +# The literal the examples ship with; pasting one unedited is likelier than a revoked key. +API_KEY_PLACEHOLDER = f"{API_KEY_PREFIX}YOUR_KEY" + + +def _invalid_api_key_detail(token: str) -> str: + """Why the key failed. Only the unedited example placeholder is called out; + every real key still gets one indistinguishable message, so this reveals + nothing about which keys exist.""" + if token == API_KEY_PLACEHOLDER: + return ( + "This is the placeholder key from the example. Create an API key in " + f"Unsloth Studio under Settings > API and use it in place of {API_KEY_PLACEHOLDER}." + ) + return "Invalid or expired API key" + + async def _get_current_subject( credentials: HTTPAuthorizationCredentials, *, allow_password_change: bool ) -> str: @@ -176,7 +192,7 @@ async def _get_current_subject( if username is None: raise HTTPException( status_code = status.HTTP_401_UNAUTHORIZED, - detail = "Invalid or expired API key", + detail = _invalid_api_key_detail(token), ) return username diff --git a/studio/backend/core/inference/api_monitor.py b/studio/backend/core/inference/api_monitor.py index f76a38576f..2de042ab37 100644 --- a/studio/backend/core/inference/api_monitor.py +++ b/studio/backend/core/inference/api_monitor.py @@ -52,6 +52,13 @@ class ApiMonitorEntry: total_tokens: Optional[int] = None total_tokens_authoritative: bool = False error: Optional[str] = None + # "request" (HTTP call) or "lifecycle" (model load/unload: event/reason, not a prompt; shared). + kind: str = "request" + event: Optional[str] = None + reason: Optional[str] = None + shared: bool = False + # 0-100 for a running download row; None when not applicable. + progress: Optional[float] = None def snapshot(self, *, include_details: bool = True) -> dict[str, Any]: duration_ms = None @@ -85,6 +92,10 @@ class ApiMonitorEntry: "completion_tokens": self.completion_tokens, "total_tokens": self.total_tokens, "error": self.error, + "kind": self.kind, + "event": self.event, + "reason": self.reason, + "progress": self.progress, } if include_details: payload["prompt"] = self.prompt @@ -127,6 +138,75 @@ class ApiMonitor: self._trim_terminal_locked() return entry.id + def record_lifecycle( + self, + *, + event: str, + model: str, + reason: Optional[str] = None, + running: bool = False, + ) -> str: + """Record a model load/unload alongside the request traffic that caused it. + + ``running=True`` opens the row (a load in progress) and the caller closes + it with the usual :meth:`finish` / :meth:`fail`; an unload is terminal on + arrival. Rows are shared, so every subject sees them, and share the same + retention budget as requests. + """ + now = time.time() + entry = ApiMonitorEntry( + id = f"apievt_{uuid.uuid4().hex[:12]}", + endpoint = f"model.{event}", + method = "", + model = model or "default", + prompt = "", + status = "running" if running else "completed", + started_at = now, + updated_at = now, + started_monotonic = time.monotonic(), + finished_at = None if running else now, + finished_monotonic = None if running else time.monotonic(), + kind = "lifecycle", + event = event, + reason = reason, + shared = True, + ) + with self._lock: + self._entries.appendleft(entry) + self._trim_terminal_locked() + return entry.id + + def relabel(self, entry_id: Optional[str], model: str) -> None: + """Rename an open lifecycle row once the load resolves its real id (the + caller only has the load path up front, which may be an HF snapshot dir).""" + if not entry_id or not model: + return + with self._lock: + entry = self._find_locked(entry_id) + if entry is not None: + entry.model = model + entry.updated_at = time.time() + + def set_progress(self, entry_id: Optional[str], progress: Optional[float]) -> None: + """Update an open download row's percentage (clamped to 0-100).""" + if not entry_id or progress is None: + return + with self._lock: + entry = self._find_locked(entry_id) + if entry is not None and entry.status == "running": + entry.progress = min(100.0, max(0.0, float(progress))) + entry.updated_at = time.time() + + def discard(self, entry_id: Optional[str]) -> None: + """Drop a row that turned out not to be an event (a load that was already + satisfied, so nothing was actually loaded).""" + if not entry_id: + return + with self._lock: + entry = self._find_locked(entry_id) + if entry is not None: + self._entries.remove(entry) + def append_reply(self, entry_id: Optional[str], text: str) -> None: if not entry_id or not text: return @@ -212,6 +292,19 @@ class ApiMonitor: self._entries.appendleft(entry) self._trim_terminal_locked() + def fail_open(self, entry_id: Optional[str], error: str) -> None: + """Fail only a still-open row. Unlike :meth:`fail` this never touches an + entry that already finished, so a catch-all in a ``finally`` cannot stamp + an error onto a request that in fact succeeded.""" + if not entry_id: + return + with self._lock: + entry = self._find_locked(entry_id) + if entry is None or entry.finished_at is not None: + return + # Same lock as the check, so a finish() cannot land in between. + self._fail_locked(entry, error) + def fail(self, entry_id: Optional[str], error: str) -> None: if not entry_id: return @@ -224,15 +317,18 @@ class ApiMonitor: if error: entry.error = _trim(error, 1000) return - now = time.time() - entry.status = "error" - entry.error = _trim(error, 1000) - entry.updated_at = now - entry.finished_at = now - entry.finished_monotonic = time.monotonic() - self._entries.remove(entry) - self._entries.appendleft(entry) - self._trim_terminal_locked() + self._fail_locked(entry, error) + + def _fail_locked(self, entry: ApiMonitorEntry, error: str) -> None: + now = time.time() + entry.status = "error" + entry.error = _trim(error, 1000) + entry.updated_at = now + entry.finished_at = now + entry.finished_monotonic = time.monotonic() + self._entries.remove(entry) + self._entries.appendleft(entry) + self._trim_terminal_locked() def snapshot( self, @@ -244,7 +340,7 @@ class ApiMonitor: return [ entry.snapshot(include_details = include_details) for entry in self._entries - if subject is None or entry.subject == subject + if self._visible(entry, subject) ] def get( @@ -257,22 +353,29 @@ class ApiMonitor: entry = self._find_locked(entry_id) if entry is None: return None - if subject is not None and entry.subject != subject: + if not self._visible(entry, subject): return None return entry.snapshot(include_details = True) def active_count(self, *, subject: Optional[str] = None) -> int: + # Lifecycle rows show as "running" while loading but are not in-flight API requests. with self._lock: return sum( 1 for entry in self._entries - if entry.status == "running" and (subject is None or entry.subject == subject) + if entry.status == "running" + and entry.kind != "lifecycle" + and (subject is None or entry.subject == subject) ) def clear(self) -> None: with self._lock: self._entries.clear() + @staticmethod + def _visible(entry: ApiMonitorEntry, subject: Optional[str]) -> bool: + return subject is None or entry.subject == subject or entry.shared + def _find_locked(self, entry_id: str) -> Optional[ApiMonitorEntry]: for entry in self._entries: if entry.id == entry_id: diff --git a/studio/backend/core/inference/llama_keepwarm.py b/studio/backend/core/inference/llama_keepwarm.py index 3380ebf5f5..f3ec5f573f 100644 --- a/studio/backend/core/inference/llama_keepwarm.py +++ b/studio/backend/core/inference/llama_keepwarm.py @@ -345,6 +345,22 @@ def _loaded_identity(backend): return (backend.model_identifier, getattr(backend, "hf_variant", None), advertised) +def _note_idle_unload_event(freed) -> None: + """Record an idle auto-unload in the API monitor, using the advertised repo id + from the stash so the row never shows the on-disk load path. Best-effort.""" + try: + from core.inference.api_monitor import api_monitor + from core.inference.model_ids import public_model_id + + identifier, variant, advertised = (list(freed) + [None, None, None])[:3] + label = public_model_id(advertised or identifier) or "model" + if variant and ":" not in label: + label = f"{label}:{variant}" + api_monitor.record_lifecycle(event = "unload", model = label, reason = "idle") + except Exception as exc: + logger.debug("idle unload monitor event failed: %s", exc) + + async def idle_unload_loop(poll_seconds: float = 15.0) -> None: """Unload the loaded GGUF once idle past the configured TTL. Inert when off.""" from utils.openai_auto_switch_settings import ( @@ -407,6 +423,8 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None: elif manifest: _delete_resume_files(manifest) logger.info("Idle auto-unload: freed GGUF after %ss idle", ttl) + # An idle unload stashes for reload and skips note_model_unloaded. + _note_idle_unload_event(freed) seen_model = None except Exception as exc: logger.debug("idle_unload_loop iteration failed: %s", exc) diff --git a/studio/backend/core/inference/local_model_resolver.py b/studio/backend/core/inference/local_model_resolver.py index e6014f442d..c4ac085ebe 100644 --- a/studio/backend/core/inference/local_model_resolver.py +++ b/studio/backend/core/inference/local_model_resolver.py @@ -34,6 +34,16 @@ class _LocalGgufEntry: _CACHE_TTL_S = 5.0 _lock = threading.Lock() _scan: tuple[float, dict[str, _LocalGgufEntry]] = (0.0, {}) +# Not _lock: that is held for the whole scan, so the request path would wait on it. +_warm_lock = threading.Lock() +# Repos that finished downloading but are not in the published index yet. The +# retained index covers what was already known; nothing covers the one that just +# landed until the next scan, and the request path must not call it absent. +_just_downloaded: set[str] = set() +_warming = False +_last_scan_s = 0.0 +# Rescan at most a tenth of the time: on the TTL alone a slow scan would run continuously. +_WARM_DUTY = 10.0 def _is_abs_path_id(value: str) -> bool: @@ -103,17 +113,28 @@ def _local_gguf_entry(loader_id: str, info) -> Optional[_LocalGgufEntry]: load_dir = _resolve_load_dir(p) variants, _ = list_local_gguf_variants(str(load_dir)) quants = tuple(v.quant for v in variants if getattr(v, "quant", None)) - return _LocalGgufEntry(loader_id, str(load_dir), quants) if quants else None + if not quants: + return None + # That call orders by descending size, so the head is the biggest quant, + # often F16. A bare id means whichever quant a plain load would take, so put + # that first: everything downstream reads [0], and answering with the + # largest can evict a working model and then OOM starting it. + from core.inference.openai_auto_download import preferred_quant + + best = preferred_quant(quants) + if best and quants[0] != best: + quants = (best, *(q for q in quants if q != best)) + return _LocalGgufEntry(loader_id, str(load_dir), quants) except Exception: return None -def info_has_local_gguf(info) -> bool: - """True when *info* (a LocalModelInfo) points to on-disk GGUF weights the - auto-switch path can load. Read from the files, not ``info.model_format``: the - HF-cache scanner leaves model_format unset for GGUF snapshots, so a - model_format filter would drop every cached GGUF. Lets /v1/models advertise - exactly what /v1 can serve.""" +def local_gguf_quants(info) -> Optional[tuple[str, ...]]: + """On-disk quant labels for *info*, or None when it is not a servable local + GGUF. Read from the files, not ``info.model_format``: the HF-cache scanner + leaves model_format unset for GGUF snapshots, so a model_format filter would + drop every cached GGUF. Lets /v1/models advertise exactly what /v1 can serve, + and which quant to name, from a single scan.""" from pathlib import Path path = getattr(info, "path", None) @@ -123,8 +144,14 @@ def info_has_local_gguf(info) -> bool: if isinstance(path, str) and any( seg in (".studio_links", "ollama_links") for seg in Path(path).parts ): - return False - return _local_gguf_entry(getattr(info, "id", "") or "", info) is not None + return None + entry = _local_gguf_entry(getattr(info, "id", "") or "", info) + return entry.variants if entry is not None else None + + +def info_has_local_gguf(info) -> bool: + """True when *info* points to on-disk GGUF weights the auto-switch path can load.""" + return local_gguf_quants(info) is not None def _build_index() -> dict[str, _LocalGgufEntry]: @@ -287,6 +314,36 @@ def _sibling_revision_entries(raw_id: str, loader_id: str): yield sibling.name, entry +def note_downloaded(repo_id: Optional[str]) -> None: + """Record a repo as present ahead of the scan that will index it.""" + if not repo_id: + return + with _lock: + _just_downloaded.add(repo_id.strip().lower()) + + +def recently_downloaded(repo_id: str) -> bool: + """Whether *repo_id* finished downloading since the last completed scan.""" + if not isinstance(repo_id, str) or not repo_id.strip(): + return False + return repo_id.strip().lower() in _just_downloaded + + +def invalidate_index() -> None: + """Mark the cached scan stale so the next resolve sees a just-finished + download, rather than waiting out the TTL. + + Keeps the entries. Callers on the request path read this cache without + scanning, so emptying it would leave them with no evidence about any local + model until the rebuild lands, and a bare request for one of them would be + answered by whatever is resident. Only a completed download invalidates, and + that only ever adds models, so the retained entries stay true. + """ + global _scan + with _lock: + _scan = (0.0, _scan[1]) + + def _index() -> dict[str, _LocalGgufEntry]: global _scan # Build under the lock so concurrent callers with an expired cache don't all @@ -301,23 +358,78 @@ def _index() -> dict[str, _LocalGgufEntry]: # an install with many local models can itself exceed the TTL, which would # store the cache already expired and make every request rebuild the index. _scan = (time.monotonic(), fresh) + # The scan supersedes the notes: whatever landed is in the index now. + _just_downloaded.clear() return fresh -def resolve_local_gguf(requested: str) -> Optional[tuple[str, Optional[str], str]]: +def index_is_built() -> bool: + """Whether a scan has ever completed, freshness aside. + + Lock-free on purpose: ``_lock`` is held for the whole scan, so taking it here + would park the request path on the very scan it is trying to stay off. Reading + ``_scan[0]`` is safe because ``_scan`` is only ever rebound, never mutated. + """ + return bool(_scan[0]) + + +def warm_index_soon() -> None: + """(Re)build the index off the request path when it is missing or past its TTL. + + Callers that cannot afford the scan use this plus ``allow_scan=False``, so this + is the only thing that ever refreshes the index for them. It has to cover a + stale index and not just an absent one: a model downloaded through the Hub UI + or dropped into a scan folder has no invalidation hook, and would otherwise stay + invisible to those callers for the life of the process. + + Never touches ``_lock``, which the scan holds throughout, and never blocks. + """ + global _warming + if time.monotonic() - _scan[0] < max(_CACHE_TTL_S, _last_scan_s * _WARM_DUTY): + return + with _warm_lock: + if _warming: + return + _warming = True + + def _run() -> None: + global _warming, _last_scan_s + started = time.monotonic() + try: + _index() + except Exception: + pass + finally: + _last_scan_s = time.monotonic() - started + with _warm_lock: + _warming = False + + threading.Thread(target = _run, name = "local-model-index-warm", daemon = True).start() + + +def resolve_local_gguf( + requested: str, *, allow_scan: bool = True +) -> Optional[tuple[str, Optional[str], str]]: """Return ``(load_path, gguf_variant, loader_id)`` for a local match, else None. ``load_path`` is the concrete on-disk path to hand /load (so it never fetches a remote), ``loader_id`` is the advertised id used as the launch-override key. ``requested`` is ``repo`` or ``repo:VARIANT``. An exact id match wins first (so ids containing a colon still resolve); else the last ``:VARIANT`` is split - off and resolves only when that quant is on disk. + off and resolves only when that quant is on disk, unless it names no quant at + all (an Ollama-style ":latest"), which means the repo. + + ``allow_scan=False`` answers from the last built index and never rebuilds, + for callers on the request path: the scan walks several model dirs and HF + caches, takes seconds on a large install, and holds a lock every other + caller queues behind. A stale answer is fine there, since what is on disk + barely moves and a finished download calls :func:`invalidate_index`. """ if not isinstance(requested, str) or not requested.strip(): return None requested = requested.strip() try: - index = _index() + index = _index() if allow_scan else _scan[1] entry = index.get(requested.lower()) if entry is not None: variant = entry.variants[0] if entry.variants else None @@ -333,8 +445,45 @@ def resolve_local_gguf(requested: str) -> Optional[tuple[str, Optional[str], str for v in entry.variants: if v.lower() == wanted: return entry.load_path, v, entry.loader_id - return None + from core.inference.openai_auto_download import looks_like_quant + + if looks_like_quant(variant): + return None + # ":latest" or ":8b" names no file, so it means the repo; a real quant that + # is not on disk still misses, or a swap would serve the wrong weights. + return entry.load_path, (entry.variants[0] if entry.variants else None), entry.loader_id except Exception: # Best-effort: any resolver failure falls through to the loaded model, # so a malformed name can never turn a servable request into a 500. return None + + +MISS_MODEL_NOT_FOUND = "model_not_found" +MISS_VARIANT_NOT_FOUND = "variant_not_found" + + +def describe_local_miss(requested: str) -> tuple[str, tuple[str, ...]]: + """Why :func:`resolve_local_gguf` missed, so an error can say "wrong quant" + instead of "no such model". + + ``(MISS_VARIANT_NOT_FOUND, )`` when the repo is downloaded but + the requested ``:VARIANT`` is not, else ``(MISS_MODEL_NOT_FOUND, ())``. Splits + the name like the resolver so the two agree. Fail-safe: a scan failure reports + the generic miss rather than raising into the handler. + """ + if not isinstance(requested, str) or not requested.strip(): + return MISS_MODEL_NOT_FOUND, () + base, sep, variant = requested.strip().rpartition(":") + from core.inference.openai_auto_download import looks_like_quant + + # Split like the resolver or the two disagree: a tag naming no quant means the + # repo there, so reporting a missing quant for it would name one nobody asked for. + if not sep or not looks_like_quant(variant): + return MISS_MODEL_NOT_FOUND, () + try: + entry = _index().get(base.strip().lower()) + except Exception: + return MISS_MODEL_NOT_FOUND, () + if entry is None or not entry.variants: + return MISS_MODEL_NOT_FOUND, () + return MISS_VARIANT_NOT_FOUND, entry.variants diff --git a/studio/backend/core/inference/model_ids.py b/studio/backend/core/inference/model_ids.py index 548cc60f94..a6270b955e 100644 --- a/studio/backend/core/inference/model_ids.py +++ b/studio/backend/core/inference/model_ids.py @@ -39,10 +39,30 @@ def _looks_like_path(identifier: str) -> bool: return False +def hf_cache_repo_id(path: Optional[str]) -> Optional[str]: + """``.../models--org--name/snapshots/`` -> ``org/name``, else None. + + A model loaded straight out of the HF cache has a snapshot directory as its + identifier, whose basename is a commit hash. Recover the repo id so callers + show ``unsloth/gemma-4-31B-it-GGUF`` rather than ``c1ac76e99d55...``. + """ + if not path: + return None + parts = str(path).replace("\\", "/").split("/") + for index, part in enumerate(parts): + # Only inside the real cache layout: a "models--" name alone is not a repo id. + if part.startswith("models--") and parts[index + 1 : index + 2] == ["snapshots"]: + return part[len("models--") :].replace("--", "/") + return None + + def public_model_id(identifier: Optional[str]) -> Optional[str]: """Return a clean, path-free public id for *identifier*. - - Local GGUF path -> the file stem with ``.gguf`` stripped, e.g. + - HF cache path -> the repo id it came from, e.g. + ``~/.cache/huggingface/hub/models--unsloth--X-GGUF/snapshots/`` -> + ``unsloth/X-GGUF``. + - Other local GGUF path -> the file stem with ``.gguf`` stripped, e.g. ``/srv/models/Qwen3-30B-A3B-Q4_K_M.gguf`` -> ``Qwen3-30B-A3B-Q4_K_M``. - HF repo id (``org/model``) and already-clean names -> returned unchanged. - ``None`` / empty -> returned unchanged. @@ -51,6 +71,9 @@ def public_model_id(identifier: Optional[str]) -> Optional[str]: return identifier if not _looks_like_path(identifier): return identifier + repo_id = hf_cache_repo_id(identifier) + if repo_id: + return repo_id name = os.path.basename(identifier.replace("\\", "/").rstrip("/")) if name.lower().endswith(_GGUF_SUFFIX): name = name[: -len(_GGUF_SUFFIX)] diff --git a/studio/backend/core/inference/openai_auto_download.py b/studio/backend/core/inference/openai_auto_download.py new file mode 100644 index 0000000000..fee87a42f2 --- /dev/null +++ b/studio/backend/core/inference/openai_auto_download.py @@ -0,0 +1,831 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Opt-in: fetch a GGUF a /v1 request names but this server doesn't have. + +Auto-switch only loads models already on disk. With +``openai_api_auto_download_model`` on, a miss that looks like a real Hub repo is +downloaded in the background instead of erroring, and the request is told to +retry rather than being held open: a quant is routinely tens of GB, far longer +than any client (or the Cloudflare edge on ``--secure``) will wait, and the +inference lifecycle gate must not be held meanwhile. The resident model keeps +serving throughout, and the retry that lands after the download is served by the +new model through the ordinary auto-switch path. + +Admission is deliberately narrow, since a request only needs an API key: +- ``namespace/name`` only, and only when the Hub confirms it is a GGUF repo. + ``gpt-4`` and ``anthropic/claude-3.5-sonnet`` alike fall through to the + resident model as before: a namespace is not evidence of intent, since LiteLLM + and OpenRouter address every provider that way. +- GGUF repos only, decided from the remote file list, not the repo name. GGUF + runs under llama.cpp, which never imports repo Python. +- Anything declaring ``auto_map`` is refused, so ``trust_remote_code`` can only + ever be granted deliberately in the UI, never by an API call. +- One download at a time, so a key holder cannot fan out fetches. +""" + +from __future__ import annotations + +import asyncio +import shutil +import threading +import time +from dataclasses import dataclass +from typing import Optional + +from loggers import get_logger + +logger = get_logger(__name__) + +# Keep the Hub probe short so a slow Hub can't stall the request path. +_MODEL_INFO_TIMEOUT_S = 8.0 +# auth_check and hf_hub_download take no timeout of their own, and both run while the +# provisional slot is held, so an unresponsive Hub would pin the single flight and stall +# the request long past the metadata budget. The code probe fetches up to three small +# configs, so it gets more room than the single auth call. +_CODE_PROBE_TIMEOUT_S = 20.0 +# Headroom left free after the download, so filling the disk can't wedge the box. +_DISK_RESERVE_BYTES = 5 * 1024**3 +_WATCH_POLL_S = 2.0 +# A stalled watcher must not pin the single-flight slot forever. +_MAX_WATCH_S = 24 * 60 * 60 +# Past the watch window the row is already resolved, so poll only to see whether +# the worker is still alive and still owns the slot. +_TIMED_OUT_POLL_S = 60.0 +_RETRY_AFTER_S = 30 +# Long enough for a client honouring Retry-After to come back and be told, short +# enough that a client that never returns cannot hold the slot. +_FAILED_HOLD_S = 3 * _RETRY_AFTER_S +_MAX_LISTED_VARIANTS = 8 + + +@dataclass(frozen = True) +class AutoDownloadRefusal: + """Why this request cannot be served yet. The route turns it into an + HTTPException with the surface's own error envelope.""" + + status: int + code: str + message: str + retry_after: Optional[int] = None + + +@dataclass +class _Active: + repo_id: str + # None while the Hub probe is still deciding which quant to fetch. + variant: Optional[str] = None + expected_bytes: int = 0 + monitor_id: Optional[str] = None + started_at: float = 0.0 + # Set when the worker failed. The slot is kept until a retry surfaces it, since + # the advertised retry interval is far longer than the watcher's poll and the + # client would otherwise just restart the same failing download. + error: Optional[str] = None + failed_at: float = 0.0 + + +_lock = threading.Lock() +_active: Optional[_Active] = None + +# Repos the Hub says are not servable, so a "vendor/model" miss doesn't re-probe every request. +_NOT_SERVABLE_TTL_S = 10 * 60 +_NOT_SERVABLE_MAX = 256 +_cache_lock = threading.Lock() +_not_servable: dict[str, float] = {} + + +def _public_label(repo_id: str, variant: Optional[str]) -> str: + return f"{repo_id}:{variant}" if variant else repo_id + + +def split_model_ref(requested: str) -> tuple[str, Optional[str]]: + """``org/repo:QUANT`` -> ``("org/repo", "QUANT")``; no suffix -> variant None. + + Splits on the last colon. A slash-bearing suffix is only a variant when a real + Hub repo precedes it: an unrecognized GGUF below a subdirectory keys on its path + ("build/llama-13b", which is_valid_gguf_variant allows and the catalog advertises), + while "C:/models/x.gguf" leaves a drive letter that is no repo id at all. + """ + text = (requested or "").strip() + base, sep, suffix = text.rpartition(":") + if not sep or not base or not suffix: + return text, None + stripped = base.strip() + if "/" in suffix: + from hub.utils.paths import is_valid_repo_id + if "/" not in stripped or not is_valid_repo_id(stripped): + return text, None + return stripped, suffix.strip() + + +def is_downloadable_ref(requested: str) -> bool: + """Whether *requested* is shaped like a Hub repo we may fetch. + + Requires an explicit namespace. That keeps ``gpt-4`` and other foreign ids + falling through untouched, and avoids the bare-name ``unsloth/`` prefixing in + ModelConfig.from_identifier turning an unrelated label into a real repo. + """ + from hub.utils.paths import is_valid_repo_id + + repo_id, variant = split_model_ref(requested) + if "/" not in repo_id or not is_valid_repo_id(repo_id): + return False + if variant is not None: + from hub.utils.paths import is_valid_gguf_variant + return is_valid_gguf_variant(variant) + return True + + +def looks_like_quant(variant: Optional[str]) -> bool: + """Whether a ``:suffix`` names a GGUF quant rather than a foreign tag. + + ``vendor/model`` is how LiteLLM and OpenRouter address every provider, and + ``name:latest`` is how Ollama tags one, so neither a namespace nor a colon + proves a request was meant for this server. A real quant label does. + """ + import re + + from utils.models.model_config import _GGUF_KNOWN_QUANT_RE + + if not variant: + return False + # _extract_quant_label can append a bpw modifier (IQ4_XS-3.53bpw); still a quant. + label = re.sub(r"-[0-9]+(?:\.[0-9]+)?bpw$", "", variant.strip(), flags = re.IGNORECASE) + return _GGUF_KNOWN_QUANT_RE.fullmatch(label) is not None + + +def _hub_token(hf_token: Optional[str]): + """The caller's token, or an explicit False. + + None makes huggingface_hub fall back to a cached login, which here would be + the server owner's. False is what actually means anonymous. + """ + return hf_token or False + + +def _servable_key(repo_id: str, hf_token: Optional[str]) -> str: + """Cache key, per credential. + + The Hub answers 404 for a private repo the caller cannot see, so a verdict + reached without a token says nothing about a caller who has one. Keyed on a + digest so the token itself is never held here. + """ + import hashlib + + seen_as = hashlib.sha256(hf_token.encode()).hexdigest()[:16] if hf_token else "anon" + return f"{repo_id.lower()}\n{seen_as}" + + +def _mark_not_servable(repo_id: str, hf_token: Optional[str]) -> None: + with _cache_lock: + if len(_not_servable) >= _NOT_SERVABLE_MAX: + _not_servable.clear() + _not_servable[_servable_key(repo_id, hf_token)] = time.monotonic() + _NOT_SERVABLE_TTL_S + + +def _is_not_servable(repo_id: str, hf_token: Optional[str]) -> bool: + key = _servable_key(repo_id, hf_token) + with _cache_lock: + expires = _not_servable.get(key) + if expires is None: + return False + if expires <= time.monotonic(): + del _not_servable[key] + return False + return True + + +def _gated_refusal(repo_id: str) -> AutoDownloadRefusal: + return AutoDownloadRefusal( + status = 403, + code = "model_access_denied", + message = ( + f"'{repo_id}' is gated on Hugging Face. Accept its licence, then retry with " + "your own token in the X-Unsloth-HF-Token header: automatic download never " + "uses this server's Hugging Face identity." + ), + ) + + +async def _bounded_probe(fn, *args, timeout: float, default): + """Run a blocking Hub probe off the loop, bounding only the wait. + + The thread is left to finish (a blocking socket read cannot be cancelled); the + caller stops waiting and takes *default*, which each call site chooses so that a + timeout errs the safe way. + """ + try: + return await asyncio.wait_for(asyncio.to_thread(fn, *args), timeout) + except (TimeoutError, asyncio.TimeoutError): + logger.debug("hub probe %s timed out after %ss", getattr(fn, "__name__", fn), timeout) + return default + + +def _auth_denied(repo_id: str, hf_token: Optional[str]) -> bool: + """Whether this token lacks file access to a gated repo. False when the + check is inconclusive: the download's own auth is the real gate.""" + from hub.utils.hf_errors import hf_error_status + + try: + from huggingface_hub import auth_check + auth_check(repo_id, token = _hub_token(hf_token)) + except Exception as exc: + return hf_error_status(exc) in (401, 403) + return False + + +def _gguf_variants(siblings) -> dict[str, int]: + """Quant label -> bytes the download will actually fetch. + + Mirrors list_gguf_variants for the selectable labels: companions (mmproj/MTP) + and big-endian builds are not quants of their own, and sharded quants sum + across their shards. The byte total comes from the download plan, which folds + the companions back into every quant, so the disk reserve is measured against + what the worker fetches rather than the main files alone. + """ + from hub.utils.gguf import extract_quant_label as canonical_quant_label + from hub.utils.gguf_plan import build_gguf_variant_plans + from utils.models.model_config import ( + _extract_quant_label, + _is_big_endian_gguf_path, + _is_mmproj, + _is_mtp_drafter, + ) + + siblings = list(siblings or []) + plans = build_gguf_variant_plans(siblings) + sizes: dict[str, int] = {} + for sibling in siblings: + name = getattr(sibling, "rfilename", "") or "" + if not name.lower().endswith(".gguf"): + continue + quant = _extract_quant_label(name) + if not looks_like_quant(quant): + # With no recognized quant token the two extractors part ways: this one + # takes the last hyphenated segment ("7b" of llama-7b) while the plan and + # the worker key the whole stem. Advertising ours dispatches a variant the + # worker cannot resolve, so take theirs for the unrecognized case only. + quant = canonical_quant_label(name) or quant + if _is_mmproj(name) or _is_mtp_drafter(name) or _is_big_endian_gguf_path(name, quant): + continue + plan = plans.get(quant.lower()) + if plan is not None: + sizes[quant] = plan.download_size_bytes + else: + sizes[quant] = sizes.get(quant, 0) + int(getattr(sibling, "size", 0) or 0) + return sizes + + +def _remaining_bytes(repo_id: str, plan, expected_bytes: int) -> int: + """Bytes still to fetch: a resumed quant or a companion shared with another + quant is already on disk, and charging for it can 507 a download that fits.""" + try: + from hub.utils.download_registry import existing_blob_bytes + + hashes = frozenset( + file.sha256 for file in getattr(plan, "expected_files", ()) or () if file.sha256 + ) + if not hashes: + return expected_bytes + return max(0, expected_bytes - existing_blob_bytes("model", repo_id, hashes)) + except Exception: + return expected_bytes + + +def _enough_disk(need_bytes: int) -> tuple[bool, int]: + """(fits, free_bytes). Fail-open on an unreadable cache root: the download + worker runs its own preflight, this only adds the reserve margin.""" + try: + from hub.utils.hf_cache_state import hf_cache_root + + root = hf_cache_root(create = True) + if root is None: + return True, 0 + free = shutil.disk_usage(root).free + except Exception: + return True, 0 + return free >= need_bytes + _DISK_RESERVE_BYTES, free + + +def _gb(num_bytes: int) -> str: + return f"{num_bytes / 1024**3:.1f} GB" + + +async def _job_state(repo_id: str, variant: Optional[str]) -> tuple[str, Optional[str]]: + from hub.services.models import downloads + try: + status = await downloads.get_download_status_response(repo_id, variant or "") + return status.state, status.error + except Exception as exc: + # "unknown", not "idle": idle ends the watch, and a failed probe proves nothing. + logger.debug("auto-download: status probe failed for %r: %s", repo_id, exc) + return "unknown", None + + +async def _progress_percent( + repo_id: str, variant: Optional[str], expected_bytes: int, hf_token: Optional[str] +) -> Optional[float]: + """0-100, or None. The hub service reports a 0-1 fraction, so scale it.""" + from hub.services.models import downloads + try: + payload = await downloads.get_gguf_download_progress_response( + repo_id, variant or "", expected_bytes, hf_token + ) + fraction = payload.get("progress") + if not isinstance(fraction, (int, float)): + return None + return min(100.0, max(0.0, float(fraction) * 100.0)) + except Exception: + return None + + +def _release(active: Optional[_Active]) -> None: + """Free the single-flight slot, but only while *active* still owns it. + + Keying the release on ``repo_id`` alone let a stale operation clear a newer + one for the same repo: variant A errors, an adopting request frees the slot, + a retry starts variant B, and A's watcher then matches on the repo and clears + B on its way out -- admitting a second repository download alongside B. + Identity ties every release to the operation that actually took the slot. + """ + global _active + if active is None: + return + with _lock: + if _active is active: + _active = None + + +async def _watch(active: _Active, hf_token: Optional[str]) -> None: + """Poll a dispatched job so the monitor row resolves and the resolver cache + is dropped the moment the weights land.""" + from core.inference import api_monitor as monitor_module + + api_monitor = monitor_module.api_monitor + deadline = time.monotonic() + _MAX_WATCH_S + timed_out = False + try: + while True: + await asyncio.sleep(_TIMED_OUT_POLL_S if timed_out else _WATCH_POLL_S) + state, error = await _job_state(active.repo_id, active.variant) + if state in ("running", "cancelling", "unknown"): + if timed_out: + # A worker still running still owns the slot: releasing it on the + # clock alone would admit a second multi-GB download alongside it. + # "unknown" cannot confirm it is alive, so stop holding it then, + # or a broken probe would wedge auto-download for good. + if state == "unknown": + return + continue + if time.monotonic() >= deadline: + api_monitor.fail_open(active.monitor_id, "Download timed out") + timed_out = True + continue + # Only "running" has progress; the others are still in flight, so keep the slot. + if state == "running": + api_monitor.set_progress( + active.monitor_id, + await _progress_percent( + active.repo_id, active.variant, active.expected_bytes, hf_token + ), + ) + continue + if state == "cancelled": + api_monitor.finish(active.monitor_id, status = "cancelled") + return + if state == "complete": + # No invalidate here: finalize_worker_exit already dropped the cache and + # started the warm, and a second one would mark that fresh scan stale and + # push a synchronous rescan onto the client's retry. + api_monitor.finish(active.monitor_id, status = "completed") + elif state == "idle": + # The job vanished without a terminal state (worker killed). + api_monitor.fail_open(active.monitor_id, "Download did not complete") + else: + api_monitor.fail_open(active.monitor_id, error or f"Download {state}") + # Keep the slot so the next retry is told it failed rather than + # silently starting the same download again. + active.error = error or f"Download {state}" + active.failed_at = time.monotonic() + return + return + except asyncio.CancelledError: + raise + except Exception as exc: + logger.warning("auto-download: watcher failed for %r: %s", active.repo_id, exc) + api_monitor.fail_open(active.monitor_id, "Download tracking failed") + finally: + if not active.failed_at: + _release(active) + + +def _downloading_refusal(label: str, percent: Optional[float]) -> AutoDownloadRefusal: + progress = f" ({percent:.0f}% done)" if percent is not None else "" + return AutoDownloadRefusal( + status = 503, + code = "model_downloading", + message = (f"Downloading '{label}'{progress}. Retry shortly. Track it in Unsloth Studio."), + retry_after = _RETRY_AFTER_S, + ) + + +async def _is_downloadable_model(repo_id: str, hf_token: Optional[str]) -> bool: + """Whether the Hub has this repo with GGUF weights we could fetch. + + Only asked while another download holds the slot, to tell a second download + apart from an ordinary foreign label. Any failure answers False: falling + through to the resident model is what such a label does anyway, and refusing + it would strand normal traffic for the length of the download. + """ + if _is_not_servable(repo_id, hf_token): + return False + + def _probe(): + from huggingface_hub import HfApi + return HfApi(token = _hub_token(hf_token)).model_info(repo_id, timeout = _MODEL_INFO_TIMEOUT_S) + + try: + info = await asyncio.to_thread(_probe) + except Exception: + return False + # The same filter admission uses, not a bare .gguf test: mmproj, MTP drafters and + # big-endian builds are companions rather than quants, so a repo holding only those + # is not downloadable here either. Answering otherwise would hold an ordinary + # foreign label at model_download_busy for the length of an unrelated download. + servable = bool(_gguf_variants(getattr(info, "siblings", None))) + if not servable: + _mark_not_servable(repo_id, hf_token) + return servable + + +async def maybe_auto_download( + requested_model: str, + *, + hf_token: Optional[str] = None, + require_vision: bool = False, +) -> Optional[AutoDownloadRefusal]: + """Start (or report on) a background fetch of *requested_model*. + + Returns None when the request should carry on unchanged, or a refusal the + caller must raise. Only called after the local resolver has already missed. + + ``require_vision`` refuses a target with no mmproj companion rather than + spending gigabytes on weights that cannot answer the request that asked for + them; the local capability guard only ever sees an already-downloaded model. + """ + global _active + + repo_id, wanted_variant = split_model_ref(requested_model) + if not is_downloadable_ref(requested_model): + return None + if _is_not_servable(repo_id, hf_token) and not looks_like_quant(wanted_variant): + return None + + # Settle the single-flight slot before the network, so retries during a download stay cheap. + busy: Optional[_Active] = None + with _lock: + current = _active + if current is not None and current.failed_at: + # A held failure only owns the slot until someone is told about it. + if current.repo_id != repo_id and time.monotonic() - current.failed_at > _FAILED_HOLD_S: + _active = current = None + if current is not None and current.repo_id == repo_id: + adopted = current + elif current is not None: + adopted = None + busy = current + else: + adopted = None + provisional = _Active(repo_id = repo_id, started_at = time.time()) + _active = provisional + + if busy is not None: + # Refusing before the probe blocks ordinary drop-in traffic: a namespaced label + # that is not a downloadable GGUF repo (LiteLLM/OpenRouter style) would be told + # to wait out a multi-hour download instead of falling through to the resident + # model. Only a label that could itself be downloaded is a second download. + if not await _is_downloadable_model(repo_id, hf_token): + return None + return AutoDownloadRefusal( + status = 503, + code = "model_download_busy", + message = ( + f"Already downloading '{_public_label(busy.repo_id, busy.variant)}'. " + f"Retry '{requested_model}' once it finishes." + ), + retry_after = _RETRY_AFTER_S, + ) + + if adopted is not None: + if adopted.variant is None: + # Still probing: no job yet, and a stale whole-repo error would free the probe's slot. + return _downloading_refusal(adopted.repo_id, None) + state, error = await _job_state(adopted.repo_id, adopted.variant) + if state in ("running", "cancelling", "unknown"): + return _downloading_refusal( + _public_label(adopted.repo_id, adopted.variant), + await _progress_percent( + adopted.repo_id, adopted.variant, adopted.expected_bytes, hf_token + ), + ) + if state == "error" or adopted.error: + error = error or adopted.error + # Surface once, then free the slot so a retry can start over. + _release(adopted) + return AutoDownloadRefusal( + status = 502, + code = "model_download_failed", + message = f"Downloading '{requested_model}' failed: {error or 'unknown error'}", + ) + # complete/idle/cancelled: the watcher is about to free the slot, so retry once more. + return _downloading_refusal( + _public_label(adopted.repo_id, adopted.variant), + 100.0 if state == "complete" else None, + ) + + try: + return await _admit_and_start( + repo_id, wanted_variant, requested_model, hf_token, provisional, require_vision + ) + except BaseException: + # Not `except Exception`: a cancel mid-probe would otherwise wedge the provisional slot. + _release(provisional) + raise + + +async def _admit_and_start( + repo_id: str, + wanted_variant: Optional[str], + requested_model: str, + hf_token: Optional[str], + active: _Active, + require_vision: bool = False, +) -> Optional[AutoDownloadRefusal]: + from hub.utils.hf_errors import hf_error_status + + def _probe(): + from huggingface_hub import HfApi + return HfApi(token = _hub_token(hf_token)).model_info( + repo_id, files_metadata = True, timeout = _MODEL_INFO_TIMEOUT_S + ) + + try: + info = await asyncio.to_thread(_probe) + except Exception as exc: + _release(active) + status = hf_error_status(exc) + if status == 401: + return AutoDownloadRefusal( + status = 401, + code = "model_access_denied", + message = ( + f"Hugging Face rejected the token sent for '{repo_id}'. Replace the " + "X-Unsloth-HF-Token header with a valid token; retrying will not help." + ), + ) + if status == 403: + return _gated_refusal(repo_id) + if status == 404: + _mark_not_servable(repo_id, hf_token) + # Unknown to the Hub reads as a foreign label; only an explicit quant makes it ours. + if not looks_like_quant(wanted_variant): + return None + # A private repo reads as absent without a token; don't confirm either way. + return AutoDownloadRefusal( + status = 404, + code = "model_not_found", + message = ( + f"'{repo_id}' was not found on Hugging Face, or is not accessible. " + "If it is private, send a token in the X-Unsloth-HF-Token header." + ), + ) + logger.warning("auto-download: Hub lookup failed for %r: %s", repo_id, exc) + return AutoDownloadRefusal( + status = 503, + code = "model_lookup_failed", + message = f"Could not reach Hugging Face to look up '{repo_id}'. Retry shortly.", + retry_after = _RETRY_AFTER_S, + ) + + # Inconclusive on timeout: the download's own auth is the real gate. + if getattr(info, "gated", False) and await _bounded_probe( + _auth_denied, repo_id, hf_token, timeout = _MODEL_INFO_TIMEOUT_S, default = False + ): + # Metadata for a gated repo is not file access; unchecked, the config read below lies. + _release(active) + return _gated_refusal(repo_id) + + variants = _gguf_variants(getattr(info, "siblings", None)) + if not variants: + _release(active) + _mark_not_servable(repo_id, hf_token) + if not looks_like_quant(wanted_variant): + return None + return AutoDownloadRefusal( + status = 400, + code = "model_not_supported", + message = ( + f"'{repo_id}' has no GGUF weights. Automatic download serves GGUF only; " + "load other formats from Unsloth Studio." + ), + ) + + # trust_remote_code gate: _config_has_auto_map is tri-state, so refuse on True and on None. + from utils.security.consent import _config_has_auto_map + + # _hub_token, not the raw token: None lets huggingface_hub fall back to a cached + # server login, so a caller-named repo would be probed with this server's identity. + # Same rule as the metadata probe and the worker. + # None on timeout, which refuses: an unchecked repo is not a cleared one. + has_auto_map = await _bounded_probe( + _config_has_auto_map, + repo_id, + _hub_token(hf_token), + timeout = _CODE_PROBE_TIMEOUT_S, + default = None, + ) + if has_auto_map is not False: + _release(active) + unknown = has_auto_map is None + return AutoDownloadRefusal( + status = 403, + code = "remote_code_consent_required", + message = ( + f"'{repo_id}' " + + ( + "could not be checked for custom code" + if unknown + else "ships custom code that runs on load" + ) + + ". Load it once in Unsloth Studio to review and approve it, then retry." + ), + ) + + variant = _match_variant(wanted_variant, variants) + if variant is None: + _release(active) + listed = sorted(variants) + shown = ", ".join(listed[:_MAX_LISTED_VARIANTS]) + extra = len(listed) - _MAX_LISTED_VARIANTS + return AutoDownloadRefusal( + status = 404, + code = "model_not_found", + message = ( + f"'{repo_id}' has no quant '{wanted_variant}'. Available quants: " + f"{shown}{f' and {extra} more' if extra > 0 else ''}." + ), + ) + + expected_bytes = variants[variant] + from hub.utils.gguf_plan import build_gguf_variant_plans + + plan = build_gguf_variant_plans(list(getattr(info, "siblings", None) or [])).get( + variant.lower() + ) + if require_vision and not (plan and plan.mmproj_filenames): + _release(active) + return AutoDownloadRefusal( + status = 400, + code = "invalid_value", + message = ( + f"'{_public_label(repo_id, variant)}' ships no mmproj companion, so it " + "cannot answer the image or audio input in this request. It was not " + "downloaded." + ), + ) + + need_bytes = _remaining_bytes(repo_id, plan, expected_bytes) + fits, free = _enough_disk(need_bytes) + if not fits: + _release(active) + return AutoDownloadRefusal( + status = 507, + code = "insufficient_disk_space", + message = ( + f"'{_public_label(repo_id, variant)}' needs {_gb(need_bytes)} plus " + f"{_gb(_DISK_RESERVE_BYTES)} headroom, but only {_gb(free)} is free." + ), + ) + + return await _dispatch(repo_id, variant, expected_bytes, requested_model, hf_token, active) + + +def preferred_quant(labels) -> Optional[str]: + """The quant a plain load would pick from *labels*, or None. + + The one ranking for "which quant did they mean": local resolution, remote + admission and what /v1/models advertises all have to agree, or a bare id + means a different quant depending on which of them answered it. + """ + from utils.models.model_config import _pick_best_gguf + + # _pick_best_gguf ranks filenames and matches upper-case tokens, so feed "