Studio: tighten the comments added by the OpenAI model-admission work (#7501)

Comment-only follow-up to #7454. That change carried 523 comment lines, many of
them three and four line preambles where one line says the same thing. This
collapses them and drops the ones restating what the code already says, for a
net 77 lines.

Scope is limited to comments #7454 itself introduced. The files it touched hold
about 3,761 comments in total; the rest predate it and are untouched, verified
by checking that every removed line is one that commit added.

Nothing that records why a non-obvious decision was made was dropped, only
compressed. Still stated: the normcase-before-versus-after Windows separator
trap, the innermost-indexed-model rule for nested directories, an HTTPException
being a decision rather than a failure to decide, that only an explicit False is
anonymous to huggingface_hub while None borrows the server owner's login, the
fail-closed tri-state custom-code gate, and the regressions each test was
written for.

Code is provably unchanged: comment_tools.py check reports 17/17 files
comments-only. Backend CI command 10337 passed, 0 failed. tsc -b clean.
This commit is contained in:
Daniel Han 2026-07-27 05:59:03 -07:00 committed by GitHub
commit 06829c2627
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 306 additions and 383 deletions

View file

@ -164,14 +164,13 @@ async def get_current_subject_allow_password_change(
)
# The literal the examples ship with; pasting one unedited is likelier than a revoked key.
# The literal the examples ship with; pasted unedited more often than a revoked key.
API_KEY_PLACEHOLDER = f"{API_KEY_PREFIX}YOUR_KEY"
def _invalid_api_key_detail(token: str) -> str:
"""Why the key failed. Only the unedited example placeholder is called out;
every real key still gets one indistinguishable message, so this reveals
nothing about which keys exist."""
"""Why the key failed. Only the example placeholder is called out; every real
key gets one indistinguishable message, so this leaks no key existence."""
if token == API_KEY_PLACEHOLDER:
return (
"This is the placeholder key from the example. Create an API key in "

View file

@ -148,10 +148,9 @@ class ApiMonitor:
) -> str:
"""Record a model load/unload alongside the request traffic that caused it.
``running=True`` opens the row (a load in progress) and the caller closes
it with the usual :meth:`finish` / :meth:`fail`; an unload is terminal on
arrival. Rows are shared, so every subject sees them, and share the same
retention budget as requests.
``running=True`` opens the row for the caller to close with :meth:`finish` /
:meth:`fail`; an unload is terminal on arrival. Rows are shared (visible to
every subject) and share the request retention budget.
"""
now = time.time()
entry = ApiMonitorEntry(
@ -177,8 +176,8 @@ class ApiMonitor:
return entry.id
def relabel(self, entry_id: Optional[str], model: str) -> None:
"""Rename an open lifecycle row once the load resolves its real id (the
caller only has the load path up front, which may be an HF snapshot dir)."""
"""Rename an open lifecycle row once the load resolves its real id: up front
the caller only has the load path, which may be an HF snapshot dir."""
if not entry_id or not model:
return
with self._lock:
@ -198,8 +197,7 @@ class ApiMonitor:
entry.updated_at = time.time()
def discard(self, entry_id: Optional[str]) -> None:
"""Drop a row that turned out not to be an event (a load that was already
satisfied, so nothing was actually loaded)."""
"""Drop a row that turned out not to be an event (an already-satisfied load)."""
if not entry_id:
return
with self._lock:
@ -293,9 +291,8 @@ class ApiMonitor:
self._trim_terminal_locked()
def fail_open(self, entry_id: Optional[str], error: str) -> None:
"""Fail only a still-open row. Unlike :meth:`fail` this never touches an
entry that already finished, so a catch-all in a ``finally`` cannot stamp
an error onto a request that in fact succeeded."""
"""Fail only a still-open row: unlike :meth:`fail`, a catch-all in a
``finally`` cannot stamp an error onto a request that already succeeded."""
if not entry_id:
return
with self._lock:

View file

@ -346,8 +346,8 @@ def _loaded_identity(backend):
def _note_idle_unload_event(freed) -> None:
"""Record an idle auto-unload in the API monitor, using the advertised repo id
from the stash so the row never shows the on-disk load path. Best-effort."""
"""Monitor row for an idle auto-unload. Best-effort; uses the stash's
advertised repo id so the row never shows the on-disk load path."""
try:
from core.inference.api_monitor import api_monitor
from core.inference.model_ids import public_model_id

View file

@ -36,9 +36,8 @@ _lock = threading.Lock()
_scan: tuple[float, dict[str, _LocalGgufEntry]] = (0.0, {})
# Not _lock: that is held for the whole scan, so the request path would wait on it.
_warm_lock = threading.Lock()
# Repos that finished downloading but are not in the published index yet. The
# retained index covers what was already known; nothing covers the one that just
# landed until the next scan, and the request path must not call it absent.
# Repos that finished downloading but are not in the published index yet: nothing
# else covers them until the next scan, and the request path must not call them absent.
_just_downloaded: set[str] = set()
_warming = False
_last_scan_s = 0.0
@ -115,10 +114,9 @@ def _local_gguf_entry(loader_id: str, info) -> Optional[_LocalGgufEntry]:
quants = tuple(v.quant for v in variants if getattr(v, "quant", None))
if not quants:
return None
# That call orders by descending size, so the head is the biggest quant,
# often F16. A bare id means whichever quant a plain load would take, so put
# that first: everything downstream reads [0], and answering with the
# largest can evict a working model and then OOM starting it.
# That call orders by descending size, so the head is the biggest quant (often
# F16). Downstream reads [0], and a bare id must mean whichever quant a plain
# load would take: answering with the largest can evict a model and then OOM.
from core.inference.openai_auto_download import preferred_quant
best = preferred_quant(quants)
@ -132,9 +130,8 @@ def _local_gguf_entry(loader_id: str, info) -> Optional[_LocalGgufEntry]:
def local_gguf_quants(info) -> Optional[tuple[str, ...]]:
"""On-disk quant labels for *info*, or None when it is not a servable local
GGUF. Read from the files, not ``info.model_format``: the HF-cache scanner
leaves model_format unset for GGUF snapshots, so a model_format filter would
drop every cached GGUF. Lets /v1/models advertise exactly what /v1 can serve,
and which quant to name, from a single scan."""
leaves that unset for GGUF snapshots, so filtering on it drops every cached
GGUF. One scan tells /v1/models what it can serve and which quant to name."""
from pathlib import Path
path = getattr(info, "path", None)
@ -330,14 +327,14 @@ def recently_downloaded(repo_id: str) -> bool:
def invalidate_index() -> None:
"""Mark the cached scan stale so the next resolve sees a just-finished
download, rather than waiting out the TTL.
"""Mark the cached scan stale so the next resolve sees a just-finished download
instead of waiting out the TTL.
Keeps the entries. Callers on the request path read this cache without
scanning, so emptying it would leave them with no evidence about any local
model until the rebuild lands, and a bare request for one of them would be
answered by whatever is resident. Only a completed download invalidates, and
that only ever adds models, so the retained entries stay true.
Keeps the entries: the request path reads this cache without scanning, so
emptying it would leave it with no evidence about any local model until the
rebuild lands, and a bare request for one would be answered by whatever is
resident. Only a completed download invalidates, and that only adds, so the
retained entries stay true.
"""
global _scan
with _lock:
@ -366,9 +363,9 @@ def _index() -> dict[str, _LocalGgufEntry]:
def index_is_built() -> bool:
"""Whether a scan has ever completed, freshness aside.
Lock-free on purpose: ``_lock`` is held for the whole scan, so taking it here
would park the request path on the very scan it is trying to stay off. Reading
``_scan[0]`` is safe because ``_scan`` is only ever rebound, never mutated.
Lock-free on purpose: ``_lock`` is held for the whole scan, so taking it would
park the request path on the scan it is trying to stay off. Safe because
``_scan`` is only ever rebound, never mutated.
"""
return bool(_scan[0])
@ -376,13 +373,10 @@ def index_is_built() -> bool:
def warm_index_soon() -> None:
"""(Re)build the index off the request path when it is missing or past its TTL.
Callers that cannot afford the scan use this plus ``allow_scan=False``, so this
is the only thing that ever refreshes the index for them. It has to cover a
stale index and not just an absent one: a model downloaded through the Hub UI
or dropped into a scan folder has no invalidation hook, and would otherwise stay
invisible to those callers for the life of the process.
Never touches ``_lock``, which the scan holds throughout, and never blocks.
The only refresh for callers using ``allow_scan=False``. Covers a stale index,
not just an absent one: a model downloaded through the Hub UI or dropped into a
scan folder has no invalidation hook and would otherwise stay invisible to them
for the life of the process. Never blocks, and never touches ``_lock``.
"""
global _warming
if time.monotonic() - _scan[0] < max(_CACHE_TTL_S, _last_scan_s * _WARM_DUTY):
@ -419,11 +413,10 @@ def resolve_local_gguf(
off and resolves only when that quant is on disk, unless it names no quant at
all (an Ollama-style ":latest"), which means the repo.
``allow_scan=False`` answers from the last built index and never rebuilds,
for callers on the request path: the scan walks several model dirs and HF
caches, takes seconds on a large install, and holds a lock every other
caller queues behind. A stale answer is fine there, since what is on disk
barely moves and a finished download calls :func:`invalidate_index`.
``allow_scan=False`` answers from the last built index and never rebuilds, for
the request path: the scan walks several model dirs and HF caches, takes seconds
on a large install, and holds a lock everyone queues behind. Stale is fine there,
since disk barely moves and a finished download calls :func:`invalidate_index`.
"""
if not isinstance(requested, str) or not requested.strip():
return None
@ -466,10 +459,9 @@ def describe_local_miss(requested: str) -> tuple[str, tuple[str, ...]]:
"""Why :func:`resolve_local_gguf` missed, so an error can say "wrong quant"
instead of "no such model".
``(MISS_VARIANT_NOT_FOUND, <local quants>)`` when the repo is downloaded but
the requested ``:VARIANT`` is not, else ``(MISS_MODEL_NOT_FOUND, ())``. Splits
the name like the resolver so the two agree. Fail-safe: a scan failure reports
the generic miss rather than raising into the handler.
``(MISS_VARIANT_NOT_FOUND, <local quants>)`` when the repo is downloaded but the
requested ``:VARIANT`` is not, else ``(MISS_MODEL_NOT_FOUND, ())``. Fail-safe: a
scan failure reports the generic miss rather than raising into the handler.
"""
if not isinstance(requested, str) or not requested.strip():
return MISS_MODEL_NOT_FOUND, ()

View file

@ -42,9 +42,8 @@ def _looks_like_path(identifier: str) -> bool:
def hf_cache_repo_id(path: Optional[str]) -> Optional[str]:
"""``.../models--org--name/snapshots/<sha>`` -> ``org/name``, else None.
A model loaded straight out of the HF cache has a snapshot directory as its
identifier, whose basename is a commit hash. Recover the repo id so callers
show ``unsloth/gemma-4-31B-it-GGUF`` rather than ``c1ac76e99d55...``.
A model loaded from the HF cache is identified by its snapshot dir, whose
basename is a commit hash; recover the repo id so callers don't show that.
"""
if not path:
return None

View file

@ -5,22 +5,21 @@
Auto-switch only loads models already on disk. With
``openai_api_auto_download_model`` on, a miss that looks like a real Hub repo is
downloaded in the background instead of erroring, and the request is told to
retry rather than being held open: a quant is routinely tens of GB, far longer
than any client (or the Cloudflare edge on ``--secure``) will wait, and the
inference lifecycle gate must not be held meanwhile. The resident model keeps
serving throughout, and the retry that lands after the download is served by the
new model through the ordinary auto-switch path.
fetched in the background and the request is told to retry rather than held
open: a quant is routinely tens of GB, far longer than any client (or the
Cloudflare edge on ``--secure``) will wait, and the inference lifecycle gate must
not be held meanwhile. The resident model keeps serving, and the retry that lands
after the download goes through the ordinary auto-switch path.
Admission is deliberately narrow, since a request only needs an API key:
- ``namespace/name`` only, and only when the Hub confirms it is a GGUF repo.
``gpt-4`` and ``anthropic/claude-3.5-sonnet`` alike fall through to the
resident model as before: a namespace is not evidence of intent, since LiteLLM
and OpenRouter address every provider that way.
- GGUF repos only, decided from the remote file list, not the repo name. GGUF
runs under llama.cpp, which never imports repo Python.
- Anything declaring ``auto_map`` is refused, so ``trust_remote_code`` can only
ever be granted deliberately in the UI, never by an API call.
- ``namespace/name`` only, and only when the Hub confirms GGUF weights. A
namespace is not evidence of intent (LiteLLM and OpenRouter address every
provider that way), so ``gpt-4`` and ``anthropic/claude-3.5-sonnet`` alike
fall through to the resident model as before.
- GGUF only, decided from the remote file list, not the repo name: GGUF runs
under llama.cpp, which never imports repo Python.
- ``auto_map`` is refused, so ``trust_remote_code`` is only ever granted
deliberately in the UI, never by an API call.
- One download at a time, so a key holder cannot fan out fetches.
"""
@ -39,30 +38,29 @@ logger = get_logger(__name__)
# Keep the Hub probe short so a slow Hub can't stall the request path.
_MODEL_INFO_TIMEOUT_S = 8.0
# auth_check and hf_hub_download take no timeout of their own, and both run while the
# provisional slot is held, so an unresponsive Hub would pin the single flight and stall
# the request long past the metadata budget. The code probe fetches up to three small
# configs, so it gets more room than the single auth call.
# auth_check and hf_hub_download take no timeout of their own and run while the
# provisional slot is held, so an unresponsive Hub would pin the single flight. The
# code probe fetches up to three configs, so it gets more room than the auth call.
_CODE_PROBE_TIMEOUT_S = 20.0
# Headroom left free after the download, so filling the disk can't wedge the box.
_DISK_RESERVE_BYTES = 5 * 1024**3
_WATCH_POLL_S = 2.0
# A stalled watcher must not pin the single-flight slot forever.
_MAX_WATCH_S = 24 * 60 * 60
# Past the watch window the row is already resolved, so poll only to see whether
# the worker is still alive and still owns the slot.
# Past the watch window the row is resolved, so poll only to see whether the
# worker is still alive and still owns the slot.
_TIMED_OUT_POLL_S = 60.0
_RETRY_AFTER_S = 30
# Long enough for a client honouring Retry-After to come back and be told, short
# enough that a client that never returns cannot hold the slot.
# enough that one that never returns cannot hold the slot.
_FAILED_HOLD_S = 3 * _RETRY_AFTER_S
_MAX_LISTED_VARIANTS = 8
@dataclass(frozen = True)
class AutoDownloadRefusal:
"""Why this request cannot be served yet. The route turns it into an
HTTPException with the surface's own error envelope."""
"""Why this request cannot be served yet; the route raises it in the
surface's own error envelope."""
status: int
code: str
@ -78,9 +76,8 @@ class _Active:
expected_bytes: int = 0
monitor_id: Optional[str] = None
started_at: float = 0.0
# Set when the worker failed. The slot is kept until a retry surfaces it, since
# the advertised retry interval is far longer than the watcher's poll and the
# client would otherwise just restart the same failing download.
# Set when the worker failed. Held until a retry surfaces it: Retry-After is far
# longer than the watcher poll, so the client would restart the same failing download.
error: Optional[str] = None
failed_at: float = 0.0
@ -103,9 +100,8 @@ def split_model_ref(requested: str) -> tuple[str, Optional[str]]:
"""``org/repo:QUANT`` -> ``("org/repo", "QUANT")``; no suffix -> variant None.
Splits on the last colon. A slash-bearing suffix is only a variant when a real
Hub repo precedes it: an unrecognized GGUF below a subdirectory keys on its path
("build/llama-13b", which is_valid_gguf_variant allows and the catalog advertises),
while "C:/models/x.gguf" leaves a drive letter that is no repo id at all.
Hub repo precedes it: "build/llama-13b" is a subdirectory GGUF key the catalog
advertises, while "C:/models/x.gguf" leaves a drive letter that is no repo id.
"""
text = (requested or "").strip()
base, sep, suffix = text.rpartition(":")
@ -122,9 +118,9 @@ def split_model_ref(requested: str) -> tuple[str, Optional[str]]:
def is_downloadable_ref(requested: str) -> bool:
"""Whether *requested* is shaped like a Hub repo we may fetch.
Requires an explicit namespace. That keeps ``gpt-4`` and other foreign ids
falling through untouched, and avoids the bare-name ``unsloth/`` prefixing in
ModelConfig.from_identifier turning an unrelated label into a real repo.
Requires an explicit namespace: keeps ``gpt-4`` and other foreign ids falling
through, and stops ModelConfig.from_identifier's bare-name ``unsloth/``
prefixing from turning an unrelated label into a real repo.
"""
from hub.utils.paths import is_valid_repo_id
@ -140,9 +136,9 @@ def is_downloadable_ref(requested: str) -> bool:
def looks_like_quant(variant: Optional[str]) -> bool:
"""Whether a ``:suffix`` names a GGUF quant rather than a foreign tag.
``vendor/model`` is how LiteLLM and OpenRouter address every provider, and
``name:latest`` is how Ollama tags one, so neither a namespace nor a colon
proves a request was meant for this server. A real quant label does.
Neither a namespace nor a colon proves a request was meant for this server
(``vendor/model`` is LiteLLM/OpenRouter, ``name:latest`` is Ollama). A real
quant label does.
"""
import re
@ -156,20 +152,16 @@ def looks_like_quant(variant: Optional[str]) -> bool:
def _hub_token(hf_token: Optional[str]):
"""The caller's token, or an explicit False.
None makes huggingface_hub fall back to a cached login, which here would be
the server owner's. False is what actually means anonymous.
"""
"""The caller's token, or an explicit False. None makes huggingface_hub fall
back to a cached login (here the server owner's); only False is anonymous."""
return hf_token or False
def _servable_key(repo_id: str, hf_token: Optional[str]) -> str:
"""Cache key, per credential.
The Hub answers 404 for a private repo the caller cannot see, so a verdict
reached without a token says nothing about a caller who has one. Keyed on a
digest so the token itself is never held here.
The Hub 404s a private repo the caller cannot see, so a tokenless verdict says
nothing about a caller who has one. Digested, so no token is held here.
"""
import hashlib
@ -212,8 +204,7 @@ async def _bounded_probe(fn, *args, timeout: float, default):
"""Run a blocking Hub probe off the loop, bounding only the wait.
The thread is left to finish (a blocking socket read cannot be cancelled); the
caller stops waiting and takes *default*, which each call site chooses so that a
timeout errs the safe way.
caller takes *default*, chosen per call site so a timeout errs the safe way.
"""
try:
return await asyncio.wait_for(asyncio.to_thread(fn, *args), timeout)
@ -239,10 +230,9 @@ def _gguf_variants(siblings) -> dict[str, int]:
"""Quant label -> bytes the download will actually fetch.
Mirrors list_gguf_variants for the selectable labels: companions (mmproj/MTP)
and big-endian builds are not quants of their own, and sharded quants sum
across their shards. The byte total comes from the download plan, which folds
the companions back into every quant, so the disk reserve is measured against
what the worker fetches rather than the main files alone.
and big-endian builds are not quants, and sharded quants sum across shards.
Bytes come from the download plan, which folds companions back into every
quant, so the disk reserve is measured against what the worker fetches.
"""
from hub.utils.gguf import extract_quant_label as canonical_quant_label
from hub.utils.gguf_plan import build_gguf_variant_plans
@ -262,10 +252,9 @@ def _gguf_variants(siblings) -> dict[str, int]:
continue
quant = _extract_quant_label(name)
if not looks_like_quant(quant):
# With no recognized quant token the two extractors part ways: this one
# takes the last hyphenated segment ("7b" of llama-7b) while the plan and
# the worker key the whole stem. Advertising ours dispatches a variant the
# worker cannot resolve, so take theirs for the unrecognized case only.
# With no recognized quant token the extractors part ways: this one takes
# the last hyphenated segment ("7b" of llama-7b) while the plan and worker
# key the whole stem, so advertising ours dispatches an unresolvable variant.
quant = canonical_quant_label(name) or quant
if _is_mmproj(name) or _is_mtp_drafter(name) or _is_big_endian_gguf_path(name, quant):
continue
@ -343,11 +332,9 @@ async def _progress_percent(
def _release(active: Optional[_Active]) -> None:
"""Free the single-flight slot, but only while *active* still owns it.
Keying the release on ``repo_id`` alone let a stale operation clear a newer
one for the same repo: variant A errors, an adopting request frees the slot,
a retry starts variant B, and A's watcher then matches on the repo and clears
B on its way out -- admitting a second repository download alongside B.
Identity ties every release to the operation that actually took the slot.
Keying on ``repo_id`` alone let a stale operation clear a newer one: variant A
errors, an adopting request frees the slot, a retry starts B, then A's watcher
matches the repo and clears B, admitting a second download alongside it.
"""
global _active
if active is None:
@ -371,10 +358,9 @@ async def _watch(active: _Active, hf_token: Optional[str]) -> None:
state, error = await _job_state(active.repo_id, active.variant)
if state in ("running", "cancelling", "unknown"):
if timed_out:
# A worker still running still owns the slot: releasing it on the
# clock alone would admit a second multi-GB download alongside it.
# "unknown" cannot confirm it is alive, so stop holding it then,
# or a broken probe would wedge auto-download for good.
# A running worker still owns the slot: releasing on the clock alone
# would admit a second multi-GB download beside it. "unknown" cannot
# confirm it is alive, so release then, or a broken probe wedges us.
if state == "unknown":
return
continue
@ -396,16 +382,16 @@ async def _watch(active: _Active, hf_token: Optional[str]) -> None:
return
if state == "complete":
# No invalidate here: finalize_worker_exit already dropped the cache and
# started the warm, and a second one would mark that fresh scan stale and
# push a synchronous rescan onto the client's retry.
# warmed it; a second would mark that fresh scan stale and push a
# synchronous rescan onto the client's retry.
api_monitor.finish(active.monitor_id, status = "completed")
elif state == "idle":
# The job vanished without a terminal state (worker killed).
api_monitor.fail_open(active.monitor_id, "Download did not complete")
else:
api_monitor.fail_open(active.monitor_id, error or f"Download {state}")
# Keep the slot so the next retry is told it failed rather than
# silently starting the same download again.
# Keep the slot so the next retry is told it failed instead of
# silently restarting the same download.
active.error = error or f"Download {state}"
active.failed_at = time.monotonic()
return
@ -434,9 +420,8 @@ async def _is_downloadable_model(repo_id: str, hf_token: Optional[str]) -> bool:
"""Whether the Hub has this repo with GGUF weights we could fetch.
Only asked while another download holds the slot, to tell a second download
apart from an ordinary foreign label. Any failure answers False: falling
through to the resident model is what such a label does anyway, and refusing
it would strand normal traffic for the length of the download.
apart from an ordinary foreign label. Any failure answers False: refusing
would strand normal traffic for the length of the download.
"""
if _is_not_servable(repo_id, hf_token):
return False
@ -450,9 +435,8 @@ async def _is_downloadable_model(repo_id: str, hf_token: Optional[str]) -> bool:
except Exception:
return False
# The same filter admission uses, not a bare .gguf test: mmproj, MTP drafters and
# big-endian builds are companions rather than quants, so a repo holding only those
# is not downloadable here either. Answering otherwise would hold an ordinary
# foreign label at model_download_busy for the length of an unrelated download.
# big-endian builds are companions, not quants. Answering otherwise would hold an
# ordinary foreign label at model_download_busy for an unrelated download.
servable = bool(_gguf_variants(getattr(info, "siblings", None)))
if not servable:
_mark_not_servable(repo_id, hf_token)
@ -470,9 +454,9 @@ async def maybe_auto_download(
Returns None when the request should carry on unchanged, or a refusal the
caller must raise. Only called after the local resolver has already missed.
``require_vision`` refuses a target with no mmproj companion rather than
spending gigabytes on weights that cannot answer the request that asked for
them; the local capability guard only ever sees an already-downloaded model.
``require_vision`` refuses a target with no mmproj companion rather than spend
gigabytes on weights that cannot answer the request; the local capability guard
only ever sees an already-downloaded model.
"""
global _active
@ -502,9 +486,8 @@ async def maybe_auto_download(
if busy is not None:
# Refusing before the probe blocks ordinary drop-in traffic: a namespaced label
# that is not a downloadable GGUF repo (LiteLLM/OpenRouter style) would be told
# to wait out a multi-hour download instead of falling through to the resident
# model. Only a label that could itself be downloaded is a second download.
# that is no downloadable GGUF repo (LiteLLM/OpenRouter style) would be told to
# wait out a multi-hour download. Only a downloadable label is a 2nd download.
if not await _is_downloadable_model(repo_id, hf_token):
return None
return AutoDownloadRefusal(
@ -636,8 +619,7 @@ async def _admit_and_start(
# _hub_token, not the raw token: None lets huggingface_hub fall back to a cached
# server login, so a caller-named repo would be probed with this server's identity.
# Same rule as the metadata probe and the worker.
# None on timeout, which refuses: an unchecked repo is not a cleared one.
# Defaults to None on timeout, which refuses: unchecked is not cleared.
has_auto_map = await _bounded_probe(
_config_has_auto_map,
repo_id,
@ -715,8 +697,8 @@ def preferred_quant(labels) -> Optional[str]:
"""The quant a plain load would pick from *labels*, or None.
The one ranking for "which quant did they mean": local resolution, remote
admission and what /v1/models advertises all have to agree, or a bare id
means a different quant depending on which of them answered it.
admission and /v1/models must agree, or a bare id means a different quant
depending on which of them answered it.
"""
from utils.models.model_config import _pick_best_gguf
@ -732,15 +714,14 @@ def _match_variant(wanted: Optional[str], variants: dict[str, int]) -> Optional[
"""Resolve the requested quant against what the repo actually has.
An explicit quant matches case-insensitively and must exist: never quietly
substitute another, unlike the loader's low-disk fallback. A bare repo id, or
an Ollama-style tag that names no quant at all (":latest", ":8b"), uses the
same preference order as a manual load, matching what the local resolver does
with the same tag.
substitute another, unlike the loader's low-disk fallback. A bare repo id, or a
tag that names no quant (":latest", ":8b"), uses the same preference order as a
manual load, matching what the local resolver does with the same tag.
"""
if wanted:
# Exact first, whatever shape it is: a repo of generically named GGUFs has
# real variants like "llama-13b" that are valid worker keys but do not look
# like quants, and defaulting past one would fetch a model nobody asked for.
# Exact first, whatever shape: a repo of generically named GGUFs has real
# variants like "llama-13b" that are valid worker keys but not quant-shaped,
# and defaulting past one would fetch a model nobody asked for.
lowered = {name.lower(): name for name in variants}
exact = lowered.get(wanted.strip().lower())
if exact is not None or looks_like_quant(wanted):

View file

@ -241,11 +241,10 @@ def finalize_worker_exit(
state = classify_exit(rc, cancel_requested = cancel_requested)
if state == "complete":
registry.set_job(key, "complete")
# Where /v1 learns a new model exists: its resolver answers the request path
# from a cached scan with no watcher, and would otherwise report the model
# absent and let the request be served by whatever is resident. Models only,
# since datasets share this path and noting one as a local model would refuse
# a bare request naming that id instead of letting a foreign id fall through.
# Where /v1 learns a new model exists: its resolver answers from a cached scan
# with no watcher, so it would report the model absent and serve whatever is
# resident. Models only: noting a dataset id as a local model would refuse a
# bare request naming it instead of letting a foreign id fall through.
if repo_type == "model":
try:
from core.inference.local_model_resolver import (
@ -256,8 +255,8 @@ def finalize_worker_exit(
note_downloaded(repo_id)
invalidate_index()
# Rebuild here rather than on the first request that needs it, so the
# new model resolves without a scan on the request path.
# Rebuild here, not on the first request, to keep the scan off the
# request path.
warm_index_soon()
except Exception:
pass

View file

@ -114,8 +114,8 @@ async def download_model_response(
):
"""Start a background download for a HuggingFace model.
``allow_ambient_token=False`` keeps the worker anonymous when the caller
supplied no token, for repos named over the API rather than chosen here.
``allow_ambient_token=False`` keeps the worker anonymous when the caller sent
no token, for repos named over the API rather than chosen here.
"""
repo_id = body.repo_id.strip()
if not _is_valid_repo_id(repo_id):

View file

@ -3118,8 +3118,8 @@ def _lifecycle_model_label(model: Optional[str], variant: Optional[str] = None)
def _close_load_event(
entry_id: Optional[str], model: Optional[str], variant: Optional[str]
) -> None:
"""Close a monitor load row, relabelled with the id the load actually resolved
(the row opened on the request's model_path, which may be an HF snapshot dir)."""
"""Close a monitor load row, relabelled with the id the load resolved: the row
opened on the request's model_path, which may be an HF snapshot dir."""
api_monitor.relabel(entry_id, _lifecycle_model_label(model, variant))
api_monitor.finish(entry_id)
@ -3127,8 +3127,8 @@ def _close_load_event(
def _monitor_active_model() -> Optional[str]:
"""The loaded model as a client-facing id, quant included when known.
Cleaned like /v1/models: this is rendered in the settings UI and served over
the public --secure tunnel, so it must never be the on-disk load path.
Cleaned like /v1/models: rendered in the settings UI and served over the public
--secure tunnel, so it must never be the on-disk load path.
"""
llama_backend = get_llama_cpp_backend()
if getattr(llama_backend, "is_loaded", False):
@ -3540,8 +3540,8 @@ _DISABLE_OPENAI_AUTO_SWITCH_SCOPE_KEY = "_unsloth_disable_openai_auto_switch"
# only restore an idle-freed model, never run the resolver (so a downloaded GGUF
# literally named "default" can't be swapped to). The NUL keeps it off any index.
_RELOAD_ONLY_MODEL = "\x00reload-only"
# One cold scan is worth paying to avoid answering a named model with another; a
# pathological install must not hang the request behind it forever.
# One cold scan is worth paying to avoid answering a named model with another,
# bounded so a pathological install cannot hang the request behind it.
_COLD_INDEX_WAIT_S = 10.0
@ -3659,9 +3659,9 @@ def _format_available_models(ids: list[str]) -> str:
async def _unavailable_model_message(requested_model: str) -> str:
"""Why a named model can't serve this request, and what can.
Auto-switch only loads already-downloaded GGUFs, so a request naming a real
model usually fails because it is not on this machine. Pointing the caller at
/inference/load cannot fix that; say what is actually wrong.
Auto-switch only loads downloaded GGUFs, so a request naming a real model
usually fails because it is not on this machine, which /inference/load cannot
fix; say what is actually wrong.
"""
from core.inference.local_model_resolver import (
MISS_VARIANT_NOT_FOUND,
@ -3694,10 +3694,10 @@ async def _no_model_loaded_error(
):
"""``(status, detail)`` for the /v1 sites that fail because nothing is loaded.
Changes only the case the generic text describes wrongly: auto-switch on, a
model named, and that name resolves to nothing local, so the switch silently
did nothing. That becomes a 404 model_not_found. Toggle off or no model named
keeps ``status`` and the :func:`_no_model_loaded_detail` text verbatim.
Changes only the case the generic text gets wrong (auto-switch on, a model
named, that name resolving to nothing local, so the switch silently did
nothing) into a 404 model_not_found. Everything else keeps ``status`` and the
:func:`_no_model_loaded_detail` text verbatim.
"""
from utils.openai_auto_switch_settings import get_openai_auto_switch_enabled
from core.inference.local_model_resolver import resolve_local_gguf
@ -3739,10 +3739,9 @@ async def _no_model_loaded_error(
def _auto_download_hf_token(fastapi_request: Optional[Request]) -> Optional[str]:
"""The token to fetch with: only one the caller sent themselves.
Never the server's ambient token. The repo here is named by whoever holds an
API key, so borrowing the owner's Hub identity would let that key pull the
owner's private repos and publish them in /v1/models for every other key.
The OpenAI bearer key is never used as an HF token either.
Never the server's ambient token, and never the OpenAI bearer key. The repo is
named by whoever holds an API key, so borrowing the owner's Hub identity would
let that key pull the owner's private repos and publish them in /v1/models.
"""
from hub.dependencies import HUB_HF_TOKEN_HEADER, HUB_HF_TOKEN_MAX_LENGTH
@ -3763,10 +3762,9 @@ async def _maybe_auto_download_model(
) -> None:
"""Opt-in: start fetching a named GGUF this server doesn't have.
Raises to stop the request when the model is downloading or cannot be
fetched. Off by default, and it never fires on a name that isn't shaped like
a Hub repo, so an unknown id like "gpt-4" still falls through to the resident
model as before.
Raises to stop the request while the model is downloading or cannot be fetched.
Off by default, and never fires on a name not shaped like a Hub repo, so an
unknown id like "gpt-4" still falls through to the resident model.
"""
from utils.openai_auto_switch_settings import get_openai_auto_download_enabled
from core.inference.openai_auto_download import is_downloadable_ref, maybe_auto_download
@ -3812,8 +3810,8 @@ async def _maybe_auto_download_model(
def _loaded_satisfies(requested: str) -> bool:
"""Whether what is serving right now actually answers to *requested*.
A bare ``org/model`` is satisfied by any loaded quant of that repo; an
explicit ``:QUANT`` must match the loaded one.
A bare ``org/model`` is satisfied by any loaded quant of that repo; an explicit
``:QUANT`` must match the loaded one.
"""
from core.inference.openai_auto_download import looks_like_quant, split_model_ref
@ -3866,8 +3864,7 @@ def _matches_any(requested: str, candidates) -> bool:
"""Whether *requested* names any of *candidates*.
A repo alias is case-insensitive, a filesystem path is not: lowercasing both
made /srv/models/foo.gguf and /srv/models/Foo.gguf the same weights, which is
the same trap _norm_path exists for one comparison further down.
made /srv/models/foo.gguf and /srv/models/Foo.gguf the same weights.
"""
lowered = requested.strip().lower()
for candidate in candidates:
@ -3893,9 +3890,8 @@ def _norm_path(value: str) -> str:
/srv/models/Foo and /srv/models/foo are different models."""
import os
# normcase after, not before: on Windows it folds case *and* rewrites the
# separator to a backslash, so normalizing first leaves the descendant checks
# below comparing a "/" against a path that no longer has any.
# normcase after, not before: on Windows it folds case *and* rewrites "/" to a
# backslash, leaving the descendant checks below comparing against a path with none.
return os.path.normcase(str(value)).replace("\\", "/").rstrip("/")
@ -3908,10 +3904,9 @@ def _resident_quant_is(variant: Optional[str]) -> bool:
def _resolves_to_resident(load_path: Optional[str], *, llama_only: bool = False) -> bool:
"""Whether a resolved on-disk path is what is already loaded.
``llama_only`` drops the Transformers backend from the comparison. Only
llama.cpp carries a quant identity, so a Transformers model active from a
directory that also holds GGUF exports would otherwise match a request for
one of those quants and answer it with the safetensors weights.
``llama_only`` drops the Transformers backend: only llama.cpp carries a quant
identity, so a Transformers model active from a directory that also holds GGUF
exports would otherwise answer a request for one of those quants.
"""
if not load_path:
return False
@ -3933,9 +3928,9 @@ def _resolves_to_resident(load_path: Optional[str], *, llama_only: bool = False)
return True
if current.startswith(f"{target}/"):
# A model directory holding the weights loaded from it. Nested entries
# (/models/A alongside /models/A/sub/B) satisfied this too, so a request
# for A was answered with B. The innermost indexed model owns the file;
# with none indexed there is no nesting to tell apart, so keep matching.
# (/models/A alongside /models/A/sub/B) matched too, so a request for A was
# answered with B. The innermost indexed model owns the file; with none
# indexed there is no nesting to tell apart, so keep matching.
owner = _innermost_indexed_owner(current)
if owner is None or owner == target:
return True
@ -3965,13 +3960,11 @@ async def _reject_unservable_model(
"""Refuse rather than answer a named model with a different one.
Only for a reference this server can tell was meant for it: an explicit GGUF
quant, or a model that is actually here. A namespace decides nothing either
way. ``vendor/model`` is how LiteLLM and OpenRouter name every provider, so
``anthropic/claude-3.5-sonnet`` falls through like ``gpt-4``; a standalone or
custom-folder GGUF is advertised without one, so a slashless id that does
resolve locally is still a concrete reference. Only runs while something is
serving; with nothing loaded the caller's own :func:`_no_model_loaded_error`
already says the right thing.
quant, or a model that is actually here. A namespace decides nothing either way
(``vendor/model`` is how LiteLLM and OpenRouter name every provider, and a
standalone GGUF is advertised without one), so a slashless id that resolves
locally is still a concrete reference. Only runs while something is serving:
with nothing loaded, :func:`_no_model_loaded_error` already says the right thing.
"""
from core.inference.openai_auto_download import looks_like_quant, split_model_ref
@ -4006,27 +3999,24 @@ async def _reject_unservable_model(
warm_index_soon()
resolved = resolve_local_gguf(requested_model, allow_scan = False)
else:
# Before the first scan there is nothing cached to reason from, and falling
# through would answer a named model with the resident one. Pay the scan
# once, off the loop and bounded, rather than reading "not scanned yet" as
# "not here". Later requests take the cached branch above.
# Nothing cached to reason from yet, and falling through would answer a
# named model with the resident one. Pay the scan once, off the loop and
# bounded, rather than read "not scanned yet" as "not here".
try:
resolved = await asyncio.wait_for(
asyncio.to_thread(resolve_local_gguf, requested_model),
_COLD_INDEX_WAIT_S,
)
except (TimeoutError, asyncio.TimeoutError):
# Still scanning, so nothing is known about this name. Falling through
# would put the resident model behind it, which is the failure this
# whole hook exists to stop, so say "not yet" instead of guessing.
# Still scanning, so nothing is known about this name: say "not yet"
# rather than guess and put the resident model behind it.
warm_index_soon()
still_indexing = True
resolved = None
# A manual load stores the on-disk path the resolver advertises under an alias, so
# match on the path too.
# Quants of one repo share a directory, so the path alone cannot tell them
# apart: without the variant check an explicit :Q8_0 would be answered by a
# resident Q4_K_M, which _loaded_satisfies has already refused by name.
# A manual load stores the on-disk path the resolver advertises under an alias,
# so match on the path too. Quants of one repo share a directory, so the path
# alone cannot tell them apart: without the variant check an explicit :Q8_0
# would be answered by a resident Q4_K_M.
if (
resolved is not None
and _resolves_to_resident(resolved[0], llama_only = quantified)
@ -4052,8 +4042,8 @@ async def _reject_unservable_model(
)
switchable = downloaded and get_openai_auto_switch_enabled()
except HTTPException:
# A refusal decided above is the answer, not a failure to decide. Without this
# the handler below would log it and fall through to the resident model.
# A refusal decided above is the answer, not a failure to decide: without this
# the handler below logs it and falls through to the resident model.
raise
except Exception as exc:
# Can't verify: an explicit quant still proves intent, so refuse; let anything else by.
@ -4189,9 +4179,9 @@ async def _maybe_auto_switch_model(
# repo, so it never reloads a different local quant that already serves it.
from core.inference.openai_auto_download import looks_like_quant, split_model_ref
# A tag that names no quant (":latest", ":8b") means the repo, exactly as
# _loaded_satisfies and the resolver read it. Treating it as a quant tears
# down a serving Q8 to load the preferred Q4 for a request either satisfies.
# A tag that names no quant (":latest", ":8b") means the repo, as
# _loaded_satisfies and the resolver read it. Treating it as a quant tears down
# a serving Q8 to load the preferred Q4 for a request either satisfies.
_, _requested_variant = split_model_ref(requested_model)
bare = not looks_like_quant(_requested_variant)
@ -11113,8 +11103,8 @@ def _quant_reference_resolves(model_id: Optional[str], quant: str) -> bool:
"""Whether ``<model_id>:<quant>`` still resolves once this model is not resident.
A standalone .gguf takes its quant from the filename, but the resolver stores
such files with no quants at all, so advertising one hands out a pin that dies
the moment another model loads.
such files with no quants, so advertising one hands out a pin that dies the
moment another model loads.
"""
from core.inference.local_model_resolver import (
index_is_built,
@ -11125,7 +11115,7 @@ def _quant_reference_resolves(model_id: Optional[str], quant: str) -> bool:
if not model_id:
return False
# Cold index proves nothing, and publishing on no proof is what hands out the
# A cold index proves nothing, and publishing on no proof is what hands out the
# dead pin; warm so the next response carries the quant.
warm_index_soon()
return resolve_local_gguf(f"{model_id}:{quant}", allow_scan = False) is not None
@ -11135,8 +11125,8 @@ def _advertised_local_path(model: str) -> Optional[str]:
"""On-disk path of *model* if the last /v1/models scan listed it, else None.
Cache-only, never scans. The catalog scans on its own schedule, so it can have
advertised a local model the resolver index has not picked up yet; having
advertised it is evidence the name means something other than the resident one.
advertised a local model the resolver index has not picked up yet, which is
evidence the name means something other than the resident one.
"""
if _ADVERTISED_CACHE["at"] != _CATALOG_CACHE["at"]:
paths = {}
@ -11231,17 +11221,16 @@ async def _openai_catalog_objects() -> list[dict]:
"object": "model",
"created": _created,
"owned_by": _OWNED_BY,
# A manual load keys the resident entry by path basename while the catalog uses
# the alias, so match on the path or the alias reads as not loaded. llama-only:
# these entries are advertised as GGUF with a GGUF quant, so a Transformers
# model live from a directory that also holds GGUF exports must not mark one
# loaded, or the examples pin a quant nothing can serve with switching off.
# A manual load keys the resident entry by path basename while the catalog
# uses the alias, so match on the path or the alias reads as not loaded.
# llama-only: a Transformers model live from a directory that also holds
# GGUF exports must not mark one of these GGUF entries loaded, or the
# examples pin a quant nothing can serve with switching off.
"loaded": _resolves_to_resident(getattr(info, "path", None), llama_only = True),
}
# The id stays bare for OpenAI compat; a client appends ":<quant>" to pin one.
# For the resident model that has to be the quant actually loaded, not the
# preferred one on disk, or the listing advertises alias:Q4 as loaded while
# Q8 is serving and pinning it 404s.
# For the resident model that must be the quant actually loaded, not the
# preferred one on disk, or the listing advertises alias:Q4 while Q8 serves.
resident_quant = getattr(get_llama_cpp_backend(), "hf_variant", None)
if obj["loaded"] and resident_quant:
obj["quant"] = resident_quant

View file

@ -62,21 +62,19 @@ def pytest_addoption(parser):
def _no_background_model_scan(monkeypatch):
"""Keep the /v1 admission hook from scanning the real HF cache during tests.
The hook warms the local-model index on a background thread. That is right in a
server and wrong here: it walks the developer's actual caches, which on a large
install takes seconds, and the resulting I/O starves the loop under the
timing-sensitive streaming tests. Tests that exercise the warm patch it back.
The hook warms the local-model index on a background thread: right in a server,
wrong here, since it walks the developer's actual caches and the I/O starves the
loop under timing-sensitive streaming tests. Warm tests patch it back.
"""
import time
from core.inference import local_model_resolver
monkeypatch.setattr(local_model_resolver, "warm_index_soon", lambda: None)
# Start from a built, empty index. Stubbing only the background warm still left the
# cold path walking those caches synchronously inside the admission wait, so on a
# large install the assertion became a 503 "still indexing". Tests that want the
# cold path set _scan back themselves (and stub the scan). _build_index is left
# alone so the tests that call it directly still exercise the real walk.
# Start from a built, empty index: stubbing only the warm left the cold path
# walking those caches inside the admission wait, so on a large install the
# assertion became a 503 "still indexing". Cold-path tests reset _scan themselves;
# _build_index is untouched so tests calling it directly still walk for real.
monkeypatch.setattr(local_model_resolver, "_scan", (time.monotonic(), {}))

View file

@ -4,9 +4,8 @@
"""Opt-in auto-download of a GGUF a /v1 request names but this server lacks.
No network: huggingface_hub, the consent probe and the Hub download service are
all mocked. The invariant these guard is that with the setting off nothing here
runs at all, and with it on a name that isn't shaped like a repo still falls
through to the resident model.
all mocked. The invariant: with the setting off nothing here runs at all, and
with it on a name not shaped like a repo still falls through to the resident model.
"""
import asyncio
@ -87,10 +86,9 @@ def _gated_error():
def _hub_error(error_type, status_code: int, message: str):
"""Build a Hub exception across huggingface_hub majors.
huggingface_hub 1.x made ``response`` a required keyword-only argument, and
the project floor is 0.34, so construct positionally and fall back. The
positional form carries no response, and hf_error_status reads the status off
it for the types that do not encode it in their name, so attach one either way.
huggingface_hub 1.x made ``response`` a required keyword-only argument and the
project floor is 0.34, so construct positionally and fall back. The positional
form carries no response, which hf_error_status reads, so attach one either way.
"""
try:
exc = error_type(message)
@ -113,9 +111,8 @@ def _hub_error(error_type, status_code: int, message: str):
def test_the_hub_error_helper_carries_a_status_on_both_majors():
# CI runs huggingface_hub 1.x and this box runs 0.x, and only one of the two
# constructor shapes works on each. hf_error_status reads the status off the
# response, so a helper that silently produced one without it would make an
# CI runs huggingface_hub 1.x and this box 0.x, and each takes only one of the
# constructor shapes. A helper that silently dropped the response would make an
# error-mapping test pass here and fail there.
from hub.utils.hf_errors import hf_error_status
@ -553,10 +550,9 @@ def test_an_unknown_state_still_reports_the_download_to_a_retry(hub, monkeypatch
def test_a_hanging_code_probe_does_not_pin_the_slot(hub, monkeypatch):
# hf_hub_download and auth_check take no timeout, and both run while the provisional
# slot is held, so an unresponsive Hub stalled the request far past the metadata
# budget and reported every other model busy meanwhile. Unchecked is not cleared,
# so the bounded probe refuses rather than admitting the repo.
# hf_hub_download and auth_check take no timeout and run while the provisional slot
# is held, so an unresponsive Hub stalled the request and reported every other model
# busy. Unchecked is not cleared, so the bounded probe refuses instead of admitting.
import threading
entered, release = threading.Event(), threading.Event()
@ -610,10 +606,9 @@ def test_a_hanging_auth_check_falls_through_to_the_download(hub, monkeypatch):
def test_a_companion_only_repo_is_not_held_at_busy(hub):
# mmproj and MTP files are companions, not quants, so admission classifies such a
# repo as non-servable and lets the label fall through to the resident model. The
# busy probe accepted any .gguf, which stranded that ordinary traffic behind an
# unrelated multi-hour download.
# mmproj and MTP files are companions, not quants, so such a repo is non-servable
# and falls through to the resident model. The busy probe accepted any .gguf, which
# stranded that ordinary traffic behind an unrelated multi-hour download.
assert _run("unsloth/x-GGUF:UD-Q4_K_XL").code == "model_downloading"
gb = 1024**3
hub["info"] = _Info([_Sibling("mmproj-F16.gguf", gb), _Sibling("mtp-model.gguf", gb)])
@ -1242,9 +1237,8 @@ def test_the_request_path_never_triggers_a_model_index_rescan(monkeypatch):
def test_a_cold_index_is_scanned_rather_than_read_as_nothing_here(monkeypatch):
# Before the first scan there is no cached evidence, and treating that as "not
# downloaded" answers a named local model with the resident one. The scan is paid
# once, off the loop; every later request reads the built index instead.
# With no cached evidence yet, reading that as "not downloaded" answers a named
# local model with the resident one. Pay the scan once, off the loop.
from core.inference import local_model_resolver as resolver
entry = resolver._LocalGgufEntry("org/other", "/srv/models/org--other", ("Q4_K_M",))
@ -1280,9 +1274,8 @@ def test_a_cold_index_is_scanned_rather_than_read_as_nothing_here(monkeypatch):
def test_a_cold_scan_that_never_finishes_says_so_instead_of_guessing(monkeypatch):
# The scan is bounded so a pathological install cannot hold the request open, but
# an unfinished scan knows nothing about the name, and falling through would put
# the resident model behind it. Answer "not yet", with a Retry-After.
# The scan is bounded, but an unfinished one knows nothing about the name, and
# falling through would put the resident model behind it: answer "not yet".
import threading
from core.inference import local_model_resolver as resolver
@ -1312,8 +1305,8 @@ def test_a_cold_scan_that_never_finishes_says_so_instead_of_guessing(monkeypatch
def test_a_refusal_is_never_swallowed_by_the_cannot_verify_handler(monkeypatch):
# The checks run inside a broad `except Exception` that turns a failure to decide
# into a fallthrough. An HTTPException raised in there is a decision, and was
# being logged as a failure and answered by the resident model instead.
# into a fallthrough. An HTTPException there is a decision, but was logged as a
# failure and answered by the resident model.
loaded = _Loaded("unsloth/A-GGUF", "UD-Q4_K_XL")
monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: loaded)
monkeypatch.setattr(
@ -1435,9 +1428,8 @@ def test_an_advertised_alias_for_the_resident_weights_is_still_served(monkeypatc
def test_a_rejected_token_says_so_instead_of_asking_for_a_retry(hub):
# Hugging Face answers an expired or invalid X-Unsloth-HF-Token with 401. Only
# 403 and 404 were handled, so it fell through to "could not reach Hugging Face"
# with a 503, telling the caller to retry something that cannot start working.
# Hugging Face 401s an expired X-Unsloth-HF-Token. Only 403/404 were handled, so it
# fell through to a 503 telling the caller to retry something that cannot work.
from huggingface_hub.utils import HfHubHTTPError
hub["raise"] = _hub_error(HfHubHTTPError, 401, "unauthorized")
@ -1448,9 +1440,8 @@ def test_a_rejected_token_says_so_instead_of_asking_for_a_retry(hub):
def test_an_image_request_does_not_download_a_text_only_model(hub):
# The capability guard only ever sees an already-local target, so without this
# an image request would spend gigabytes on weights that cannot answer it and
# then 400 on every retry.
# The capability guard only ever sees an already-local target, so without this an
# image request spends gigabytes on weights that then 400 on every retry.
gb = 1024**3
hub["info"] = _Info([_Sibling("model-UD-Q5_K_XL.gguf", 5 * gb)])
refusal = asyncio.run(
@ -1489,10 +1480,9 @@ def test_two_models_differing_only_in_case_are_not_the_same_weights(monkeypatch)
def test_a_quant_request_is_not_satisfied_by_transformers_weights(monkeypatch):
# A Transformers model active from a directory that also holds GGUF exports
# resolves to that same directory, and the path match let admission answer an
# explicit quant with the safetensors weights. Only llama.cpp has a quant
# identity, which is why _loaded_satisfies already refuses this by name.
# A Transformers model active from a directory that also holds GGUF exports resolves
# to that directory, so the path match let admission answer an explicit quant with
# the safetensors weights. Only llama.cpp has a quant identity.
from core.inference import local_model_resolver as resolver
entry = resolver._LocalGgufEntry("alias", "/srv/models/tuned", ("Q4_K_M",))
@ -1564,9 +1554,8 @@ def test_a_timed_out_download_stops_holding_the_slot_once_unprobeable(monkeypatc
def test_a_sibling_quant_in_the_same_directory_is_not_the_resident_one(monkeypatch):
# Quants of one repo share a directory, so the path match alone cannot tell
# them apart, and an explicit :Q8_0 was answered by a resident Q4_K_M that
# _loaded_satisfies had already refused by name.
# Quants of one repo share a directory, so the path match alone cannot tell them
# apart, and an explicit :Q8_0 was answered by a resident Q4_K_M.
from core.inference import local_model_resolver as resolver
entry = resolver._LocalGgufEntry("org/model", "/hf/org--model/snap", ("Q4_K_M", "Q8_0"))
@ -1587,9 +1576,8 @@ def test_a_sibling_quant_in_the_same_directory_is_not_the_resident_one(monkeypat
def test_a_remote_tag_that_names_no_quant_picks_the_preferred_one(hub):
# ":latest" and ":8b" name no quant, so remote admission must default-select
# like a bare repo id instead of 404ing on a quant that never existed. Matches
# what the local resolver now does with the same tag.
# ":latest" and ":8b" name no quant, so remote admission must default-select like a
# bare repo id (as the local resolver does) instead of 404ing on a non-quant.
assert _run("unsloth/x-GGUF").code == "model_downloading"
bare_repo, bare_variant, _ = hub["started"][0]
for tag in (":latest", ":8b"):
@ -1608,9 +1596,9 @@ def test_a_remote_tag_that_names_no_quant_picks_the_preferred_one(hub):
def test_a_generic_gguf_advertises_the_label_the_worker_resolves(hub):
# With no recognized quant token the label extractors part ways: one takes the
# last hyphenated segment, the plan and the worker key the whole stem. Dispatching
# ours made the worker exit with "No GGUF shards matching variant".
# With no recognized quant token the extractors part ways: one takes the last
# hyphenated segment, the plan and worker key the whole stem. Dispatching ours
# made the worker exit with "No GGUF shards matching variant".
from hub.utils.gguf import extract_quant_label as canonical
from hub.utils.gguf_plan import build_gguf_variant_plans
@ -1624,9 +1612,9 @@ def test_a_generic_gguf_advertises_the_label_the_worker_resolves(hub):
def test_windows_style_paths_still_match_their_own_directory(monkeypatch):
# normcase folds case and rewrites the separator to a backslash on Windows, so
# normalizing to "/" before it left the descendant checks comparing a "/" against
# a path that had none, and a resident model read as a different one.
# normcase rewrites "/" to a backslash on Windows, so normalizing before it left the
# descendant checks comparing against a path with none, and a resident model read
# as a different one.
import ntpath
monkeypatch.setattr(inference_route.os.path, "normcase", ntpath.normcase)
@ -1646,9 +1634,8 @@ def test_windows_style_paths_still_match_their_own_directory(monkeypatch):
def test_a_bare_request_for_a_just_downloaded_model_is_refused(monkeypatch):
# End of the same chain: the note has to reach admission, or a bare request in
# the window between the download landing and the scan is served by the resident
# model, which is the whole failure this hook exists to stop.
# End of the same chain: the note has to reach admission, or a bare request between
# the download landing and the scan is served by the resident model.
from core.inference import local_model_resolver as resolver
monkeypatch.setattr(resolver, "_scan", (time.monotonic(), {}))
@ -1668,9 +1655,9 @@ def test_a_bare_request_for_a_just_downloaded_model_is_refused(monkeypatch):
def test_a_non_quant_tag_does_not_tear_down_a_serving_quant(monkeypatch):
# _already_serving split on ":" rather than on whether the suffix names a quant,
# so org/model:latest against a serving Q8_0 counted as a quant mismatch and
# swapped in the preferred Q4_K_M, for a request either one satisfies.
# _already_serving split on ":" rather than on whether the suffix names a quant, so
# org/model:latest against a serving Q8_0 counted as a mismatch and swapped in the
# preferred Q4_K_M, for a request either one satisfies.
from core.inference import local_model_resolver as resolver
entry = resolver._LocalGgufEntry("org/model", "/hf/org--model/snap", ("Q4_K_M", "Q8_0"))
@ -1698,9 +1685,9 @@ def test_a_non_quant_tag_does_not_tear_down_a_serving_quant(monkeypatch):
def test_the_trust_probe_never_falls_back_to_the_server_identity(hub, monkeypatch):
# huggingface_hub treats None as "use the cached login", so only an explicit
# False is anonymous. The metadata probe and the worker already pass one; this
# probe did not, so a caller-named repo was read with the server's identity.
# huggingface_hub treats None as "use the cached login", so only an explicit False
# is anonymous. This probe passed None, so a caller-named repo was read with the
# server's identity.
seen: list = []
def _probe(model_name, hf_token = None):
@ -1718,9 +1705,8 @@ def test_the_trust_probe_never_falls_back_to_the_server_identity(hub, monkeypatc
def test_a_foreign_label_is_not_told_to_wait_for_someone_elses_download(hub):
# The busy refusal fired before the probe, so any namespaced label a drop-in
# client sends (LiteLLM/OpenRouter style) was told to wait out a download that
# has nothing to do with it, for as long as that download runs.
# The busy refusal fired before the probe, so any namespaced label a drop-in client
# sends (LiteLLM/OpenRouter style) was told to wait out an unrelated download.
assert _run("unsloth/first-GGUF").code == "model_downloading"
hub["info"] = _Info([_Sibling("README.md", 1024)]) # real repo, no GGUF
@ -1733,9 +1719,8 @@ def test_a_foreign_label_is_not_told_to_wait_for_someone_elses_download(hub):
def test_a_failed_download_keeps_the_slot_until_someone_is_told(monkeypatch):
# The watcher freed the slot the moment it saw the error, but Retry-After is 30s
# and the poll is 2s, so the client came back to an empty slot and started the
# identical failing download again instead of being told it had failed.
# The watcher freed the slot on the error, but Retry-After is 30s and the poll 2s,
# so the client came back to an empty slot and restarted the same failing download.
monkeypatch.setattr(auto_dl, "_MAX_WATCH_S", 60.0)
monkeypatch.setattr(auto_dl, "_WATCH_POLL_S", 0.001)
@ -1782,9 +1767,8 @@ def test_a_completed_download_does_not_restage_the_scan_it_just_warmed(monkeypat
def test_an_exact_generic_variant_beats_the_default_pick(hub):
# Canonicalizing generic labels made them real worker keys, but the matcher still
# read anything non-quant-shaped as a tag, so repo:llama-13b default-selected and
# fetched llama-7b instead of the model that was actually asked for.
# Canonicalizing generic labels made them real worker keys, but the matcher read
# anything non-quant-shaped as a tag, so repo:llama-13b default-selected llama-7b.
gb = 1024**3
hub["info"] = _Info([_Sibling("llama-7b.gguf", 4 * gb), _Sibling("llama-13b.gguf", 8 * gb)])
assert _run("unsloth/generic-GGUF:llama-13b").code == "model_downloading"

View file

@ -23,9 +23,8 @@ from utils import openai_auto_switch_settings as settings
def _clean_resolver_index():
"""Drop the scan cache around every test.
The /v1 admission hook warms the index in the background, so without this a
test that exercises the hook can publish its own fixture's scan and, inside the
TTL, hand it to the next test that expects a fresh one.
The /v1 admission hook warms the index in the background, so a test exercising it
can publish its fixture's scan and, inside the TTL, hand it to the next test.
"""
resolver.invalidate_index()
yield
@ -4146,11 +4145,10 @@ def test_env_idle_below_floor_is_clamped(monkeypatch):
def test_a_tag_that_names_no_quant_resolves_to_the_repo(monkeypatch):
# A downloaded but unloaded GGUF asked for as org/model:latest missed the
# resolver, so the switch path could not load it: with auto-download on it
# probed the Hub and 404d on a quant that was never a quant, and with it off it
# refused without switching. A real quant that is not on disk must still miss,
# or a swap would serve the wrong weights under the right name.
# A downloaded but unloaded GGUF asked for as org/model:latest missed the resolver,
# so the switch could not load it (404ing on a quant that was never a quant with
# auto-download on, refusing with it off). A real quant that is not on disk must
# still miss, or a swap would serve the wrong weights under the right name.
from core.inference.local_model_resolver import _LocalGgufEntry
import time
@ -4173,9 +4171,9 @@ def test_a_tag_that_names_no_quant_resolves_to_the_repo(monkeypatch):
def test_any_finished_download_drops_the_resolver_cache(monkeypatch):
# Only the API auto-download watcher invalidated, so a GGUF fetched in the Hub
# UI stayed absent to the cache-only request path and the request was answered
# by the resident model instead. Every worker exits through here.
# Only the API auto-download watcher invalidated, so a GGUF fetched in the Hub UI
# stayed absent to the cache-only request path and the resident model answered.
# Every worker exits through here.
import logging
from hub.services import download_lifecycle
@ -4222,9 +4220,9 @@ def test_any_finished_download_drops_the_resolver_cache(monkeypatch):
def test_invalidating_keeps_the_entries_it_already_had(monkeypatch):
# The request path reads this cache without scanning, so emptying it leaves it
# with no evidence about any local model until the rebuild lands. Only a
# completed download invalidates, and that only adds, so the entries stay true.
# The request path reads this cache without scanning, so emptying it leaves no
# evidence until the rebuild lands. Only a completed download invalidates, and
# that only adds, so the entries stay true.
import time
entry = resolver._LocalGgufEntry("org/old", "/srv/models/org--old", ("Q4_K_M",))
@ -4240,9 +4238,8 @@ def test_invalidating_keeps_the_entries_it_already_had(monkeypatch):
def test_a_bare_local_id_takes_the_quant_a_plain_load_would(monkeypatch, tmp_path):
# list_local_gguf_variants orders by descending size, so the head is the biggest
# quant. Resolving a bare id to that could evict a working model and then OOM
# starting an F16 on a box sized for the Q4 sitting right next to it, and
# /v1/models advertised the same head for pinning.
# quant. Resolving a bare id to that could evict a working model and then OOM on an
# F16 next to a fitting Q4, and /v1/models advertised the same head for pinning.
from core.inference.local_model_resolver import _local_gguf_entry
for name, size in (("model-F16.gguf", 900), ("model-Q4_K_M.gguf", 100)):
@ -4263,9 +4260,8 @@ def test_local_and_remote_agree_on_the_preferred_quant():
def test_a_just_downloaded_model_is_evidence_before_the_scan_indexes_it(monkeypatch):
# Retaining the old index covers what was already known, but nothing covers the
# model that just landed until the next scan finishes. A bare request for it in
# that window was answered by the unrelated resident model.
# The retained index covers what was known, but nothing covers the model that just
# landed until the next scan: a bare request for it was answered by the resident one.
import logging
from hub.services import download_lifecycle
@ -4312,10 +4308,9 @@ def test_a_just_downloaded_model_is_evidence_before_the_scan_indexes_it(monkeypa
def test_a_finished_dataset_is_not_recorded_as_a_local_model(monkeypatch):
# finalize_worker_exit is shared with dataset downloads. Noting one as a local
# model would refuse a bare /v1 request naming that id while another model is
# resident, instead of letting a foreign id fall through, and would kick off a
# multi-directory model scan for nothing.
# finalize_worker_exit is shared with dataset downloads. Noting one as a local model
# would refuse a bare /v1 request naming that id instead of letting a foreign id
# fall through, and would kick off a multi-directory scan for nothing.
import logging
import time
@ -4359,9 +4354,8 @@ def test_a_finished_dataset_is_not_recorded_as_a_local_model(monkeypatch):
def test_two_local_paths_differing_only_in_case_are_not_the_same_model(monkeypatch):
# _loaded_satisfies lowercased the request and every backend identifier, so on a
# case-sensitive filesystem /srv/models/foo.gguf counted as satisfied by a
# resident /srv/models/Foo.gguf and returned before the case-preserving compare
# further down ever ran. A repo alias must stay case-insensitive.
# case-sensitive filesystem /srv/models/foo.gguf read as satisfied by a resident
# /srv/models/Foo.gguf. A repo alias must still stay case-insensitive.
import os
loaded = _FakeBackend(loaded_id = "/srv/models/Foo.gguf")

View file

@ -301,9 +301,9 @@ def test_a_loaded_alias_advertises_the_quant_that_is_actually_loaded(monkeypatch
def test_a_nested_model_directory_is_not_the_resident_one(monkeypatch):
# Two separately indexed models can nest (/models/A holding A, /models/A/sub/B
# holding B). A plain prefix test made loading B mark A resident, so a request for
# A was answered with B's weights. The innermost indexed model owns the file.
# Two indexed models can nest (/models/A holding A, /models/A/sub/B holding B). A
# plain prefix test made loading B mark A resident, so a request for A was answered
# with B's weights. The innermost indexed model owns the file.
outer = _Info("/models/A", "A", model_id = "publisher/A")
outer.path = "/models/A"
inner = _Info("/models/A/sub/B", "B", model_id = "publisher/B")
@ -325,10 +325,9 @@ def test_a_nested_model_directory_is_not_the_resident_one(monkeypatch):
def test_a_transformers_model_does_not_mark_a_gguf_alias_loaded(monkeypatch):
# Every entry in this loop is advertised as GGUF and carries a GGUF quant. A
# Transformers model live from a directory that also holds GGUF exports is not one
# of them, and marking the alias loaded had the usage examples pin alias:quant that
# nothing can serve while switching is off.
# Every entry in this loop is advertised as GGUF with a GGUF quant. A Transformers
# model live from a directory that also holds GGUF exports is not one, and marking
# the alias loaded had the examples pin a quant nothing can serve with switching off.
unsloth = _FakeUnsloth()
unsloth.active_model_name = "/srv/models"
monkeypatch.setattr(inf, "get_inference_backend", lambda: unsloth)

View file

@ -212,14 +212,11 @@ def _force_missing_fla_imports(monkeypatch):
def _pin_fla_model_types(monkeypatch):
"""Pin the auto-discovered FLA allowlist to the Qwen GDN families.
`_discover_fla_model_types` scans the *installed* transformers for modeling
files importing `from fla.`, and `models/qwen3_5/` only exists from
transformers 5.x. The backend supports `transformers>=4.51`, so on a 4.x
install the gate returns False and every Qwen3.5 assertion below silently
passes through a no-op instead of exercising the install path. Pinning keeps
these tests hermetic across the whole supported transformers range, the same
way test_hook_does_not_install_tilelang_for_model_outside_allowlist pins it
against newly added FLA model_types.
`_discover_fla_model_types` scans the *installed* transformers, and
`models/qwen3_5/` only exists from 5.x. The backend supports
`transformers>=4.51`, so on a 4.x install the gate returns False and every
Qwen3.5 assertion below silently no-ops. Pinning keeps these tests hermetic
across the supported range.
"""
monkeypatch.setattr(
worker,

View file

@ -7,10 +7,9 @@ All off by default so existing API behavior is unchanged:
- ``openai_api_auto_switch_model``: when on, a ``/v1`` request whose ``model``
names a downloaded local GGUF different from the loaded one transparently
loads it before serving (llama-swap-style). Unknown names pass through.
- ``openai_api_auto_download_model``: when on (and auto-switch is too), a
``/v1`` request naming a GGUF repo that is *not* downloaded starts a
background download instead of failing. Gated on auto-switch, which is what
serves the model once it lands.
- ``openai_api_auto_download_model``: when on, a ``/v1`` request naming an
undownloaded GGUF repo starts a background download instead of failing.
Gated on auto-switch, which is what serves the model once it lands.
- ``openai_api_auto_unload_idle_seconds``: when > 0, the loaded GGUF is
unloaded after this many idle seconds to free VRAM. Enabled values have a
60s floor (0 stays "off"): a tiny TTL tears the model down between turns of
@ -102,11 +101,8 @@ def get_openai_auto_switch_enabled() -> bool:
def get_stored_openai_auto_download_enabled() -> bool:
"""The persisted auto-download flag, independent of auto-switch.
The settings UI reads this so toggling auto-switch off displays and
round-trips the saved value rather than erasing it.
"""
"""The persisted auto-download flag, independent of auto-switch, so the UI
round-trips the saved value across an auto-switch toggle instead of erasing it."""
parsed = _coerce_bool(_cached_setting(OPENAI_AUTO_DOWNLOAD_SETTING_KEY, None))
return parsed if parsed is not None else DEFAULT_OPENAI_AUTO_DOWNLOAD_ENABLED
@ -114,8 +110,8 @@ def get_stored_openai_auto_download_enabled() -> bool:
def get_openai_auto_download_enabled() -> bool:
"""Whether a /v1 request may download a GGUF repo it names but doesn't have.
Gated on auto-switch: auto-switch is what loads the model once it lands, so
downloading without it would fetch gigabytes nothing can then serve.
Gated on auto-switch: that is what loads the model once it lands, so without
it we would fetch gigabytes nothing can serve.
"""
return get_stored_openai_auto_download_enabled() and get_openai_auto_switch_enabled()

View file

@ -334,7 +334,7 @@ const KEY_PLACEHOLDER = "sk-unsloth-YOUR_KEY";
const USE_TUNNEL_KEY = "unsloth_api_use_tunnel";
// Slow retry while /v1 has nothing to name: a download or load moves no store state.
const CATALOG_RETRY_MS = 15000;
// Slower beat once something is servable: idle unload frees a model without
// Slower beat once something is servable: an idle unload frees a model without
// touching the store, so residency is never settled for good.
const CATALOG_IDLE_MS = 60000;
@ -382,9 +382,8 @@ function useExampleModelName(): string | null {
const [catalog, setCatalog] = useState<OpenAIModel[] | null>(null);
// A downloaded but unloaded model is only runnable when switching is on.
const [autoSwitch, setAutoSwitch] = useState(false);
// Idle-unload running on its own (UNSLOTH_MODEL_IDLE_TTL, switching off) still
// reloads exactly what it freed on the next request. That restores the stored
// checkpoint only, never an arbitrary catalog entry, so it is tracked apart.
// Idle-unload on its own (UNSLOTH_MODEL_IDLE_TTL, switching off) reloads exactly
// what it freed: the stored checkpoint only, never an arbitrary catalog entry.
const [idleReload, setIdleReload] = useState(false);
const usableCheckpoint =
!!checkpoint && !checkpoint.startsWith("external::") && !looksLikePath(checkpoint);
@ -396,9 +395,9 @@ function useExampleModelName(): string | null {
let timeoutId: number | null = null;
const update = () => {
// null on failure, never [] or false: a transient error is not evidence that the
// server holds nothing, and feeding those negatives in blanked every example while
// the model was still servable. Keep the last answer and retry.
// null on failure, never [] or false: a transient error is no evidence that the
// server holds nothing, and those negatives blanked every example while the
// model was still servable. Keep the last answer and retry.
void Promise.all([
listOpenAIModels().catch(() => null),
loadOpenAIAutoSwitchSettings()
@ -443,10 +442,10 @@ function useExampleModelName(): string | null {
? `${pick.id}:${pick.quant}`
: pick.id;
};
// The store keeps a checkpoint across an idle unload, and across the model
// being deleted, so it only names a runnable model while the catalog still
// lists it: resident, or downloaded with switching able to reload it. A null
// catalog means /v1/models has not answered, which is not evidence against it.
// The store keeps a checkpoint across an idle unload and across the model being
// deleted, so it only names a runnable model while the catalog still lists it:
// resident, or downloaded with switching able to reload it. A null catalog means
// /v1/models has not answered, which is not evidence against it.
const entry = catalog?.find((m) => sameBaseModelId(m.id, checkpoint ?? ""));
const backed =
catalog === null || (!!entry && (entry.loaded || autoSwitch || idleReload));
@ -454,9 +453,9 @@ function useExampleModelName(): string | null {
if (checkpoint.includes(":")) {
return checkpoint;
}
// Pin the quant the catalog advertises, not the stored one: membership proves
// the repo, and the saved quant can name a file deleted while another quant of
// the same repo remains. Fall back to the store only before /v1/models answers.
// Pin the quant the catalog advertises, not the stored one: membership proves the
// repo, and the saved quant can name a file deleted while another quant remains.
// Fall back to the store only before /v1/models answers.
const quant = catalog === null ? ggufVariant : entry?.quant;
return quant ? `${checkpoint}:${quant}` : checkpoint;
}

View file

@ -50,9 +50,9 @@ def test_examples_never_print_a_hardcoded_model_id():
def test_catalog_refresh_follows_the_loaded_model():
# A dep list that misses these never re-ran, so a finished load left the first
# fetch's name. It must not be gated on having no checkpoint either: the store
# keeps one across an idle unload, which changes nothing React can see.
# A dep list missing these never re-ran, so a finished load left the first fetch's
# name. Nor may it be gated on having no checkpoint: the store keeps one across an
# idle unload, which changes nothing React can see.
src = USAGE_EXAMPLES_TSX.read_text(encoding = "utf-8")
hook = src[src.find("function useExampleModelName") : src.find("// Backend PATH detection")]
assert "}, [checkpoint, ggufVariant]);" in hook
@ -66,9 +66,9 @@ def test_catalog_refresh_follows_the_loaded_model():
def test_a_stored_checkpoint_needs_catalog_evidence():
# The store keeps a checkpoint across an idle unload and across the model being
# deleted. Preferring it on the switch setting alone kept naming one /v1/models
# had already proved absent, so the snippets 404d instead of falling back.
# The store keeps a checkpoint across an idle unload and across a deletion, so
# preferring it on the switch setting alone named a model /v1/models had proved
# absent, and the snippets 404d instead of falling back.
src = USAGE_EXAMPLES_TSX.read_text(encoding = "utf-8")
hook = src[src.find("function useExampleModelName") : src.find("// Backend PATH detection")]
assert 'const entry = catalog?.find((m) => sameBaseModelId(m.id, checkpoint ?? ""));' in hook
@ -79,8 +79,8 @@ def test_a_stored_checkpoint_needs_catalog_evidence():
def test_standalone_idle_unload_still_names_the_stored_checkpoint():
# UNSLOTH_MODEL_IDLE_TTL without auto-switch reloads exactly what it freed, so the
# stored checkpoint stays runnable after an idle unload and the panel must keep
# showing it. The stash restores only that model, so it can never pick catalog[0].
# stored checkpoint stays runnable and the panel must keep showing it. The stash
# restores only that model, so it can never pick catalog[0].
src = USAGE_EXAMPLES_TSX.read_text(encoding = "utf-8")
hook = src[src.find("function useExampleModelName") : src.find("// Backend PATH detection")]
assert "const [idleReload, setIdleReload] = useState(false);" in hook
@ -92,9 +92,9 @@ def test_standalone_idle_unload_still_names_the_stored_checkpoint():
def test_a_failed_refresh_does_not_erase_what_the_server_holds():
# Catching into [] and false made a transient error authoritative: the panel
# dropped a still-servable model and printed "No model" until the next poll.
# The catalog is deliberately tri-state, and a failure must stay the unknown one.
# Catching into [] and false made a transient error authoritative: the panel dropped
# a still-servable model and printed "No model". The catalog is deliberately
# tri-state, and a failure must stay the unknown state.
src = USAGE_EXAMPLES_TSX.read_text(encoding = "utf-8")
hook = src[src.find("function useExampleModelName") : src.find("// Backend PATH detection")]
assert "listOpenAIModels().catch(() => null)" in hook
@ -108,9 +108,9 @@ def test_a_failed_refresh_does_not_erase_what_the_server_holds():
def test_the_pinned_quant_comes_from_the_catalog():
# Catalog membership proves the repo, not the saved quant. The stored one can
# name a file deleted while another quant of the same repo remains, and pinning
# it emitted repo:deleted-quant, a missing-quant 404 with a runnable one listed.
# Catalog membership proves the repo, not the saved quant: the stored one can name
# a file deleted while another quant remains, so pinning it 404d on a missing quant
# with a runnable one listed.
src = USAGE_EXAMPLES_TSX.read_text(encoding = "utf-8")
hook = src[src.find("function useExampleModelName") : src.find("// Backend PATH detection")]
assert "const quant = catalog === null ? ggufVariant : entry?.quant;" in hook