ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Studio autoload: extend local-only enforcement to GGUF companions, live cache dir, processor fallback

Browse source 
Background GGUF loads pass local_files_only through the llama.cpp path:
the main quant resolves from the cache alone and a miss fails closed
instead of downloading, while the optional mmproj and MTP drafter
companions resolve cached-or-skipped, including on the crash replay. The
safetensors snapshot rewrite resolves against the live Studio-managed
hub cache rather than huggingface_hub's import-time default, which goes
stale when the cache location changes at runtime. The vision processor
fallback loads from config.path instead of config.identifier: identical
for ordinary loads, but a local-only rewrite must keep the fallback on
the local snapshot instead of refetching by repo id.
This commit is contained in:
Unsloth 2026-07-26 20:49:19 -07:00
commit 90f9e97302
4 changed files with 95 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

6
studio/backend/core/inference/inference.py
View file

@ -562,7 +562,11 @@ class InferenceBackend:
isinstance(processor, ProcessorMixin) or hasattr(processor, "image_processor")
):
# LoRA adapters: use base model. Local merged exports: read base from export_metadata.json.
processor_source = config.base_model if config.is_lora else config.identifier
# Non-LoRA: config.path, not config.identifier. They are the
# same for ordinary loads, but a local-only load rewrites
# path to the cached snapshot and this fallback must stay
# on those local files instead of refetching by repo id.
processor_source = config.base_model if config.is_lora else config.path
if not config.is_lora and config.is_local:
_meta_path = Path(config.path) / "export_metadata.json"
try:

50
studio/backend/core/inference/llama_cpp.py
View file

@ -5221,6 +5221,7 @@ class LlamaCppBackend:
force: bool = False,
allow_smaller_fallback: bool = True,
cancel_event: Optional[threading.Event] = None,
local_files_only: bool = False,
) -> str:
"""Download GGUF file(s) from HuggingFace. Returns local path.
@ -5258,16 +5259,19 @@ class LlamaCppBackend:
gguf_filename = None
gguf_extra_shards: list[str] = []
if hf_variant:
try:
from huggingface_hub import list_repo_files
# Local-only loads resolve from the cache alone: the live listing
# is both network and unnecessary for files already on disk.
if not local_files_only:
try:
from huggingface_hub import list_repo_files
files = list_repo_files(hf_repo, token = hf_token)
gguf_files = _gguf_files_for_variant(files, hf_variant)
if gguf_files:
gguf_filename = gguf_files[0]
gguf_extra_shards = _gguf_extra_shards(gguf_files, gguf_filename)
except Exception as e:
logger.warning(f"Could not list repo files: {e}")
files = list_repo_files(hf_repo, token = hf_token)
gguf_files = _gguf_files_for_variant(files, hf_variant)
if gguf_files:
gguf_filename = gguf_files[0]
gguf_extra_shards = _gguf_extra_shards(gguf_files, gguf_filename)
except Exception as e:
logger.warning(f"Could not list repo files: {e}")
# Fall back to the local cache when the repo listing is unavailable.
if not gguf_filename:
@ -5309,6 +5313,14 @@ class LlamaCppBackend:
logger.info(f"Reusing cached GGUF: {cached_main}")
return cached_main
# A local-only load reaching this point has no complete cached copy;
# fail closed instead of downloading (background loads never fetch).
if local_files_only:
raise RuntimeError(
f"GGUF '{hf_repo}' ({hf_variant or 'default'}) is not fully "
"available in the local cache; select it explicitly to download it."
)
# Check disk space; fall back to a smaller variant if needed
all_gguf_files = [gguf_filename] + gguf_extra_shards
try:
@ -5471,6 +5483,7 @@ class LlamaCppBackend:
label: str,
cancel_event: Optional[threading.Event] = None,
near_path: Optional[str] = None,
local_files_only: bool = False,
) -> Optional[str]:
"""Resolve and fetch a companion GGUF (mmproj / MTP drafter) by name.
@ -5492,6 +5505,12 @@ class LlamaCppBackend:
logger.info("Reusing cached %s: %s", label, cached)
return cached
# Background loads never download: without a cached copy the model
# runs without the optional companion instead of fetching it.
if local_files_only:
logger.info("Skipping %s fetch (local-only load)", label)
return None
from utils.hf_cache_settings import get_hf_cache_paths
companion_cache_dir = _hub_cache_dir_for_snapshot_path(near_path) or str(
@ -5580,6 +5599,7 @@ class LlamaCppBackend:
hf_token: Optional[str] = None,
cancel_event: Optional[threading.Event] = None,
near_path: Optional[str] = None,
local_files_only: bool = False,
) -> Optional[str]:
"""Download the mmproj (vision projection) file from a GGUF repo.
@ -5596,6 +5616,7 @@ class LlamaCppBackend:
label = "mmproj",
cancel_event = cancel_event,
near_path = near_path,
local_files_only = local_files_only,
)
def _cached_repo_mtp_drafter(
@ -5637,6 +5658,7 @@ class LlamaCppBackend:
hf_repo: str,
hf_token: Optional[str] = None,
near_path: Optional[str] = None,
local_files_only: bool = False,
) -> Optional[str]:
"""Download the separate MTP drafter (speculative head) from a GGUF repo.
@ -5670,7 +5692,7 @@ class LlamaCppBackend:
# fetched). Online, _download_companion_gguf/hf_hub_download reuse the
# current cached file and refetch a changed one, so skip the probe here
# rather than pair new weights with a stale draft.
if _hf_env_offline():
if local_files_only or _hf_env_offline():
cached = self._cached_repo_mtp_drafter(
hf_repo,
cache_dir = _hub_cache_dir_for_snapshot_path(near_path),
@ -5685,6 +5707,7 @@ class LlamaCppBackend:
pick = _pick_mtp,
label = "MTP drafter",
near_path = near_path,
local_files_only = local_files_only,
)
def _resolve_launch_mmproj_path(
@ -6381,6 +6404,9 @@ class LlamaCppBackend:
hf_repo: Optional[str] = None,
hf_variant: Optional[str] = None,
hf_token: Optional[str] = None,
# Background auto-loads: resolve every file (main quant, mmproj, MTP
# drafter) from the local cache only and never download.
local_files_only: bool = False,
# Common
model_identifier: str,
is_vision: bool = False,
@ -6418,6 +6444,7 @@ class LlamaCppBackend:
"gguf_path": gguf_path,
"mmproj_path": mmproj_path,
"mtp_draft_path": mtp_draft_path,
"local_files_only": local_files_only,
"hf_repo": hf_repo,
"hf_variant": hf_variant,
"hf_token": hf_token,
@ -6575,6 +6602,7 @@ class LlamaCppBackend:
hf_repo = hf_repo,
hf_variant = hf_variant,
hf_token = hf_token,
local_files_only = local_files_only,
)
# Auto-download mmproj for vision models unless opted out.
if is_vision and not mmproj_path and not extra_args_disable_mmproj(extra_args):
@ -6582,6 +6610,7 @@ class LlamaCppBackend:
hf_repo = hf_repo,
hf_token = hf_token,
near_path = model_path,
local_files_only = local_files_only,
)
# Auto-download the separate MTP drafter (e.g. Gemma) when
# the requested spec mode can use it. Repos with the head
@ -6600,6 +6629,7 @@ class LlamaCppBackend:
hf_repo = hf_repo,
hf_token = hf_token,
near_path = model_path,
local_files_only = local_files_only,
)
elif gguf_path:
if not Path(gguf_path).is_file():

14
studio/backend/routes/inference.py
View file

@ -4508,9 +4508,17 @@ async def _load_model_impl(
if request.local_files_only and not config.is_gguf and not is_local_path(config.path):
from hub.utils.local_snapshot import resolve_local_snapshot_path
from utils.hf_cache_settings import get_hf_cache_paths
# Resolve against the LIVE Studio-managed hub cache: Studio can
# move the cache at runtime without rewriting Hugging Face's
# import-time env constants, so the helper's default would search
# the stale location and 409 models the inventory just found.
local_snapshot = await asyncio.to_thread(
resolve_local_snapshot_path, config.path, request.hf_token
resolve_local_snapshot_path,
config.path,
request.hf_token,
str(get_hf_cache_paths().hub_cache),
)
if local_snapshot is None:
raise HTTPException(
@ -4664,6 +4672,10 @@ async def _load_model_impl(
hf_repo = config.gguf_hf_repo,
hf_variant = config.gguf_variant,
hf_token = request.hf_token,
# Background auto-loads never download: main quant resolves
# cache-only and the optional mmproj/MTP companions are
# cached-or-skipped instead of fetched.
local_files_only = request.local_files_only,
)
else:
# Local mode: llama-server loads via -m <path>

37
tests/studio/test_model_picker_contracts.py
View file

@ -1072,3 +1072,40 @@ def test_background_loads_resolve_local_files_only():
helper = _read_backend("hub/utils/local_snapshot.py")
assert "local_files_only = True" in helper
# The rewrite resolves against the LIVE Studio-managed cache location,
# not huggingface_hub's import-time default.
assert "str(get_hf_cache_paths().hub_cache)," in route
def test_gguf_background_loads_never_download_companions():
"""A cached GGUF load can still fetch from the Hub through its optional
companions (mmproj, MTP drafter) or a cache-miss main quant. Background
loads pass local_files_only into the llama.cpp path: companions resolve
cached-or-skipped, and a main-quant cache miss raises instead of
downloading."""
route = _read_backend("routes/inference.py")
gguf_source = route.split("if config.gguf_hf_repo:", 1)[1]
gguf_source = gguf_source.split("else:", 1)[0]
assert "local_files_only = request.local_files_only," in gguf_source
llama = _read_backend("core/inference/llama_cpp.py")
# The flag flows to the main quant and both companion helpers, and the
# crash-replay kwargs keep it so a reload stays local-only.
assert llama.count("local_files_only = local_files_only,") >= 3
assert '"local_files_only": local_files_only,' in llama
# Companions: cached-or-skipped, never fetched.
assert 'logger.info("Skipping %s fetch (local-only load)", label)' in llama
assert "if local_files_only or _hf_env_offline():" in llama
# Main quant: a cache miss fails closed.
download = llama.split("def _download_gguf", 1)[1]
download = download.split("def _download_companion_gguf", 1)[0]
assert "if local_files_only:" in download
assert "select it explicitly to download it." in download
inference = _read_backend("core/inference/inference.py")
# The vision processor fallback stays on the (possibly rewritten local)
# load path instead of refetching by repo id.
assert (
"config.base_model if config.is_lora else config.path" in inference
)
assert "config.base_model if config.is_lora else config.identifier" not in inference