diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py index 2f46470091..edae02e50f 100644 --- a/studio/backend/core/inference/inference.py +++ b/studio/backend/core/inference/inference.py @@ -562,7 +562,11 @@ class InferenceBackend: isinstance(processor, ProcessorMixin) or hasattr(processor, "image_processor") ): # LoRA adapters: use base model. Local merged exports: read base from export_metadata.json. - processor_source = config.base_model if config.is_lora else config.identifier + # Non-LoRA: config.path, not config.identifier. They are the + # same for ordinary loads, but a local-only load rewrites + # path to the cached snapshot and this fallback must stay + # on those local files instead of refetching by repo id. + processor_source = config.base_model if config.is_lora else config.path if not config.is_lora and config.is_local: _meta_path = Path(config.path) / "export_metadata.json" try: diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 50144893e1..e724cd9415 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -5221,6 +5221,7 @@ class LlamaCppBackend: force: bool = False, allow_smaller_fallback: bool = True, cancel_event: Optional[threading.Event] = None, + local_files_only: bool = False, ) -> str: """Download GGUF file(s) from HuggingFace. Returns local path. @@ -5258,16 +5259,19 @@ class LlamaCppBackend: gguf_filename = None gguf_extra_shards: list[str] = [] if hf_variant: - try: - from huggingface_hub import list_repo_files + # Local-only loads resolve from the cache alone: the live listing + # is both network and unnecessary for files already on disk. + if not local_files_only: + try: + from huggingface_hub import list_repo_files - files = list_repo_files(hf_repo, token = hf_token) - gguf_files = _gguf_files_for_variant(files, hf_variant) - if gguf_files: - gguf_filename = gguf_files[0] - gguf_extra_shards = _gguf_extra_shards(gguf_files, gguf_filename) - except Exception as e: - logger.warning(f"Could not list repo files: {e}") + files = list_repo_files(hf_repo, token = hf_token) + gguf_files = _gguf_files_for_variant(files, hf_variant) + if gguf_files: + gguf_filename = gguf_files[0] + gguf_extra_shards = _gguf_extra_shards(gguf_files, gguf_filename) + except Exception as e: + logger.warning(f"Could not list repo files: {e}") # Fall back to the local cache when the repo listing is unavailable. if not gguf_filename: @@ -5309,6 +5313,14 @@ class LlamaCppBackend: logger.info(f"Reusing cached GGUF: {cached_main}") return cached_main + # A local-only load reaching this point has no complete cached copy; + # fail closed instead of downloading (background loads never fetch). + if local_files_only: + raise RuntimeError( + f"GGUF '{hf_repo}' ({hf_variant or 'default'}) is not fully " + "available in the local cache; select it explicitly to download it." + ) + # Check disk space; fall back to a smaller variant if needed all_gguf_files = [gguf_filename] + gguf_extra_shards try: @@ -5471,6 +5483,7 @@ class LlamaCppBackend: label: str, cancel_event: Optional[threading.Event] = None, near_path: Optional[str] = None, + local_files_only: bool = False, ) -> Optional[str]: """Resolve and fetch a companion GGUF (mmproj / MTP drafter) by name. @@ -5492,6 +5505,12 @@ class LlamaCppBackend: logger.info("Reusing cached %s: %s", label, cached) return cached + # Background loads never download: without a cached copy the model + # runs without the optional companion instead of fetching it. + if local_files_only: + logger.info("Skipping %s fetch (local-only load)", label) + return None + from utils.hf_cache_settings import get_hf_cache_paths companion_cache_dir = _hub_cache_dir_for_snapshot_path(near_path) or str( @@ -5580,6 +5599,7 @@ class LlamaCppBackend: hf_token: Optional[str] = None, cancel_event: Optional[threading.Event] = None, near_path: Optional[str] = None, + local_files_only: bool = False, ) -> Optional[str]: """Download the mmproj (vision projection) file from a GGUF repo. @@ -5596,6 +5616,7 @@ class LlamaCppBackend: label = "mmproj", cancel_event = cancel_event, near_path = near_path, + local_files_only = local_files_only, ) def _cached_repo_mtp_drafter( @@ -5637,6 +5658,7 @@ class LlamaCppBackend: hf_repo: str, hf_token: Optional[str] = None, near_path: Optional[str] = None, + local_files_only: bool = False, ) -> Optional[str]: """Download the separate MTP drafter (speculative head) from a GGUF repo. @@ -5670,7 +5692,7 @@ class LlamaCppBackend: # fetched). Online, _download_companion_gguf/hf_hub_download reuse the # current cached file and refetch a changed one, so skip the probe here # rather than pair new weights with a stale draft. - if _hf_env_offline(): + if local_files_only or _hf_env_offline(): cached = self._cached_repo_mtp_drafter( hf_repo, cache_dir = _hub_cache_dir_for_snapshot_path(near_path), @@ -5685,6 +5707,7 @@ class LlamaCppBackend: pick = _pick_mtp, label = "MTP drafter", near_path = near_path, + local_files_only = local_files_only, ) def _resolve_launch_mmproj_path( @@ -6381,6 +6404,9 @@ class LlamaCppBackend: hf_repo: Optional[str] = None, hf_variant: Optional[str] = None, hf_token: Optional[str] = None, + # Background auto-loads: resolve every file (main quant, mmproj, MTP + # drafter) from the local cache only and never download. + local_files_only: bool = False, # Common model_identifier: str, is_vision: bool = False, @@ -6418,6 +6444,7 @@ class LlamaCppBackend: "gguf_path": gguf_path, "mmproj_path": mmproj_path, "mtp_draft_path": mtp_draft_path, + "local_files_only": local_files_only, "hf_repo": hf_repo, "hf_variant": hf_variant, "hf_token": hf_token, @@ -6575,6 +6602,7 @@ class LlamaCppBackend: hf_repo = hf_repo, hf_variant = hf_variant, hf_token = hf_token, + local_files_only = local_files_only, ) # Auto-download mmproj for vision models unless opted out. if is_vision and not mmproj_path and not extra_args_disable_mmproj(extra_args): @@ -6582,6 +6610,7 @@ class LlamaCppBackend: hf_repo = hf_repo, hf_token = hf_token, near_path = model_path, + local_files_only = local_files_only, ) # Auto-download the separate MTP drafter (e.g. Gemma) when # the requested spec mode can use it. Repos with the head @@ -6600,6 +6629,7 @@ class LlamaCppBackend: hf_repo = hf_repo, hf_token = hf_token, near_path = model_path, + local_files_only = local_files_only, ) elif gguf_path: if not Path(gguf_path).is_file(): diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 0769a687d3..6a085e4ac1 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -4508,9 +4508,17 @@ async def _load_model_impl( if request.local_files_only and not config.is_gguf and not is_local_path(config.path): from hub.utils.local_snapshot import resolve_local_snapshot_path + from utils.hf_cache_settings import get_hf_cache_paths + # Resolve against the LIVE Studio-managed hub cache: Studio can + # move the cache at runtime without rewriting Hugging Face's + # import-time env constants, so the helper's default would search + # the stale location and 409 models the inventory just found. local_snapshot = await asyncio.to_thread( - resolve_local_snapshot_path, config.path, request.hf_token + resolve_local_snapshot_path, + config.path, + request.hf_token, + str(get_hf_cache_paths().hub_cache), ) if local_snapshot is None: raise HTTPException( @@ -4664,6 +4672,10 @@ async def _load_model_impl( hf_repo = config.gguf_hf_repo, hf_variant = config.gguf_variant, hf_token = request.hf_token, + # Background auto-loads never download: main quant resolves + # cache-only and the optional mmproj/MTP companions are + # cached-or-skipped instead of fetched. + local_files_only = request.local_files_only, ) else: # Local mode: llama-server loads via -m diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index d707c9d345..26aa8ec2e5 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -1072,3 +1072,40 @@ def test_background_loads_resolve_local_files_only(): helper = _read_backend("hub/utils/local_snapshot.py") assert "local_files_only = True" in helper + # The rewrite resolves against the LIVE Studio-managed cache location, + # not huggingface_hub's import-time default. + assert "str(get_hf_cache_paths().hub_cache)," in route + + +def test_gguf_background_loads_never_download_companions(): + """A cached GGUF load can still fetch from the Hub through its optional + companions (mmproj, MTP drafter) or a cache-miss main quant. Background + loads pass local_files_only into the llama.cpp path: companions resolve + cached-or-skipped, and a main-quant cache miss raises instead of + downloading.""" + route = _read_backend("routes/inference.py") + gguf_source = route.split("if config.gguf_hf_repo:", 1)[1] + gguf_source = gguf_source.split("else:", 1)[0] + assert "local_files_only = request.local_files_only," in gguf_source + + llama = _read_backend("core/inference/llama_cpp.py") + # The flag flows to the main quant and both companion helpers, and the + # crash-replay kwargs keep it so a reload stays local-only. + assert llama.count("local_files_only = local_files_only,") >= 3 + assert '"local_files_only": local_files_only,' in llama + # Companions: cached-or-skipped, never fetched. + assert 'logger.info("Skipping %s fetch (local-only load)", label)' in llama + assert "if local_files_only or _hf_env_offline():" in llama + # Main quant: a cache miss fails closed. + download = llama.split("def _download_gguf", 1)[1] + download = download.split("def _download_companion_gguf", 1)[0] + assert "if local_files_only:" in download + assert "select it explicitly to download it." in download + + inference = _read_backend("core/inference/inference.py") + # The vision processor fallback stays on the (possibly rewritten local) + # load path instead of refetching by repo id. + assert ( + "config.base_model if config.is_lora else config.path" in inference + ) + assert "config.base_model if config.is_lora else config.identifier" not in inference