From adbf88d8b66e16eb5c8c0c198933ae935238d1b1 Mon Sep 17 00:00:00 2001 From: Unsloth Date: Tue, 28 Jul 2026 21:38:08 -0700 Subject: [PATCH 01/10] Offline: detect an unreachable hub, not just dead DNS Loading an already-downloaded model with no internet took 11 minutes. The offline guard only checked whether huggingface.co resolved, so the common offline shapes where DNS still answers (WAN down behind a live router, captive portal, stale DNS cache) were treated as online and every hub call burned its full retry backoff. Two fixes: - Escalate from the DNS check to the bounded, proxy-aware reachability probe already used by export, memoised for 60s and opt-outable with UNSLOTH_OFFLINE_PROBE=0. - Force offline in-process, not just via env vars. huggingface_hub and transformers read their offline constants at import and hub sessions cache a non-offline adapter, so setting the env mid-process left the calls retrying anyway. Also guards the metadata routes that had none (/models/config, /models/check-vision, /picker/chat-template, the per-request vision probe) and applies the same detection in the training worker. Measured on a cached GGUF repo with the endpoint blackholed: POST /inference/load 686s -> 4s GET /models/config 378s -> 0s GET /models/check-vision 28s -> 0s Online is unchanged: reachable endpoints skip the guard entirely, and a fresh download still resolves, downloads and loads normally. --- studio/backend/core/inference/llama_cpp.py | 58 +++++-- studio/backend/core/training/trainer.py | 2 +- studio/backend/core/training/worker.py | 15 +- studio/backend/picker/routes/templates.py | 12 +- studio/backend/routes/inference.py | 11 +- studio/backend/routes/models.py | 131 ++++++++-------- .../backend/tests/test_active_generations.py | 6 +- .../tests/test_chat_load_during_training.py | 2 +- .../tests/test_gguf_load_cache_reuse.py | 2 +- studio/backend/tests/test_gpu_memory_mode.py | 4 +- studio/backend/tests/test_gpu_selection.py | 10 +- .../tests/test_offline_gguf_cache_fallback.py | 119 ++++++++++++-- studio/backend/utils/utils.py | 146 ++++++++++++++++++ 13 files changed, 407 insertions(+), 111 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 144aa1fd37..47917004f9 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -527,29 +527,63 @@ def _hf_env_offline() -> bool: return os.environ.get("HF_HUB_OFFLINE", "").strip().lower() in {"1", "true", "yes", "on"} +def _hf_unreachable() -> bool: + """True when the Hub can't be reached: dead DNS, or DNS that resolves with no egress. + + DNS alone misses the common offline shapes (WAN down behind a live router, captive + portal, stale DNS cache), leaving every hub call to burn its full retry backoff. + """ + if _probe_dns_dead(): + return True + try: + from utils.utils import hf_unreachable + return hf_unreachable() + except Exception: + return False + + @contextlib.contextmanager -def _hf_offline_if_dns_dead(): - """Set HF_HUB_OFFLINE for this block only when DNS to huggingface.co fails; +def _hf_offline_if_unreachable(): + """Set HF_HUB_OFFLINE for this block only when the Hub is unreachable; restores env on exit so a transient hiccup can't quarantine the process. No-op if the user already set it.""" if "HF_HUB_OFFLINE" in os.environ: yield False return - if not _probe_dns_dead(): + if not _hf_unreachable(): yield False return + logger.warning("Hugging Face endpoint unreachable; using local HF cache for this load.") + + # Env vars alone do not take effect mid-process: huggingface_hub and transformers read + # their offline constants at import, so the calls below would still retry. + force_ctx = None + try: + from utils.utils import force_hf_offline + + force_ctx = force_hf_offline() + force_ctx.__enter__() + except Exception: + force_ctx = None + transformers_was_set = "TRANSFORMERS_OFFLINE" in os.environ - os.environ["HF_HUB_OFFLINE"] = "1" - if not transformers_was_set: - os.environ["TRANSFORMERS_OFFLINE"] = "1" - logger.warning("huggingface.co unreachable; using local HF cache for this load.") + if force_ctx is None: + os.environ["HF_HUB_OFFLINE"] = "1" + if not transformers_was_set: + os.environ["TRANSFORMERS_OFFLINE"] = "1" try: yield True finally: - os.environ.pop("HF_HUB_OFFLINE", None) - if not transformers_was_set: - os.environ.pop("TRANSFORMERS_OFFLINE", None) + if force_ctx is not None: + try: + force_ctx.__exit__(None, None, None) + except Exception: + pass + else: + os.environ.pop("HF_HUB_OFFLINE", None) + if not transformers_was_set: + os.environ.pop("TRANSFORMERS_OFFLINE", None) try: @@ -6889,7 +6923,7 @@ class LlamaCppBackend: hf_repo, ) hf_repo = _resolved_repo - with _hf_offline_if_dns_dead(): + with _hf_offline_if_unreachable(): _preflight_model_path = self._download_gguf( hf_repo = hf_repo, hf_variant = hf_variant, @@ -6931,7 +6965,7 @@ class LlamaCppBackend: hf_repo, ) hf_repo = _resolved_repo - with _hf_offline_if_dns_dead(): + with _hf_offline_if_unreachable(): model_path = _preflight_model_path or self._download_gguf( hf_repo = hf_repo, hf_variant = hf_variant, diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py index b858fe6f17..daee45950f 100644 --- a/studio/backend/core/training/trainer.py +++ b/studio/backend/core/training/trainer.py @@ -68,7 +68,7 @@ import pandas as pd from datasets import Dataset from utils.datasets.cache_safe import load_dataset_cache_safe as load_dataset -from core.inference.llama_cpp import _hf_offline_if_dns_dead +from core.inference.llama_cpp import _hf_offline_if_unreachable from utils.models import is_vision_model, detect_audio_type from utils.models.model_config import _env_offline from utils.datasets import format_and_template_dataset diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py index baf6329dae..23b5c4a371 100644 --- a/studio/backend/core/training/worker.py +++ b/studio/backend/core/training/worker.py @@ -2344,7 +2344,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) -> flush = True, ) - # Offline auto-detect: skip ~25s of HF retries per call when DNS is dead. + # Offline auto-detect: skip ~25s of HF retries per call when the hub is unreachable. if "HF_HUB_OFFLINE" not in os.environ: import socket as _socket import threading as _threading @@ -2362,13 +2362,24 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) -> _t = _threading.Thread(target = _probe, daemon = True) _t.start() _t.join(2.0) + if _result[0] is False: + # DNS answers even when there is no egress (WAN down, captive portal), so + # confirm with the bounded, proxy-aware reachability probe. HF_ENDPOINT aware. + try: + from utils.transformers_version import hf_endpoint_unreachable + from utils.utils import hf_probe_disabled + + if not hf_probe_disabled() and hf_endpoint_unreachable(): + _result[0] = True + except Exception: + pass if _result[0] is None or _result[0] is True: os.environ["HF_HUB_OFFLINE"] = "1" os.environ.setdefault("TRANSFORMERS_OFFLINE", "1") os.environ.setdefault("HF_DATASETS_OFFLINE", "1") # logger isn't configured yet; print to stderr instead. print( - "huggingface.co unreachable; HF_HUB_OFFLINE=1 set for this worker.", + "Hugging Face endpoint unreachable; HF_HUB_OFFLINE=1 set for this worker.", file = sys.stderr, flush = True, ) diff --git a/studio/backend/picker/routes/templates.py b/studio/backend/picker/routes/templates.py index 02b8bf7184..a070c8d054 100644 --- a/studio/backend/picker/routes/templates.py +++ b/studio/backend/picker/routes/templates.py @@ -37,9 +37,15 @@ async def get_default_chat_template_route( hf_token: Optional[str] = Depends(get_hf_token), current_subject: str = Depends(get_current_subject), ) -> ModelTemplateResponse: - template = await asyncio.to_thread( - read_default_chat_template, model_name, hf_token, gguf_variant - ) + # Cached repos resolve from disk, but a cache miss falls through to the hub and + # offline that costs one retry backoff per candidate template file. + from core.inference.llama_cpp import _hf_offline_if_unreachable + + def _read(): + with _hf_offline_if_unreachable(): + return read_default_chat_template(model_name, hf_token, gguf_variant) + + template = await asyncio.to_thread(_read) if template is not None and len(template.encode("utf-8")) > MAX_CHAT_TEMPLATE_BYTES: template = None return ModelTemplateResponse(model_name = model_name, chat_template = template) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index d0a2d97f74..b317133d5c 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1007,7 +1007,7 @@ try: _canonicalize_spec_mode, _extra_args_n_ubatch, _extra_args_set_spec_type, - _hf_offline_if_dns_dead, + _hf_offline_if_unreachable, _kv_bytes_per_elem, _kv_unified_from_args, _planned_main_cache_types, @@ -1051,7 +1051,7 @@ except ImportError: _canonicalize_spec_mode, _extra_args_n_ubatch, _extra_args_set_spec_type, - _hf_offline_if_dns_dead, + _hf_offline_if_unreachable, _kv_bytes_per_elem, _kv_unified_from_args, _planned_main_cache_types, @@ -3672,7 +3672,10 @@ def _target_is_vision(load_path: str) -> bool: # paths, where the token is unused, but the rule requires it regardless). from utils.models.model_config import is_vision_model try: - return bool(is_vision_model(load_path, hf_token = os.environ.get("HF_TOKEN"))) + # Guarded: this runs per request, so an unreachable hub would re-pay its retry + # backoff on every image/audio call. + with _hf_offline_if_unreachable(): + return bool(is_vision_model(load_path, hf_token = os.environ.get("HF_TOKEN"))) except Exception as exc: # Detection failure: don't block the swap, let the load decide. logger.debug("auto-switch: vision probe failed for %s: %s", load_path, exc) @@ -5467,7 +5470,7 @@ async def _load_model_impl( # is_lora auto-detected from adapter_config.json on disk/HF. # DNS-probe wrap so offline loads skip 30-60s of soft-failed network # checks before the worker starts. - with _hf_offline_if_dns_dead(): + with _hf_offline_if_unreachable(): config = ModelConfig.from_identifier( model_id = model_identifier, hf_token = request.hf_token, diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py index 96c5b96d73..cc2a402b59 100644 --- a/studio/backend/routes/models.py +++ b/studio/backend/routes/models.py @@ -1855,75 +1855,80 @@ async def get_model_config( ): """Get configuration for a specific model (wraps load_model_defaults).""" hf_token = _normalize_hf_token(header_hf_token) or _normalize_hf_token(hf_token) + from core.inference.llama_cpp import _hf_offline_if_unreachable + try: - if not is_local_path(model_name): - resolved = resolve_cached_repo_id_case(model_name) - if resolved != model_name: - logger.info( - "Using cached repo_id casing '%s' for requested '%s'", - resolved, - model_name, - ) - model_name = resolved + # Each probe below can reach the hub, so the guard wraps the whole handler: + # offline they must all resolve from the HF cache instead of retrying. + with _hf_offline_if_unreachable(): + if not is_local_path(model_name): + resolved = resolve_cached_repo_id_case(model_name) + if resolved != model_name: + logger.info( + "Using cached repo_id casing '%s' for requested '%s'", + resolved, + model_name, + ) + model_name = resolved - logger.info(f"Getting model config for: {model_name}") - from utils.models.model_config import detect_audio_type + logger.info(f"Getting model config for: {model_name}") + from utils.models.model_config import detect_audio_type - config_dict = load_model_defaults(model_name) + config_dict = load_model_defaults(model_name) - # Detect capabilities (HF token for gated models). - is_vision = is_vision_model(model_name, hf_token = hf_token) - is_embedding = is_embedding_model(model_name, hf_token = hf_token) - audio_type = detect_audio_type(model_name, hf_token = hf_token) + # Detect capabilities (HF token for gated models). + is_vision = is_vision_model(model_name, hf_token = hf_token) + is_embedding = is_embedding_model(model_name, hf_token = hf_token) + audio_type = detect_audio_type(model_name, hf_token = hf_token) - is_lora = False - base_model = None - max_position_embeddings = None - try: - model_config = ModelConfig.from_identifier(model_name) - is_lora = model_config.is_lora - base_model = model_config.base_model if is_lora else None - max_position_embeddings = _get_max_position_embeddings(model_config) - except Exception: - pass - - # Fallback: read raw config.json (declarative fields only) -- a selection-time - # metadata probe that must never execute a repo's auto_map Python. - if max_position_embeddings is None: + is_lora = False + base_model = None + max_position_embeddings = None try: - from utils.transformers_version import _load_config_json - from types import SimpleNamespace - - _cfg = _load_config_json(model_name, hf_token = hf_token) - if _cfg is not None: - - def _to_ns(d): - if isinstance(d, dict): - return SimpleNamespace(**{k: _to_ns(v) for k, v in d.items()}) - return d - - max_position_embeddings = _get_max_position_embeddings(_to_ns(_cfg)) + model_config = ModelConfig.from_identifier(model_name) + is_lora = model_config.is_lora + base_model = model_config.base_model if is_lora else None + max_position_embeddings = _get_max_position_embeddings(model_config) except Exception: pass - logger.info( - f"Model config result for {model_name}: is_vision={is_vision}, is_embedding={is_embedding}, audio_type={audio_type}, is_lora={is_lora}, max_position_embeddings={max_position_embeddings}" - ) - return ModelDetails( - id = model_name, - model_name = model_name, - config = config_dict, - is_vision = is_vision, - is_embedding = is_embedding, - is_lora = is_lora, - is_audio = audio_type is not None, - audio_type = audio_type, - has_audio_input = is_audio_input_type(audio_type), - model_type = derive_model_type(is_vision, audio_type, is_embedding), - base_model = base_model, - max_position_embeddings = max_position_embeddings, - model_size_bytes = _get_model_size_bytes(model_name, hf_token), - ) + # Fallback: read raw config.json (declarative fields only) -- a selection-time + # metadata probe that must never execute a repo's auto_map Python. + if max_position_embeddings is None: + try: + from utils.transformers_version import _load_config_json + from types import SimpleNamespace + + _cfg = _load_config_json(model_name, hf_token = hf_token) + if _cfg is not None: + + def _to_ns(d): + if isinstance(d, dict): + return SimpleNamespace(**{k: _to_ns(v) for k, v in d.items()}) + return d + + max_position_embeddings = _get_max_position_embeddings(_to_ns(_cfg)) + except Exception: + pass + + logger.info( + f"Model config result for {model_name}: is_vision={is_vision}, is_embedding={is_embedding}, audio_type={audio_type}, is_lora={is_lora}, max_position_embeddings={max_position_embeddings}" + ) + return ModelDetails( + id = model_name, + model_name = model_name, + config = config_dict, + is_vision = is_vision, + is_embedding = is_embedding, + is_lora = is_lora, + is_audio = audio_type is not None, + audio_type = audio_type, + has_audio_input = is_audio_input_type(audio_type), + model_type = derive_model_type(is_vision, audio_type, is_embedding), + base_model = base_model, + max_position_embeddings = max_position_embeddings, + model_size_bytes = _get_model_size_bytes(model_name, hf_token), + ) except Exception as e: raise log_and_http_error( @@ -2585,7 +2590,11 @@ async def check_vision_model( try: logger.info(f"Checking if vision model: {model_name}") # Authenticate so a gated/private VLM classifies correctly (else 404 -> non-vision). - is_vision = is_vision_model(model_name, hf_token = hf_token) + # Offline the guard keeps this on the HF cache instead of retrying the hub. + from core.inference.llama_cpp import _hf_offline_if_unreachable + + with _hf_offline_if_unreachable(): + is_vision = is_vision_model(model_name, hf_token = hf_token) logger.info(f"Vision check result for {model_name}: is_vision={is_vision}") return VisionCheckResponse( diff --git a/studio/backend/tests/test_active_generations.py b/studio/backend/tests/test_active_generations.py index aa087fe4ea..2f86f8efdc 100644 --- a/studio/backend/tests/test_active_generations.py +++ b/studio/backend/tests/test_active_generations.py @@ -344,7 +344,7 @@ def test_a_forced_load_that_fails_preflight_leaves_the_chats_alone(monkeypatch): from models.inference import LoadRequest inf_mod = _stub_load_route(monkeypatch, active_model_name = "org/OTHER") - monkeypatch.setattr(inf_mod, "_hf_offline_if_dns_dead", contextlib.nullcontext) + monkeypatch.setattr(inf_mod, "_hf_offline_if_unreachable", contextlib.nullcontext) # Stands in for any preflight refusal; a None here is the route's own 400. monkeypatch.setattr(inf_mod.ModelConfig, "from_identifier", staticmethod(lambda **kwargs: None)) @@ -375,7 +375,7 @@ def _stub_standard_load_route(monkeypatch): _stub_load_route(monkeypatch, active_model_name = "org/OTHER") # _stub_load_route neutralises the sidecar guard; this test is about it. monkeypatch.setattr(inf_mod, "_raise_if_sidecar_swap_in_progress", real_sidecar_check) - monkeypatch.setattr(inf_mod, "_hf_offline_if_dns_dead", contextlib.nullcontext) + monkeypatch.setattr(inf_mod, "_hf_offline_if_unreachable", contextlib.nullcontext) monkeypatch.setattr(inf_mod, "_mlx_distributed_launch_detected", lambda: False) monkeypatch.setattr( inf_mod.ModelConfig, @@ -1582,7 +1582,7 @@ def test_a_forced_load_that_loses_to_a_sidecar_install_leaves_the_chats_alone(mo from models.inference import LoadRequest inf_mod = _stub_load_route(monkeypatch, active_model_name = "org/OTHER") - monkeypatch.setattr(inf_mod, "_hf_offline_if_dns_dead", contextlib.nullcontext) + monkeypatch.setattr(inf_mod, "_hf_offline_if_unreachable", contextlib.nullcontext) monkeypatch.setattr( inf_mod.ModelConfig, "from_identifier", diff --git a/studio/backend/tests/test_chat_load_during_training.py b/studio/backend/tests/test_chat_load_during_training.py index 6ec9c44e88..1ead0afe4b 100644 --- a/studio/backend/tests/test_chat_load_during_training.py +++ b/studio/backend/tests/test_chat_load_during_training.py @@ -1163,7 +1163,7 @@ class TestLoadModelGuardIntegration(unittest.TestCase): patch.object(self.route, "resolve_effective_chat_template_override", return_value = None), patch.object(self.route, "get_inference_backend", return_value = inf), patch.object(self.route, "get_llama_cpp_backend", return_value = llama), - patch.object(self.route, "_hf_offline_if_dns_dead", lambda: contextlib.nullcontext()), + patch.object(self.route, "_hf_offline_if_unreachable", lambda: contextlib.nullcontext()), patch.object(self.route.ModelConfig, "from_identifier", return_value = cfg), _stub_guard_deps(training_active = True, decision = (False, info)), ): diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.py index ccbe50bcb9..8b68af885a 100644 --- a/studio/backend/tests/test_gguf_load_cache_reuse.py +++ b/studio/backend/tests/test_gguf_load_cache_reuse.py @@ -914,7 +914,7 @@ class TestLoadHubDownloadExclusion: patch.object(route, "_resolve_gguf_gpu_ids_for_request", _no_gguf_gpu_ids), patch.object(route, "_guard_chat_load_against_training", return_value = None), patch.object(route, "_effective_load_in_4bit", return_value = False), - patch.object(route, "_hf_offline_if_dns_dead", nullcontext), + patch.object(route, "_hf_offline_if_unreachable", nullcontext), patch.object(route.asyncio, "to_thread", new = _inline_to_thread), patch.object(llama_cpp_module, "_hub_download_blocks_gguf_load", _fake_blocks), ): diff --git a/studio/backend/tests/test_gpu_memory_mode.py b/studio/backend/tests/test_gpu_memory_mode.py index 43365bd3ca..354ce1dccc 100644 --- a/studio/backend/tests/test_gpu_memory_mode.py +++ b/studio/backend/tests/test_gpu_memory_mode.py @@ -737,7 +737,7 @@ def test_remote_vulkan_diffusion_rejection_keeps_active_server(monkeypatch): ) monkeypatch.setattr( llama_cpp_module, - "_hf_offline_if_dns_dead", + "_hf_offline_if_unreachable", lambda: __import__("contextlib").nullcontext(), ) @@ -786,7 +786,7 @@ def test_remote_vulkan_preflight_download_failure_keeps_active_server(monkeypatc monkeypatch.setattr(llama_cpp_module, "_resolve_repo_id_casing", lambda repo: repo) monkeypatch.setattr( llama_cpp_module, - "_hf_offline_if_dns_dead", + "_hf_offline_if_unreachable", lambda: __import__("contextlib").nullcontext(), ) diff --git a/studio/backend/tests/test_gpu_selection.py b/studio/backend/tests/test_gpu_selection.py index 7999eb4f73..93dff5cbd5 100644 --- a/studio/backend/tests/test_gpu_selection.py +++ b/studio/backend/tests/test_gpu_selection.py @@ -1025,7 +1025,7 @@ class TestRouteErrors(unittest.TestCase): return_value = None, ), patch.object(inference_route.asyncio, "to_thread", new = _inline_to_thread), - patch.object(inference_route, "_hf_offline_if_dns_dead", nullcontext), + patch.object(inference_route, "_hf_offline_if_unreachable", nullcontext), ): with self.assertRaises(HTTPException) as exc_info: asyncio.run( @@ -1095,7 +1095,7 @@ class TestRouteErrors(unittest.TestCase): return_value = None, ) as training_guard, patch.object(inference_route.asyncio, "to_thread", new = _inline_to_thread), - patch.object(inference_route, "_hf_offline_if_dns_dead", nullcontext), + patch.object(inference_route, "_hf_offline_if_unreachable", nullcontext), ): with self.assertRaises(HTTPException) as exc_info: asyncio.run( @@ -1194,7 +1194,7 @@ class TestRouteErrors(unittest.TestCase): return_value = None, ), patch.object(inference_route.asyncio, "to_thread", new = _inline_to_thread), - patch.object(inference_route, "_hf_offline_if_dns_dead", nullcontext), + patch.object(inference_route, "_hf_offline_if_unreachable", nullcontext), ): with self.assertRaises(HTTPException) as exc_info: asyncio.run( @@ -1340,7 +1340,7 @@ class TestRouteErrors(unittest.TestCase): return_value = None, ), patch.object(inference_route.asyncio, "to_thread", new = _inline_to_thread), - patch.object(inference_route, "_hf_offline_if_dns_dead", nullcontext), + patch.object(inference_route, "_hf_offline_if_unreachable", nullcontext), patch( "core.export.get_export_backend", return_value = SimpleNamespace(current_checkpoint = None), @@ -1411,7 +1411,7 @@ class TestRouteErrors(unittest.TestCase): return_value = None, ), patch.object(inference_route.asyncio, "to_thread", new = _inline_to_thread), - patch.object(inference_route, "_hf_offline_if_dns_dead", nullcontext), + patch.object(inference_route, "_hf_offline_if_unreachable", nullcontext), patch( "core.export.get_export_backend", return_value = SimpleNamespace(current_checkpoint = None), diff --git a/studio/backend/tests/test_offline_gguf_cache_fallback.py b/studio/backend/tests/test_offline_gguf_cache_fallback.py index d1e61d0546..be9c92eb3a 100644 --- a/studio/backend/tests/test_offline_gguf_cache_fallback.py +++ b/studio/backend/tests/test_offline_gguf_cache_fallback.py @@ -81,7 +81,7 @@ from core.inference.llama_cpp import ( LlamaCppBackend, _cached_colocated_split_main, _gguf_files_for_variant, - _hf_offline_if_dns_dead, + _hf_offline_if_unreachable, _probe_dns_dead, _resolve_repo_id_casing, ) @@ -871,7 +871,7 @@ class TestDetectGgufModelRemoteOffline: # --------------------------------------------------------------------------- -# _probe_dns_dead / _hf_offline_if_dns_dead +# _probe_dns_dead / _hf_offline_if_unreachable # --------------------------------------------------------------------------- @@ -900,6 +900,18 @@ def dns(monkeypatch): return _DnsState(monkeypatch) +@pytest.fixture +def reachable(monkeypatch): + """Control the endpoint reachability probe; defaults to reachable so no test hits the network.""" + import utils.utils as _utils + + def _set(unreachable: bool): + monkeypatch.setattr(_utils, "hf_unreachable", lambda *a, **k: unreachable) + + _set(False) + return _set + + class TestProbeDnsDead: def test_returns_false_on_success(self, dns): dns.ok() @@ -919,11 +931,11 @@ class TestProbeDnsDead: socket.setdefaulttimeout(None) -class TestHfOfflineIfDnsDead: - def test_dns_fail_sets_env_inside_block_only(self, dns, clean_offline_env): +class TestHfOfflineIfUnreachable: + def test_dns_fail_sets_env_inside_block_only(self, dns, reachable, clean_offline_env): dns.fail() assert "HF_HUB_OFFLINE" not in os.environ - with _hf_offline_if_dns_dead() as did_set: + with _hf_offline_if_unreachable() as did_set: assert did_set is True assert os.environ.get("HF_HUB_OFFLINE") == "1" assert os.environ.get("TRANSFORMERS_OFFLINE") == "1" @@ -931,38 +943,54 @@ class TestHfOfflineIfDnsDead: assert "HF_HUB_OFFLINE" not in os.environ assert "TRANSFORMERS_OFFLINE" not in os.environ - def test_dns_ok_is_noop(self, dns, clean_offline_env): + def test_dns_ok_and_reachable_is_noop(self, dns, reachable, clean_offline_env): dns.ok() - with _hf_offline_if_dns_dead() as did_set: + with _hf_offline_if_unreachable() as did_set: assert did_set is False assert "HF_HUB_OFFLINE" not in os.environ - def test_dns_recovers_between_calls(self, dns, clean_offline_env): + def test_dns_ok_but_endpoint_unreachable_engages(self, dns, reachable, clean_offline_env): + # WAN down behind a live router: DNS answers, egress does not. + dns.ok() + reachable(True) + with _hf_offline_if_unreachable() as did_set: + assert did_set is True + assert os.environ.get("HF_HUB_OFFLINE") == "1" + assert "HF_HUB_OFFLINE" not in os.environ + + def test_probe_opt_out_keeps_dns_only_behaviour(self, dns, clean_offline_env, monkeypatch): + monkeypatch.setenv("UNSLOTH_OFFLINE_PROBE", "0") + dns.ok() + with _hf_offline_if_unreachable() as did_set: + assert did_set is False + assert "HF_HUB_OFFLINE" not in os.environ + + def test_dns_recovers_between_calls(self, dns, reachable, clean_offline_env): # First call: DNS dead -> env set inside, cleared on exit. dns.fail() - with _hf_offline_if_dns_dead(): + with _hf_offline_if_unreachable(): pass assert "HF_HUB_OFFLINE" not in os.environ # Second call: DNS healthy -> no env mutation. dns.ok() - with _hf_offline_if_dns_dead() as did_set: + with _hf_offline_if_unreachable() as did_set: assert did_set is False assert "HF_HUB_OFFLINE" not in os.environ - def test_user_set_hf_hub_offline_is_preserved(self, dns, clean_offline_env, monkeypatch): + def test_user_set_hf_hub_offline_is_preserved(self, dns, reachable, clean_offline_env, monkeypatch): # User explicitly set offline before launching Unsloth. monkeypatch.setenv("HF_HUB_OFFLINE", "1") dns.fail() - with _hf_offline_if_dns_dead() as did_set: + with _hf_offline_if_unreachable() as did_set: assert did_set is False assert os.environ.get("HF_HUB_OFFLINE") == "1" # Helper must not pop a variable it did not set. assert os.environ.get("HF_HUB_OFFLINE") == "1" - def test_user_set_transformers_offline_is_preserved(self, dns, clean_offline_env, monkeypatch): + def test_user_set_transformers_offline_is_preserved(self, dns, reachable, clean_offline_env, monkeypatch): monkeypatch.setenv("TRANSFORMERS_OFFLINE", "1") dns.fail() - with _hf_offline_if_dns_dead(): + with _hf_offline_if_unreachable(): assert os.environ.get("HF_HUB_OFFLINE") == "1" assert os.environ.get("TRANSFORMERS_OFFLINE") == "1" # HF_HUB_OFFLINE was set by helper -> removed. @@ -970,16 +998,75 @@ class TestHfOfflineIfDnsDead: # TRANSFORMERS_OFFLINE pre-existed -> preserved. assert os.environ.get("TRANSFORMERS_OFFLINE") == "1" - def test_exception_inside_block_still_restores_env(self, dns, clean_offline_env): + def test_exception_inside_block_still_restores_env(self, dns, reachable, clean_offline_env): dns.fail() with pytest.raises(RuntimeError, match = "boom"): - with _hf_offline_if_dns_dead(): + with _hf_offline_if_unreachable(): raise RuntimeError("boom") # Cleanup must happen on exception as well. assert "HF_HUB_OFFLINE" not in os.environ assert "TRANSFORMERS_OFFLINE" not in os.environ +class TestHfUnreachableProbe: + """``utils.utils.hf_unreachable``: memoised, opt-outable, fails open.""" + + @pytest.fixture(autouse = True) + def _reset(self): + from utils.utils import reset_hf_reachability_cache + + reset_hf_reachability_cache() + yield + reset_hf_reachability_cache() + + def _patch_probe(self, monkeypatch, result, calls): + import utils.transformers_version as tv + + def _probe(*_a, **_k): + calls.append(1) + if isinstance(result, Exception): + raise result + return result + + monkeypatch.setattr(tv, "hf_endpoint_unreachable", _probe) + + def test_probes_once_then_memoises(self, monkeypatch, clean_offline_env): + from utils.utils import hf_unreachable + + calls: list = [] + self._patch_probe(monkeypatch, True, calls) + assert hf_unreachable() is True + assert hf_unreachable() is True + assert len(calls) == 1 + + def test_opt_out_skips_probe(self, monkeypatch, clean_offline_env): + from utils.utils import hf_unreachable + + calls: list = [] + self._patch_probe(monkeypatch, True, calls) + monkeypatch.setenv("UNSLOTH_OFFLINE_PROBE", "0") + assert hf_unreachable() is False + assert calls == [] + + def test_probe_failure_reports_reachable(self, monkeypatch, clean_offline_env): + from utils.utils import hf_unreachable + + calls: list = [] + self._patch_probe(monkeypatch, RuntimeError("boom"), calls) + # Fail open: a broken probe must not strand a working install offline. + assert hf_unreachable() is False + + def test_reset_forces_reprobe(self, monkeypatch, clean_offline_env): + from utils.utils import hf_unreachable, reset_hf_reachability_cache + + calls: list = [] + self._patch_probe(monkeypatch, True, calls) + assert hf_unreachable() is True + reset_hf_reachability_cache() + assert hf_unreachable() is True + assert len(calls) == 2 + + class TestExtractQuantLabelSubdir: """``_extract_quant_label`` must consider parent dirs when the basename has no quant token (subdir layouts like ``BF16/foo.gguf``).""" diff --git a/studio/backend/utils/utils.py b/studio/backend/utils/utils.py index e4964b8d04..0871681fb6 100644 --- a/studio/backend/utils/utils.py +++ b/studio/backend/utils/utils.py @@ -5,6 +5,8 @@ import os import structlog +import threading +import time from loggers import get_logger from contextlib import contextmanager from pathlib import Path @@ -35,6 +37,150 @@ def hf_env_offline() -> bool: return False +# One load makes many hub calls, so the reachability verdict is shared for a short window. +_HF_REACHABILITY_TTL_S = 60.0 +_hf_reachability: Optional[tuple] = None +_hf_reachability_lock = threading.Lock() + + +def hf_probe_disabled() -> bool: + """True when UNSLOTH_OFFLINE_PROBE opts out of the reachability probe.""" + return os.environ.get("UNSLOTH_OFFLINE_PROBE", "1").strip().lower() in { + "0", + "false", + "no", + "off", + } + + +def reset_hf_reachability_cache() -> None: + """Drop the memoised verdict so the next call re-probes (tests, network changes).""" + global _hf_reachability + with _hf_reachability_lock: + _hf_reachability = None + + +def hf_unreachable(timeout: int = 3) -> bool: + """True when the HF endpoint is unreachable, memoised for _HF_REACHABILITY_TTL_S. + + Resolving DNS does not mean the Hub is reachable: a router that is up with the WAN + down, a captive portal or a stale DNS cache all answer lookups while every request + then burns huggingface_hub's retry backoff. This is the bounded, proxy-aware probe + the export path already uses; disable it with UNSLOTH_OFFLINE_PROBE=0. + + Fails open: an unavailable probe reports reachable, so the load decides as it does today. + """ + if hf_probe_disabled(): + return False + + global _hf_reachability + cached = _hf_reachability + if cached is not None and time.monotonic() - cached[0] < _HF_REACHABILITY_TTL_S: + return cached[1] + + with _hf_reachability_lock: + cached = _hf_reachability + if cached is not None and time.monotonic() - cached[0] < _HF_REACHABILITY_TTL_S: + return cached[1] + try: + from utils.transformers_version import hf_endpoint_unreachable + + unreachable = hf_endpoint_unreachable(timeout) + except Exception: + unreachable = False + _hf_reachability = (time.monotonic(), unreachable) + return unreachable + + +def _reset_hf_sessions() -> None: + """Drop cached hub sessions so they remount with the current offline adapter.""" + try: + from huggingface_hub.utils import _http + + for name in ("_get_session_from_cache", "get_session"): + cache_clear = getattr(getattr(_http, name, None), "cache_clear", None) + if cache_clear is not None: + cache_clear() + reset = getattr(_http, "reset_sessions", None) + if reset is not None: + reset() + except Exception: + pass + + +# Process-global, so nested/concurrent loads refcount rather than restore out from under +# each other. +_force_offline_depth = 0 +_force_offline_saved: list = [] +_force_offline_saved_env: dict = {} +_force_offline_lock = threading.Lock() + +_OFFLINE_ENV_KEYS = ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE") +_OFFLINE_CONSTANTS = ( + ("huggingface_hub.constants", ("HF_HUB_OFFLINE",)), + ("transformers.utils.hub", ("_is_offline_mode", "OFFLINE")), +) + + +@contextmanager +def force_hf_offline(): + """Force HF offline for this block, in-process. + + Setting the env vars is not enough once the process is running: huggingface_hub and + transformers read their offline constants at import, and hub sessions cache a + non-offline adapter. Flip the constants too and rebuild the sessions, so hub calls + fail fast instead of retrying. Everything is restored on exit. + """ + global _force_offline_depth, _force_offline_saved, _force_offline_saved_env + import importlib + + with _force_offline_lock: + if _force_offline_depth == 0: + saved: list = [] + saved_env: dict = {} + # Snapshot constants BEFORE forcing the env, else a module first imported + # inside the window reads the "1" and we would restore it as offline. + for mod_name, attrs in _OFFLINE_CONSTANTS: + try: + mod = importlib.import_module(mod_name) + except Exception: + continue + for attr in attrs: + if hasattr(mod, attr): + saved.append((mod, attr, getattr(mod, attr))) + for key in _OFFLINE_ENV_KEYS: + saved_env[key] = os.environ.get(key) + os.environ[key] = "1" + for mod, attr, _ in saved: + try: + setattr(mod, attr, True) + except Exception: + pass + _force_offline_saved = saved + _force_offline_saved_env = saved_env + _reset_hf_sessions() + _force_offline_depth += 1 + try: + yield + finally: + with _force_offline_lock: + _force_offline_depth -= 1 + if _force_offline_depth == 0: + for mod, attr, val in _force_offline_saved: + try: + setattr(mod, attr, val) + except Exception: + pass + _force_offline_saved = [] + for key, val in _force_offline_saved_env.items(): + if val is None: + os.environ.pop(key, None) + else: + os.environ[key] = val + _force_offline_saved_env = {} + _reset_hf_sessions() + + def st_repo_id_candidates(model_name: str) -> list: """Repo ids a Sentence-Transformers load may resolve model_name to; a slashless name also resolves under the sentence-transformers/ namespace, so both are candidates.""" From b3ca10d04ec41d53b3545eaeffa62bdc1ec847a4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 04:39:55 +0000 Subject: [PATCH 02/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/core/inference/llama_cpp.py | 1 - studio/backend/core/training/worker.py | 1 - studio/backend/tests/test_chat_load_during_training.py | 4 +++- studio/backend/tests/test_offline_gguf_cache_fallback.py | 9 ++++++--- studio/backend/utils/utils.py | 1 - 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 47917004f9..520b368a75 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -561,7 +561,6 @@ def _hf_offline_if_unreachable(): force_ctx = None try: from utils.utils import force_hf_offline - force_ctx = force_hf_offline() force_ctx.__enter__() except Exception: diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py index 23b5c4a371..dbbc755966 100644 --- a/studio/backend/core/training/worker.py +++ b/studio/backend/core/training/worker.py @@ -2368,7 +2368,6 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) -> try: from utils.transformers_version import hf_endpoint_unreachable from utils.utils import hf_probe_disabled - if not hf_probe_disabled() and hf_endpoint_unreachable(): _result[0] = True except Exception: diff --git a/studio/backend/tests/test_chat_load_during_training.py b/studio/backend/tests/test_chat_load_during_training.py index 1ead0afe4b..cce266446d 100644 --- a/studio/backend/tests/test_chat_load_during_training.py +++ b/studio/backend/tests/test_chat_load_during_training.py @@ -1163,7 +1163,9 @@ class TestLoadModelGuardIntegration(unittest.TestCase): patch.object(self.route, "resolve_effective_chat_template_override", return_value = None), patch.object(self.route, "get_inference_backend", return_value = inf), patch.object(self.route, "get_llama_cpp_backend", return_value = llama), - patch.object(self.route, "_hf_offline_if_unreachable", lambda: contextlib.nullcontext()), + patch.object( + self.route, "_hf_offline_if_unreachable", lambda: contextlib.nullcontext() + ), patch.object(self.route.ModelConfig, "from_identifier", return_value = cfg), _stub_guard_deps(training_active = True, decision = (False, info)), ): diff --git a/studio/backend/tests/test_offline_gguf_cache_fallback.py b/studio/backend/tests/test_offline_gguf_cache_fallback.py index be9c92eb3a..692d0ca783 100644 --- a/studio/backend/tests/test_offline_gguf_cache_fallback.py +++ b/studio/backend/tests/test_offline_gguf_cache_fallback.py @@ -977,7 +977,9 @@ class TestHfOfflineIfUnreachable: assert did_set is False assert "HF_HUB_OFFLINE" not in os.environ - def test_user_set_hf_hub_offline_is_preserved(self, dns, reachable, clean_offline_env, monkeypatch): + def test_user_set_hf_hub_offline_is_preserved( + self, dns, reachable, clean_offline_env, monkeypatch + ): # User explicitly set offline before launching Unsloth. monkeypatch.setenv("HF_HUB_OFFLINE", "1") dns.fail() @@ -987,7 +989,9 @@ class TestHfOfflineIfUnreachable: # Helper must not pop a variable it did not set. assert os.environ.get("HF_HUB_OFFLINE") == "1" - def test_user_set_transformers_offline_is_preserved(self, dns, reachable, clean_offline_env, monkeypatch): + def test_user_set_transformers_offline_is_preserved( + self, dns, reachable, clean_offline_env, monkeypatch + ): monkeypatch.setenv("TRANSFORMERS_OFFLINE", "1") dns.fail() with _hf_offline_if_unreachable(): @@ -1021,7 +1025,6 @@ class TestHfUnreachableProbe: def _patch_probe(self, monkeypatch, result, calls): import utils.transformers_version as tv - def _probe(*_a, **_k): calls.append(1) if isinstance(result, Exception): diff --git a/studio/backend/utils/utils.py b/studio/backend/utils/utils.py index 0871681fb6..99a2386e40 100644 --- a/studio/backend/utils/utils.py +++ b/studio/backend/utils/utils.py @@ -84,7 +84,6 @@ def hf_unreachable(timeout: int = 3) -> bool: return cached[1] try: from utils.transformers_version import hf_endpoint_unreachable - unreachable = hf_endpoint_unreachable(timeout) except Exception: unreachable = False From c4973c76f6e025c7ad0dbf08b55e0a85c8e665be Mon Sep 17 00:00:00 2001 From: Unsloth Date: Tue, 28 Jul 2026 22:30:24 -0700 Subject: [PATCH 03/10] Address review: endpoint-aware DNS check, shorter memo, strict gateway mode Four issues raised on the first commit, all reproduced before fixing: - The DNS pre-check hardcoded huggingface.co, so a reachable HF_ENDPOINT mirror was forced offline whenever huggingface.co did not resolve. It now follows the configured endpoint. - The reachability verdict was memoised for 60s, and a stale "reachable" hid the user pulling the plug right after a download, which is the exact workflow this fix targets. Window is now 5s in both directions: long enough to dedupe the probes within one load, short enough that neither direction goes stale. - hf_endpoint_unreachable counts 502/503/504 as offline, and the training worker used it to set flags for the whole job, so a momentary hub blip blocked every download for the rest of the run. Added gateway_errors_offline=False for callers setting lifetime flags; scoped callers keep the existing behaviour. - Dropped the guard from _target_is_vision. The resolver only yields local paths there, so it returns from the mmproj filesystem branch without touching the hub, and the probe only added latency per request. Verified unchanged offline: load 686s -> 5s, /models/config 378s -> 0s, /models/check-vision 28s -> 0s. --- studio/backend/core/inference/llama_cpp.py | 18 +++- studio/backend/core/training/worker.py | 6 +- studio/backend/routes/inference.py | 8 +- .../tests/test_offline_gguf_cache_fallback.py | 99 +++++++++++++++++++ studio/backend/utils/transformers_version.py | 9 +- studio/backend/utils/utils.py | 16 ++- 6 files changed, 142 insertions(+), 14 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 520b368a75..cdfbfeeba3 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -495,9 +495,23 @@ _SWA_CACHE: Optional[dict] = None _SWA_CACHE_LOCK = threading.Lock() -def _probe_dns_dead(host: str = "huggingface.co", timeout: float = 2.0) -> bool: +def _hf_endpoint_host() -> str: + """Host of the configured hub endpoint. Mirror users point HF_ENDPOINT elsewhere, and + probing huggingface.co would then report their working mirror as offline.""" + endpoint = (os.environ.get("HF_ENDPOINT") or "").strip() or "https://huggingface.co" + if "://" not in endpoint: + endpoint = "https://" + endpoint + try: + from urllib.parse import urlparse + return urlparse(endpoint).hostname or "huggingface.co" + except Exception: + return "huggingface.co" + + +def _probe_dns_dead(host: Optional[str] = None, timeout: float = 2.0) -> bool: """Quick DNS check on a daemon thread, so concurrent sockets aren't - affected by socket.setdefaulttimeout.""" + affected by socket.setdefaulttimeout. Defaults to the configured endpoint's host.""" + host = host or _hf_endpoint_host() result: list[Optional[bool]] = [None] def _probe() -> None: diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py index dbbc755966..fd6cb21d8b 100644 --- a/studio/backend/core/training/worker.py +++ b/studio/backend/core/training/worker.py @@ -2365,10 +2365,14 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) -> if _result[0] is False: # DNS answers even when there is no egress (WAN down, captive portal), so # confirm with the bounded, proxy-aware reachability probe. HF_ENDPOINT aware. + # These flags last the whole job, so only a connection failure counts: a + # momentary 502/503 must not block every download for the rest of the run. try: from utils.transformers_version import hf_endpoint_unreachable from utils.utils import hf_probe_disabled - if not hf_probe_disabled() and hf_endpoint_unreachable(): + if not hf_probe_disabled() and hf_endpoint_unreachable( + gateway_errors_offline = False + ): _result[0] = True except Exception: pass diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index b317133d5c..6f4cddc146 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -3672,10 +3672,10 @@ def _target_is_vision(load_path: str) -> bool: # paths, where the token is unused, but the rule requires it regardless). from utils.models.model_config import is_vision_model try: - # Guarded: this runs per request, so an unreachable hub would re-pay its retry - # backoff on every image/audio call. - with _hf_offline_if_unreachable(): - return bool(is_vision_model(load_path, hf_token = os.environ.get("HF_TOKEN"))) + # Deliberately unguarded: the resolver only yields local paths, so this returns + # from the mmproj filesystem branch without touching the hub. A reachability + # probe here would add seconds per request and prevent nothing. + return bool(is_vision_model(load_path, hf_token = os.environ.get("HF_TOKEN"))) except Exception as exc: # Detection failure: don't block the swap, let the load decide. logger.debug("auto-switch: vision probe failed for %s: %s", load_path, exc) diff --git a/studio/backend/tests/test_offline_gguf_cache_fallback.py b/studio/backend/tests/test_offline_gguf_cache_fallback.py index 692d0ca783..3c62fb7964 100644 --- a/studio/backend/tests/test_offline_gguf_cache_fallback.py +++ b/studio/backend/tests/test_offline_gguf_cache_fallback.py @@ -1012,6 +1012,80 @@ class TestHfOfflineIfUnreachable: assert "TRANSFORMERS_OFFLINE" not in os.environ +class TestEndpointAwareOfflineDetection: + """A reachable HF_ENDPOINT mirror must not be declared offline just because + huggingface.co does not resolve (air-gapped / corporate networks).""" + + @pytest.fixture + def no_upstream_dns(self, monkeypatch): + real_host = socket.gethostbyname + + def _host(h, *a, **k): + if "huggingface.co" in str(h): + raise socket.gaierror(-2, "Name or service not known") + return "127.0.0.1" + + monkeypatch.setattr(socket, "gethostbyname", _host) + + @pytest.mark.parametrize( + "endpoint,expected", + [ + ("https://hf-mirror.com", "hf-mirror.com"), + ("hf-mirror.com", "hf-mirror.com"), + ("https://hf-mirror.com:8443", "hf-mirror.com"), + ("https://hf-mirror.com/path", "hf-mirror.com"), + ("", "huggingface.co"), + ], + ) + def test_endpoint_host_parsing(self, monkeypatch, endpoint, expected): + from core.inference.llama_cpp import _hf_endpoint_host + + monkeypatch.setenv("HF_ENDPOINT", endpoint) + assert _hf_endpoint_host() == expected + + def test_dns_precheck_follows_endpoint(self, monkeypatch, no_upstream_dns): + monkeypatch.setenv("HF_ENDPOINT", "https://hf-mirror.com") + assert _probe_dns_dead() is False + + def test_default_endpoint_still_probes_huggingface(self, monkeypatch, no_upstream_dns): + monkeypatch.delenv("HF_ENDPOINT", raising = False) + assert _probe_dns_dead() is True + + +class TestGatewayErrorsAreNotConnectionFailures: + """Lifetime offline flags must not be set by a momentary 502/503/504.""" + + def _probe_with(self, monkeypatch, exc): + import urllib.request + + def _urlopen(*a, **k): + raise exc + + monkeypatch.setattr(urllib.request, "urlopen", _urlopen) + from utils.transformers_version import hf_endpoint_unreachable + + return hf_endpoint_unreachable + + @pytest.mark.parametrize("code", [502, 503, 504]) + def test_strict_mode_treats_gateway_error_as_reachable(self, monkeypatch, code): + import urllib.error + + exc = urllib.error.HTTPError("u", code, "err", {}, None) + probe = self._probe_with(monkeypatch, exc) + assert probe(timeout = 1, gateway_errors_offline = False) is False + # Default (scoped callers) keeps treating a downed hub as offline. + assert probe(timeout = 1) is True + + @pytest.mark.parametrize("code", [401, 403, 404, 429]) + def test_other_http_errors_always_reachable(self, monkeypatch, code): + import urllib.error + + exc = urllib.error.HTTPError("u", code, "err", {}, None) + probe = self._probe_with(monkeypatch, exc) + assert probe(timeout = 1) is False + assert probe(timeout = 1, gateway_errors_offline = False) is False + + class TestHfUnreachableProbe: """``utils.utils.hf_unreachable``: memoised, opt-outable, fails open.""" @@ -1069,6 +1143,31 @@ class TestHfUnreachableProbe: assert hf_unreachable() is True assert len(calls) == 2 + def test_memo_window_is_short_in_both_directions(self): + """Stale either way is a bug: a stale 'reachable' hides the plug being pulled, + a stale 'unreachable' fails a download after the user reconnects.""" + import utils.utils as uu + + assert uu._HF_REACHABILITY_TTL_S <= 10.0 + + def test_verdict_expires_so_a_disconnect_is_noticed(self, monkeypatch, clean_offline_env): + import time as _time + + import utils.utils as uu + from utils.utils import hf_unreachable + + monkeypatch.setattr(uu, "_HF_REACHABILITY_TTL_S", 0.2) + verdict = {"value": False} + monkeypatch.setattr( + __import__("utils.transformers_version", fromlist = ["x"]), + "hf_endpoint_unreachable", + lambda *a, **k: verdict["value"], + ) + assert hf_unreachable() is False # online during the download + verdict["value"] = True # plug pulled + _time.sleep(0.3) + assert hf_unreachable() is True + class TestExtractQuantLabelSubdir: """``_extract_quant_label`` must consider parent dirs when the basename has diff --git a/studio/backend/utils/transformers_version.py b/studio/backend/utils/transformers_version.py index b0a2da0e66..bbc51d0774 100644 --- a/studio/backend/utils/transformers_version.py +++ b/studio/backend/utils/transformers_version.py @@ -63,7 +63,7 @@ def _env_offline() -> bool: ) -def hf_endpoint_unreachable(timeout: int = 3) -> bool: +def hf_endpoint_unreachable(timeout: int = 3, *, gateway_errors_offline: bool = True) -> bool: """Bounded reachability probe to the HF endpoint. A HEAD request runs in a daemon thread joined with a deadline, so a resolver blackhole cannot block past ~timeout+1s. True if unreachable. urllib natively honors *_PROXY / NO_PROXY, so this verifies real egress @@ -86,8 +86,11 @@ def hf_endpoint_unreachable(timeout: int = 3) -> bool: with urllib.request.urlopen(req, timeout = timeout): result["online"] = True except urllib.error.HTTPError as exc: - # The server/proxy answered: reachable unless it is a gateway error. - result["online"] = exc.code not in (502, 503, 504) + # The server/proxy answered, so we have egress. A gateway error usually means + # the hub itself is down, which callers scoping offline to one operation want + # to treat as offline; callers setting a lifetime flag pass + # gateway_errors_offline=False so a momentary 503 can't strand the process. + result["online"] = True if not gateway_errors_offline else exc.code not in (502, 503, 504) except urllib.error.URLError as exc: # A TLS/cert failure means we DID reach the server; treat as reachable so the real # load surfaces it (consistent with _is_offline_related_error not retrying TLS). diff --git a/studio/backend/utils/utils.py b/studio/backend/utils/utils.py index 99a2386e40..b2738f4d56 100644 --- a/studio/backend/utils/utils.py +++ b/studio/backend/utils/utils.py @@ -37,12 +37,20 @@ def hf_env_offline() -> bool: return False -# One load makes many hub calls, so the reachability verdict is shared for a short window. -_HF_REACHABILITY_TTL_S = 60.0 +# One load makes many hub calls, so the verdict is shared briefly to avoid re-probing on +# each. Kept short in BOTH directions: a stale "reachable" misses the plug being pulled +# (the case this whole path exists for), and a stale "unreachable" sends a load to the +# cache after the user reconnected, failing it if the model is not cached. +_HF_REACHABILITY_TTL_S = 5.0 _hf_reachability: Optional[tuple] = None _hf_reachability_lock = threading.Lock() +def _reachability_fresh(entry) -> bool: + """True while a cached (timestamp, unreachable) verdict may still be reused.""" + return entry is not None and (time.monotonic() - entry[0]) < _HF_REACHABILITY_TTL_S + + def hf_probe_disabled() -> bool: """True when UNSLOTH_OFFLINE_PROBE opts out of the reachability probe.""" return os.environ.get("UNSLOTH_OFFLINE_PROBE", "1").strip().lower() in { @@ -75,12 +83,12 @@ def hf_unreachable(timeout: int = 3) -> bool: global _hf_reachability cached = _hf_reachability - if cached is not None and time.monotonic() - cached[0] < _HF_REACHABILITY_TTL_S: + if _reachability_fresh(cached): return cached[1] with _hf_reachability_lock: cached = _hf_reachability - if cached is not None and time.monotonic() - cached[0] < _HF_REACHABILITY_TTL_S: + if _reachability_fresh(cached): return cached[1] try: from utils.transformers_version import hf_endpoint_unreachable From 1ec30c54d1f946523006eebc4eb1186e41ab005a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 05:34:19 +0000 Subject: [PATCH 04/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/tests/test_offline_gguf_cache_fallback.py | 2 -- studio/backend/utils/transformers_version.py | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/studio/backend/tests/test_offline_gguf_cache_fallback.py b/studio/backend/tests/test_offline_gguf_cache_fallback.py index 3c62fb7964..3240c2dc5e 100644 --- a/studio/backend/tests/test_offline_gguf_cache_fallback.py +++ b/studio/backend/tests/test_offline_gguf_cache_fallback.py @@ -1039,7 +1039,6 @@ class TestEndpointAwareOfflineDetection: ) def test_endpoint_host_parsing(self, monkeypatch, endpoint, expected): from core.inference.llama_cpp import _hf_endpoint_host - monkeypatch.setenv("HF_ENDPOINT", endpoint) assert _hf_endpoint_host() == expected @@ -1147,7 +1146,6 @@ class TestHfUnreachableProbe: """Stale either way is a bug: a stale 'reachable' hides the plug being pulled, a stale 'unreachable' fails a download after the user reconnects.""" import utils.utils as uu - assert uu._HF_REACHABILITY_TTL_S <= 10.0 def test_verdict_expires_so_a_disconnect_is_noticed(self, monkeypatch, clean_offline_env): diff --git a/studio/backend/utils/transformers_version.py b/studio/backend/utils/transformers_version.py index bbc51d0774..34576decc2 100644 --- a/studio/backend/utils/transformers_version.py +++ b/studio/backend/utils/transformers_version.py @@ -90,7 +90,9 @@ def hf_endpoint_unreachable(timeout: int = 3, *, gateway_errors_offline: bool = # the hub itself is down, which callers scoping offline to one operation want # to treat as offline; callers setting a lifetime flag pass # gateway_errors_offline=False so a momentary 503 can't strand the process. - result["online"] = True if not gateway_errors_offline else exc.code not in (502, 503, 504) + result["online"] = ( + True if not gateway_errors_offline else exc.code not in (502, 503, 504) + ) except urllib.error.URLError as exc: # A TLS/cert failure means we DID reach the server; treat as reachable so the real # load surfaces it (consistent with _is_offline_related_error not retrying TLS). From d5758ecf58757b4444a905819a0f8a1075d9d1ee Mon Sep 17 00:00:00 2001 From: Unsloth Date: Tue, 28 Jul 2026 23:46:34 -0700 Subject: [PATCH 05/10] Address review round 2: proxy-aware detection, shared worker probe, local skip All four reproduced before fixing, and re-measured after. - Proxy-only egress was declared offline. With HTTP(S)_PROXY set, the proxy resolves the hub host, so a failing local lookup says nothing. The DNS shortcut now stands down whenever a proxy applies (and honours NO_PROXY), letting the proxy-aware probe decide. Measured: endpoint probe reachable through the proxy while the guard still forced offline. - The training worker kept its own inline probe hardcoded to huggingface.co, so a reachable HF_ENDPOINT mirror set lifetime offline flags. It now uses the shared endpoint- and proxy-aware helper. - /models/check-vision, /models/config and /picker/chat-template ran the probe even for local paths, which never reach the hub. Measured 0.9s of pure latency per request; now skipped via _hf_offline_if_unreachable_for. DNS/endpoint/proxy helpers now live in utils.utils so llama_cpp and the training worker share one implementation instead of three copies. The static pin in test_offline_inference_parent moved with the probe: the worker block must delegate to the shared helper and must not hardcode a host, and the daemon-thread/no-setdefaulttimeout property is pinned on dns_host_dead where it now lives. Offline path unchanged: load 686s -> 6s, /models/config 378s -> 0s, and a local-path vision check is back to 0s. --- studio/backend/core/inference/llama_cpp.py | 56 +++++++------- studio/backend/core/training/worker.py | 44 ++++------- studio/backend/picker/routes/templates.py | 4 +- studio/backend/routes/models.py | 12 +-- .../tests/test_offline_gguf_cache_fallback.py | 60 +++++++++++++++ .../tests/test_offline_inference_parent.py | 23 +++++- studio/backend/utils/utils.py | 74 +++++++++++++++++++ 7 files changed, 205 insertions(+), 68 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index cdfbfeeba3..c9df5a4599 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -496,36 +496,16 @@ _SWA_CACHE_LOCK = threading.Lock() def _hf_endpoint_host() -> str: - """Host of the configured hub endpoint. Mirror users point HF_ENDPOINT elsewhere, and - probing huggingface.co would then report their working mirror as offline.""" - endpoint = (os.environ.get("HF_ENDPOINT") or "").strip() or "https://huggingface.co" - if "://" not in endpoint: - endpoint = "https://" + endpoint - try: - from urllib.parse import urlparse - return urlparse(endpoint).hostname or "huggingface.co" - except Exception: - return "huggingface.co" + """Host of the configured hub endpoint (HF_ENDPOINT aware).""" + from utils.utils import hf_endpoint_host + return hf_endpoint_host() def _probe_dns_dead(host: Optional[str] = None, timeout: float = 2.0) -> bool: """Quick DNS check on a daemon thread, so concurrent sockets aren't affected by socket.setdefaulttimeout. Defaults to the configured endpoint's host.""" - host = host or _hf_endpoint_host() - result: list[Optional[bool]] = [None] - - def _probe() -> None: - try: - socket.gethostbyname(host) - result[0] = False - except Exception: - result[0] = True - - t = threading.Thread(target = _probe, daemon = True) - t.start() - t.join(timeout) - # Thread still running -> resolver wedged -> dead. - return True if result[0] is None else result[0] + from utils.utils import dns_host_dead, hf_endpoint_host + return dns_host_dead(host or hf_endpoint_host(), timeout) def _hf_env_offline() -> bool: @@ -546,14 +526,16 @@ def _hf_unreachable() -> bool: DNS alone misses the common offline shapes (WAN down behind a live router, captive portal, stale DNS cache), leaving every hub call to burn its full retry backoff. + The DNS shortcut is skipped when a proxy is configured, since the proxy resolves the + hub host and local DNS then says nothing about reachability. """ - if _probe_dns_dead(): - return True try: - from utils.utils import hf_unreachable - return hf_unreachable() + from utils.utils import hf_dns_dead, hf_unreachable except Exception: return False + if hf_dns_dead(): + return True + return hf_unreachable() @contextlib.contextmanager @@ -599,6 +581,22 @@ def _hf_offline_if_unreachable(): os.environ.pop("TRANSFORMERS_OFFLINE", None) +def _hf_offline_if_unreachable_for(model_name): + """Guard, but only for remote repo ids. + + A local path is served from the filesystem and never reaches the hub, so probing + would add seconds per request and prevent no retry. + """ + try: + from utils.paths import is_local_path + + if isinstance(model_name, str) and is_local_path(model_name): + return contextlib.nullcontext() + except Exception: + pass + return _hf_offline_if_unreachable() + + try: _SLOT_SAVE_MAX_BYTES = int(os.environ.get("UNSLOTH_SLOT_SAVE_MAX_BYTES") or (10 << 30)) except ValueError: diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py index fd6cb21d8b..adca92d0cc 100644 --- a/studio/backend/core/training/worker.py +++ b/studio/backend/core/training/worker.py @@ -2346,37 +2346,23 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) -> # Offline auto-detect: skip ~25s of HF retries per call when the hub is unreachable. if "HF_HUB_OFFLINE" not in os.environ: - import socket as _socket - import threading as _threading + _offline = False + try: + from utils.utils import hf_dns_dead, hf_probe_disabled - # Daemon thread so we don't mutate process-wide setdefaulttimeout. - _result: list = [None] - - def _probe() -> None: - try: - _socket.gethostbyname("huggingface.co") - _result[0] = False - except Exception: - _result[0] = True - - _t = _threading.Thread(target = _probe, daemon = True) - _t.start() - _t.join(2.0) - if _result[0] is False: - # DNS answers even when there is no egress (WAN down, captive portal), so - # confirm with the bounded, proxy-aware reachability probe. HF_ENDPOINT aware. - # These flags last the whole job, so only a connection failure counts: a - # momentary 502/503 must not block every download for the rest of the run. - try: + # hf_dns_dead follows HF_ENDPOINT and stands down when a proxy is configured, + # so a reachable mirror (or proxy-only egress) is never called offline. + _offline = hf_dns_dead() + if not _offline and not hf_probe_disabled(): + # DNS answers even without egress (WAN down, captive portal). These flags + # last the whole job, so only a connection failure counts: a momentary + # 502/503 must not block every download for the rest of the run. from utils.transformers_version import hf_endpoint_unreachable - from utils.utils import hf_probe_disabled - if not hf_probe_disabled() and hf_endpoint_unreachable( - gateway_errors_offline = False - ): - _result[0] = True - except Exception: - pass - if _result[0] is None or _result[0] is True: + + _offline = hf_endpoint_unreachable(gateway_errors_offline = False) + except Exception: + _offline = False + if _offline: os.environ["HF_HUB_OFFLINE"] = "1" os.environ.setdefault("TRANSFORMERS_OFFLINE", "1") os.environ.setdefault("HF_DATASETS_OFFLINE", "1") diff --git a/studio/backend/picker/routes/templates.py b/studio/backend/picker/routes/templates.py index a070c8d054..628cc03841 100644 --- a/studio/backend/picker/routes/templates.py +++ b/studio/backend/picker/routes/templates.py @@ -39,10 +39,10 @@ async def get_default_chat_template_route( ) -> ModelTemplateResponse: # Cached repos resolve from disk, but a cache miss falls through to the hub and # offline that costs one retry backoff per candidate template file. - from core.inference.llama_cpp import _hf_offline_if_unreachable + from core.inference.llama_cpp import _hf_offline_if_unreachable_for def _read(): - with _hf_offline_if_unreachable(): + with _hf_offline_if_unreachable_for(model_name): return read_default_chat_template(model_name, hf_token, gguf_variant) template = await asyncio.to_thread(_read) diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py index cc2a402b59..58b7433070 100644 --- a/studio/backend/routes/models.py +++ b/studio/backend/routes/models.py @@ -1855,12 +1855,13 @@ async def get_model_config( ): """Get configuration for a specific model (wraps load_model_defaults).""" hf_token = _normalize_hf_token(header_hf_token) or _normalize_hf_token(hf_token) - from core.inference.llama_cpp import _hf_offline_if_unreachable + from core.inference.llama_cpp import _hf_offline_if_unreachable_for try: # Each probe below can reach the hub, so the guard wraps the whole handler: - # offline they must all resolve from the HF cache instead of retrying. - with _hf_offline_if_unreachable(): + # offline they must all resolve from the HF cache instead of retrying. Local + # paths stay on disk, so they skip the probe entirely. + with _hf_offline_if_unreachable_for(model_name): if not is_local_path(model_name): resolved = resolve_cached_repo_id_case(model_name) if resolved != model_name: @@ -2591,9 +2592,10 @@ async def check_vision_model( logger.info(f"Checking if vision model: {model_name}") # Authenticate so a gated/private VLM classifies correctly (else 404 -> non-vision). # Offline the guard keeps this on the HF cache instead of retrying the hub. - from core.inference.llama_cpp import _hf_offline_if_unreachable + # A local path resolves from disk, so it skips the probe. + from core.inference.llama_cpp import _hf_offline_if_unreachable_for - with _hf_offline_if_unreachable(): + with _hf_offline_if_unreachable_for(model_name): is_vision = is_vision_model(model_name, hf_token = hf_token) logger.info(f"Vision check result for {model_name}: is_vision={is_vision}") diff --git a/studio/backend/tests/test_offline_gguf_cache_fallback.py b/studio/backend/tests/test_offline_gguf_cache_fallback.py index 3240c2dc5e..80bcf82377 100644 --- a/studio/backend/tests/test_offline_gguf_cache_fallback.py +++ b/studio/backend/tests/test_offline_gguf_cache_fallback.py @@ -1051,6 +1051,66 @@ class TestEndpointAwareOfflineDetection: assert _probe_dns_dead() is True +class TestProxyOnlyEgress: + """With a proxy, the proxy resolves the hub host, so local DNS proves nothing and + must not be used to declare the hub offline.""" + + @pytest.fixture + def dns_all_dead(self, monkeypatch): + def _fail(*a, **k): + raise socket.gaierror(-2, "Name or service not known") + + monkeypatch.setattr(socket, "gethostbyname", _fail) + + def test_dns_shortcut_stands_down_when_proxy_configured(self, monkeypatch, dns_all_dead): + from utils.utils import hf_dns_dead + + monkeypatch.delenv("HF_ENDPOINT", raising = False) + monkeypatch.setenv("HTTPS_PROXY", "http://proxy.internal:3128") + monkeypatch.setenv("HTTP_PROXY", "http://proxy.internal:3128") + assert hf_dns_dead() is False + + def test_dns_shortcut_applies_without_a_proxy(self, monkeypatch, dns_all_dead): + from utils.utils import hf_dns_dead + + for key in ("HTTPS_PROXY", "HTTP_PROXY", "https_proxy", "http_proxy", "ALL_PROXY"): + monkeypatch.delenv(key, raising = False) + monkeypatch.setattr("utils.utils.hf_proxy_configured", lambda: False) + assert hf_dns_dead() is True + + def test_no_proxy_bypass_restores_the_shortcut(self, monkeypatch, dns_all_dead): + """A host listed in NO_PROXY does not go through the proxy, so DNS matters again.""" + from utils.utils import hf_proxy_configured + + monkeypatch.setenv("HF_ENDPOINT", "https://huggingface.co") + monkeypatch.setenv("HTTPS_PROXY", "http://proxy.internal:3128") + monkeypatch.setenv("NO_PROXY", "huggingface.co") + assert hf_proxy_configured() is False + + +class TestGuardSkipsLocalPaths: + """A local path never reaches the hub, so probing costs time and prevents nothing.""" + + def test_local_path_is_a_noop(self, tmp_path, monkeypatch): + from core.inference.llama_cpp import _hf_offline_if_unreachable_for + + called: list = [] + monkeypatch.setattr( + "utils.utils.hf_unreachable", lambda *a, **k: called.append(1) or True + ) + with _hf_offline_if_unreachable_for(str(tmp_path / "model.gguf")) as engaged: + assert engaged is None # nullcontext yields None + assert called == [], "probed the hub for a local path" + + def test_remote_id_still_guarded(self, monkeypatch, clean_offline_env): + import core.inference.llama_cpp as lc + + monkeypatch.setattr(lc, "_hf_unreachable", lambda: True) + with lc._hf_offline_if_unreachable_for("unsloth/Qwen3.5-4B-GGUF") as engaged: + assert engaged is True + assert os.environ.get("HF_HUB_OFFLINE") == "1" + + class TestGatewayErrorsAreNotConnectionFailures: """Lifetime offline flags must not be set by a momentary 502/503/504.""" diff --git a/studio/backend/tests/test_offline_inference_parent.py b/studio/backend/tests/test_offline_inference_parent.py index 3e9f09bb2f..c8c440be46 100644 --- a/studio/backend/tests/test_offline_inference_parent.py +++ b/studio/backend/tests/test_offline_inference_parent.py @@ -218,6 +218,23 @@ class TestTrainingWorkerProbeNoGlobalTimeout: "training worker still calls socket.setdefaulttimeout; " "concurrent sockets would inherit the probe timeout" ) - assert ( - "threading" in block and "Thread" in block - ), "training worker probe must run on a daemon thread" + # The probe now lives in the shared helper (endpoint- and proxy-aware), so the + # worker must delegate to it rather than resolve a hardcoded host itself. + assert "hf_dns_dead" in block, "training worker must use the shared DNS helper" + assert 'gethostbyname("huggingface.co")' not in block, ( + "training worker must not hardcode huggingface.co; a reachable HF_ENDPOINT " + "mirror would be declared offline" + ) + + def test_shared_dns_helper_uses_thread_probe(self): + """The daemon-thread property moved with the probe; pin it where it now lives.""" + import inspect + + from utils.utils import dns_host_dead + + src = inspect.getsource(dns_host_dead) + assert ".setdefaulttimeout(" not in src, ( + "shared DNS probe calls socket.setdefaulttimeout; " + "concurrent sockets would inherit the probe timeout" + ) + assert "Thread" in src and "daemon" in src, "shared DNS probe must run on a daemon thread" diff --git a/studio/backend/utils/utils.py b/studio/backend/utils/utils.py index b2738f4d56..7e91307a02 100644 --- a/studio/backend/utils/utils.py +++ b/studio/backend/utils/utils.py @@ -37,6 +37,80 @@ def hf_env_offline() -> bool: return False +def hf_endpoint_url() -> str: + """Configured hub endpoint, scheme-normalised. Mirror users point this elsewhere.""" + endpoint = (os.environ.get("HF_ENDPOINT") or "").strip() or "https://huggingface.co" + return endpoint if "://" in endpoint else "https://" + endpoint + + +def hf_endpoint_host() -> str: + """Host of the configured endpoint; probing huggingface.co would misjudge a mirror.""" + try: + from urllib.parse import urlparse + + return urlparse(hf_endpoint_url()).hostname or "huggingface.co" + except Exception: + return "huggingface.co" + + +def hf_proxy_configured() -> bool: + """True when egress to the endpoint goes through a proxy. + + The proxy resolves the hub host, so local DNS proves nothing about reachability and + must not be used to declare the hub offline. + """ + try: + import urllib.request + from urllib.parse import urlparse + + url = hf_endpoint_url() + proxies = urllib.request.getproxies() + scheme = urlparse(url).scheme or "https" + if scheme not in proxies and "all" not in proxies: + return False + host = urlparse(url).hostname or "" + try: + if host and urllib.request.proxy_bypass(host): + return False + except Exception: + pass + return True + except Exception: + return False + + +def dns_host_dead(host: str, timeout: float = 2.0) -> bool: + """True when host does not resolve. Runs on a daemon thread so a wedged resolver + cannot block past the deadline and so socket.setdefaulttimeout is left alone.""" + result: list = [None] + + def _probe() -> None: + import socket as _socket + + try: + _socket.gethostbyname(host) + result[0] = False + except Exception: + result[0] = True + + t = threading.Thread(target = _probe, daemon = True) + t.start() + t.join(timeout) + # Still running -> resolver wedged -> treat as dead. + return True if result[0] is None else result[0] + + +def hf_dns_dead(timeout: float = 2.0) -> bool: + """Fast offline shortcut: the endpoint's host does not resolve and no proxy is in play. + + Returns False whenever a proxy is configured, so proxy-only setups fall through to the + real reachability probe instead of being wrongly declared offline. + """ + if hf_proxy_configured(): + return False + return dns_host_dead(hf_endpoint_host(), timeout) + + # One load makes many hub calls, so the verdict is shared briefly to avoid re-probing on # each. Kept short in BOTH directions: a stale "reachable" misses the plug being pulled # (the case this whole path exists for), and a stale "unreachable" sends a load to the From 26d205b9102c85256d60a8e3e09a402c34dcc46f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 06:47:19 +0000 Subject: [PATCH 06/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/core/inference/llama_cpp.py | 1 - studio/backend/core/training/worker.py | 1 - studio/backend/tests/test_offline_gguf_cache_fallback.py | 5 +---- studio/backend/utils/utils.py | 2 -- 4 files changed, 1 insertion(+), 8 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index c9df5a4599..611184655e 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -589,7 +589,6 @@ def _hf_offline_if_unreachable_for(model_name): """ try: from utils.paths import is_local_path - if isinstance(model_name, str) and is_local_path(model_name): return contextlib.nullcontext() except Exception: diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py index adca92d0cc..ac27750373 100644 --- a/studio/backend/core/training/worker.py +++ b/studio/backend/core/training/worker.py @@ -2358,7 +2358,6 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) -> # last the whole job, so only a connection failure counts: a momentary # 502/503 must not block every download for the rest of the run. from utils.transformers_version import hf_endpoint_unreachable - _offline = hf_endpoint_unreachable(gateway_errors_offline = False) except Exception: _offline = False diff --git a/studio/backend/tests/test_offline_gguf_cache_fallback.py b/studio/backend/tests/test_offline_gguf_cache_fallback.py index 80bcf82377..c2f8b3840e 100644 --- a/studio/backend/tests/test_offline_gguf_cache_fallback.py +++ b/studio/backend/tests/test_offline_gguf_cache_fallback.py @@ -1095,16 +1095,13 @@ class TestGuardSkipsLocalPaths: from core.inference.llama_cpp import _hf_offline_if_unreachable_for called: list = [] - monkeypatch.setattr( - "utils.utils.hf_unreachable", lambda *a, **k: called.append(1) or True - ) + monkeypatch.setattr("utils.utils.hf_unreachable", lambda *a, **k: called.append(1) or True) with _hf_offline_if_unreachable_for(str(tmp_path / "model.gguf")) as engaged: assert engaged is None # nullcontext yields None assert called == [], "probed the hub for a local path" def test_remote_id_still_guarded(self, monkeypatch, clean_offline_env): import core.inference.llama_cpp as lc - monkeypatch.setattr(lc, "_hf_unreachable", lambda: True) with lc._hf_offline_if_unreachable_for("unsloth/Qwen3.5-4B-GGUF") as engaged: assert engaged is True diff --git a/studio/backend/utils/utils.py b/studio/backend/utils/utils.py index 7e91307a02..db9bace997 100644 --- a/studio/backend/utils/utils.py +++ b/studio/backend/utils/utils.py @@ -47,7 +47,6 @@ def hf_endpoint_host() -> str: """Host of the configured endpoint; probing huggingface.co would misjudge a mirror.""" try: from urllib.parse import urlparse - return urlparse(hf_endpoint_url()).hostname or "huggingface.co" except Exception: return "huggingface.co" @@ -86,7 +85,6 @@ def dns_host_dead(host: str, timeout: float = 2.0) -> bool: def _probe() -> None: import socket as _socket - try: _socket.gethostbyname(host) result[0] = False From 5c9b82e0240e807b3e04ac0d2f2a26ed953269e1 Mon Sep 17 00:00:00 2001 From: Unsloth Date: Wed, 29 Jul 2026 00:25:12 -0700 Subject: [PATCH 07/10] Address review round 3: slow links, blank endpoint, IPv6 resolution - A reachable endpoint that answers slower than the probe deadline was classified offline, so an uncached load failed instead of merely being slow. A clean socket timeout is now resolved with a bounded TCP connect: a loaded server still completes the handshake, a blackholed route does not. A refused connection counts as egress. - An empty or whitespace HF_ENDPOINT made the DNS shortcut fall back to the default hub while the HTTP probe probed "https://" and reported offline. Both stages now share one normaliser. - dns_host_dead used gethostbyname, which is IPv4-only and called an AAAA-only mirror or an IPv6 literal dead. It now uses getaddrinfo. A hang past the deadline still counts as unreachable: the real hub calls would hang the same way, so cache-only is the useful answer there. That distinction is what test_hung_probe_is_bounded pins, and it caught an earlier version of this change that treated every deadline overrun as inconclusive. Verified: slow endpoint reachable, blackholed route still offline, blank endpoint falls back, IPv6 literal resolves. Offline path unchanged, load 686s -> 9s and /models/config 378s -> 0s, chat still answers. --- .../tests/test_offline_gguf_cache_fallback.py | 107 +++++++++++++++++- studio/backend/utils/transformers_version.py | 46 ++++++-- studio/backend/utils/utils.py | 50 +++++++- 3 files changed, 191 insertions(+), 12 deletions(-) diff --git a/studio/backend/tests/test_offline_gguf_cache_fallback.py b/studio/backend/tests/test_offline_gguf_cache_fallback.py index c2f8b3840e..e49045df03 100644 --- a/studio/backend/tests/test_offline_gguf_cache_fallback.py +++ b/studio/backend/tests/test_offline_gguf_cache_fallback.py @@ -876,23 +876,32 @@ class TestDetectGgufModelRemoteOffline: class _DnsState: - """Tiny helper that toggles ``socket.gethostbyname`` failure mode.""" + """Toggles resolver failure. The probe uses ``getaddrinfo`` (IPv4 + IPv6), so patch + both entry points.""" def __init__(self, monkeypatch): self._mp = monkeypatch self._real = socket.gethostbyname + self._real_addr = socket.getaddrinfo def fail(self): def _fail(*a, **k): raise socket.gaierror(-2, "Name or service not known") self._mp.setattr(socket, "gethostbyname", _fail) + self._mp.setattr(socket, "getaddrinfo", _fail) def ok(self): self._mp.setattr(socket, "gethostbyname", lambda *a, **k: "127.0.0.1") + self._mp.setattr( + socket, + "getaddrinfo", + lambda *a, **k: [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 0))], + ) def restore(self): self._mp.setattr(socket, "gethostbyname", self._real) + self._mp.setattr(socket, "getaddrinfo", self._real_addr) @pytest.fixture @@ -1018,14 +1027,18 @@ class TestEndpointAwareOfflineDetection: @pytest.fixture def no_upstream_dns(self, monkeypatch): - real_host = socket.gethostbyname - def _host(h, *a, **k): if "huggingface.co" in str(h): raise socket.gaierror(-2, "Name or service not known") return "127.0.0.1" + def _addr(h, *a, **k): + if "huggingface.co" in str(h): + raise socket.gaierror(-2, "Name or service not known") + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 0))] + monkeypatch.setattr(socket, "gethostbyname", _host) + monkeypatch.setattr(socket, "getaddrinfo", _addr) @pytest.mark.parametrize( "endpoint,expected", @@ -1061,6 +1074,7 @@ class TestProxyOnlyEgress: raise socket.gaierror(-2, "Name or service not known") monkeypatch.setattr(socket, "gethostbyname", _fail) + monkeypatch.setattr(socket, "getaddrinfo", _fail) def test_dns_shortcut_stands_down_when_proxy_configured(self, monkeypatch, dns_all_dead): from utils.utils import hf_dns_dead @@ -1088,6 +1102,93 @@ class TestProxyOnlyEgress: assert hf_proxy_configured() is False +class TestSlowLinkIsNotOffline: + """A slow but reachable endpoint must not be quarantined: that would fail an uncached + load that would otherwise merely be slow. A blackholed route must still be offline.""" + + def _probe_raising(self, monkeypatch, exc): + import urllib.request + + monkeypatch.setattr(urllib.request, "urlopen", lambda *a, **k: (_ for _ in ()).throw(exc)) + from utils.transformers_version import hf_endpoint_unreachable + + return hf_endpoint_unreachable + + def test_timeout_with_tcp_egress_is_reachable(self, monkeypatch): + import urllib.error + + probe = self._probe_raising(monkeypatch, urllib.error.URLError(TimeoutError("slow"))) + monkeypatch.setattr("utils.utils.hf_tcp_reachable", lambda *a, **k: True) + assert probe(timeout = 1) is False + + def test_timeout_without_tcp_egress_is_offline(self, monkeypatch): + import urllib.error + + probe = self._probe_raising(monkeypatch, urllib.error.URLError(TimeoutError("dead"))) + monkeypatch.setattr("utils.utils.hf_tcp_reachable", lambda *a, **k: False) + assert probe(timeout = 1) is True + + def test_bare_timeout_error_takes_the_same_path(self, monkeypatch): + probe = self._probe_raising(monkeypatch, TimeoutError("slow")) + monkeypatch.setattr("utils.utils.hf_tcp_reachable", lambda *a, **k: True) + assert probe(timeout = 1) is False + + def test_refused_connection_counts_as_egress(self, monkeypatch): + """Something answered, so the network works even though nothing is listening.""" + import socket as _socket + + from utils.utils import hf_tcp_reachable + + def _refuse(*a, **k): + raise ConnectionRefusedError() + + monkeypatch.setattr(_socket, "create_connection", _refuse) + assert hf_tcp_reachable(1, "http://127.0.0.1:9") is True + + +class TestEndpointNormalisation: + """An empty or whitespace HF_ENDPOINT must fall back to the default hub in BOTH the + DNS shortcut and the HTTP probe, or the two stages disagree.""" + + @pytest.mark.parametrize("value", ["", " ", "\t"]) + def test_blank_endpoint_falls_back(self, monkeypatch, value): + from utils.utils import hf_endpoint_host, hf_endpoint_url + + monkeypatch.setenv("HF_ENDPOINT", value) + assert hf_endpoint_host() == "huggingface.co" + assert hf_endpoint_url() == "https://huggingface.co" + + def test_probe_uses_the_same_normalised_url(self, monkeypatch): + seen: list = [] + import urllib.request + + def _capture(req, *a, **k): + seen.append(req.full_url) + raise urllib.error.URLError("stop") + + import urllib.error + + monkeypatch.setenv("HF_ENDPOINT", " ") + monkeypatch.setattr(urllib.request, "urlopen", _capture) + from utils.transformers_version import hf_endpoint_unreachable + + hf_endpoint_unreachable(timeout = 1) + assert seen and seen[0].startswith("https://huggingface.co"), seen + + +class TestIpv6Endpoint: + def test_ipv6_literal_resolves(self): + """gethostbyname is IPv4-only and would call an AAAA-only mirror dead.""" + from utils.utils import dns_host_dead + + assert dns_host_dead("::1", timeout = 2.0) is False + + def test_unresolvable_host_still_dead(self): + from utils.utils import dns_host_dead + + assert dns_host_dead("no-such-host.invalid", timeout = 2.0) is True + + class TestGuardSkipsLocalPaths: """A local path never reaches the hub, so probing costs time and prevents nothing.""" diff --git a/studio/backend/utils/transformers_version.py b/studio/backend/utils/transformers_version.py index 34576decc2..7ca47198a0 100644 --- a/studio/backend/utils/transformers_version.py +++ b/studio/backend/utils/transformers_version.py @@ -66,7 +66,10 @@ def _env_offline() -> bool: def hf_endpoint_unreachable(timeout: int = 3, *, gateway_errors_offline: bool = True) -> bool: """Bounded reachability probe to the HF endpoint. A HEAD request runs in a daemon thread joined with a deadline, so a resolver blackhole cannot block past ~timeout+1s. True if - unreachable. urllib natively honors *_PROXY / NO_PROXY, so this verifies real egress + unreachable. A clean socket timeout is resolved with a TCP connect, so a slow-but-live + endpoint is not mistaken for no egress (a hang past the deadline still counts as + unreachable, since the real hub calls would hang too). + urllib natively honors *_PROXY / NO_PROXY, so this verifies real egress (the proxy can reach HF), not just that the proxy is up. No ML imports, so it is safe to call before transformers version activation. Mirrors the probe in export._hf_offline.""" import ssl @@ -74,11 +77,18 @@ def hf_endpoint_unreachable(timeout: int = 3, *, gateway_errors_offline: bool = import urllib.error import urllib.request - endpoint = os.environ.get("HF_ENDPOINT", "https://huggingface.co") - if "://" not in endpoint: - endpoint = "https://" + endpoint + # Shared normaliser, so an empty or whitespace HF_ENDPOINT falls back to the default + # hub here exactly as it does in the DNS shortcut, instead of probing "https://". + try: + from utils.utils import hf_endpoint_url - result = {"online": False} + endpoint = hf_endpoint_url() + except Exception: + endpoint = (os.environ.get("HF_ENDPOINT") or "").strip() or "https://huggingface.co" + if "://" not in endpoint: + endpoint = "https://" + endpoint + + result = {"online": False, "timed_out": False} def _probe(): try: @@ -96,16 +106,38 @@ def hf_endpoint_unreachable(timeout: int = 3, *, gateway_errors_offline: bool = except urllib.error.URLError as exc: # A TLS/cert failure means we DID reach the server; treat as reachable so the real # load surfaces it (consistent with _is_offline_related_error not retrying TLS). - result["online"] = isinstance(exc.reason, ssl.SSLError) + if isinstance(exc.reason, ssl.SSLError): + result["online"] = True + elif isinstance(exc.reason, TimeoutError): + # Resolved below, off-thread, so the extra probe cannot outrun the join. + result["timed_out"] = True + else: + result["online"] = False except ssl.SSLError: result["online"] = True + except TimeoutError: + result["timed_out"] = True except Exception: result["online"] = False t = threading.Thread(target = _probe, daemon = True) t.start() t.join(timeout + 1) - return t.is_alive() or not result["online"] + if t.is_alive(): + # Hung past its own timeout (wedged resolver): the real hub calls would hang the + # same way, so cache-only is the useful answer. Distinct from the slow-answer case + # below, where the socket timed out cleanly. + return True + if result["timed_out"]: + # A slow server still completes the TCP handshake; a blackholed route does not. + # Bounded separately so the whole probe stays within a predictable deadline. + try: + from utils.utils import hf_tcp_reachable + + return not hf_tcp_reachable(min(timeout, 2.0), endpoint) + except Exception: + return True + return not result["online"] def _safe_is_file(p: Path) -> bool: diff --git a/studio/backend/utils/utils.py b/studio/backend/utils/utils.py index db9bace997..1f9f1e9580 100644 --- a/studio/backend/utils/utils.py +++ b/studio/backend/utils/utils.py @@ -80,13 +80,17 @@ def hf_proxy_configured() -> bool: def dns_host_dead(host: str, timeout: float = 2.0) -> bool: """True when host does not resolve. Runs on a daemon thread so a wedged resolver - cannot block past the deadline and so socket.setdefaulttimeout is left alone.""" + cannot block past the deadline and so socket.setdefaulttimeout is left alone. + + Uses getaddrinfo, not gethostbyname: the latter is IPv4-only and would call an + AAAA-only mirror or an IPv6 literal dead. + """ result: list = [None] def _probe() -> None: import socket as _socket try: - _socket.gethostbyname(host) + _socket.getaddrinfo(host, None) result[0] = False except Exception: result[0] = True @@ -98,6 +102,48 @@ def dns_host_dead(host: str, timeout: float = 2.0) -> bool: return True if result[0] is None else result[0] +def hf_connect_target(endpoint: Optional[str] = None): + """(host, port) egress actually has to reach: the proxy when one applies, else the endpoint.""" + from urllib.parse import urlparse + + url = endpoint or hf_endpoint_url() + parsed = urlparse(url) + default_port = 443 if parsed.scheme == "https" else 80 + try: + import urllib.request + + proxies = urllib.request.getproxies() + proxy = proxies.get(parsed.scheme) or proxies.get("all") + host = parsed.hostname or "" + if proxy and not (host and urllib.request.proxy_bypass(host)): + p = urlparse(proxy if "://" in proxy else "http://" + proxy) + return p.hostname, p.port or 80 + except Exception: + pass + return parsed.hostname, parsed.port or default_port + + +def hf_tcp_reachable(timeout: float = 3.0, endpoint: Optional[str] = None) -> bool: + """True when a TCP connection to the hub (or its proxy) can be established. + + Separates "no egress" from "slow to answer": a loaded server still completes the + handshake promptly, whereas a blackholed route times out. A refused connection also + counts as reachable, since something answered. + """ + import socket as _socket + + host, port = hf_connect_target(endpoint) + if not host: + return False + try: + with _socket.create_connection((host, port), timeout = timeout): + return True + except ConnectionRefusedError: + return True + except Exception: + return False + + def hf_dns_dead(timeout: float = 2.0) -> bool: """Fast offline shortcut: the endpoint's host does not resolve and no proxy is in play. From 59929802c0b419c75802cab44258d74f6a310870 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 07:25:57 +0000 Subject: [PATCH 08/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/tests/test_offline_gguf_cache_fallback.py | 2 -- studio/backend/utils/transformers_version.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/studio/backend/tests/test_offline_gguf_cache_fallback.py b/studio/backend/tests/test_offline_gguf_cache_fallback.py index e49045df03..c92391f2f0 100644 --- a/studio/backend/tests/test_offline_gguf_cache_fallback.py +++ b/studio/backend/tests/test_offline_gguf_cache_fallback.py @@ -1180,12 +1180,10 @@ class TestIpv6Endpoint: def test_ipv6_literal_resolves(self): """gethostbyname is IPv4-only and would call an AAAA-only mirror dead.""" from utils.utils import dns_host_dead - assert dns_host_dead("::1", timeout = 2.0) is False def test_unresolvable_host_still_dead(self): from utils.utils import dns_host_dead - assert dns_host_dead("no-such-host.invalid", timeout = 2.0) is True diff --git a/studio/backend/utils/transformers_version.py b/studio/backend/utils/transformers_version.py index 7ca47198a0..02e29677d5 100644 --- a/studio/backend/utils/transformers_version.py +++ b/studio/backend/utils/transformers_version.py @@ -81,7 +81,6 @@ def hf_endpoint_unreachable(timeout: int = 3, *, gateway_errors_offline: bool = # hub here exactly as it does in the DNS shortcut, instead of probing "https://". try: from utils.utils import hf_endpoint_url - endpoint = hf_endpoint_url() except Exception: endpoint = (os.environ.get("HF_ENDPOINT") or "").strip() or "https://huggingface.co" @@ -133,7 +132,6 @@ def hf_endpoint_unreachable(timeout: int = 3, *, gateway_errors_offline: bool = # Bounded separately so the whole probe stays within a predictable deadline. try: from utils.utils import hf_tcp_reachable - return not hf_tcp_reachable(min(timeout, 2.0), endpoint) except Exception: return True From 6492d367192c73948926b7c3b5b31e186ba4c550 Mon Sep 17 00:00:00 2001 From: Unsloth Date: Wed, 29 Jul 2026 01:49:45 -0700 Subject: [PATCH 09/10] Address review round 4: refcount concurrent guards, distrust proxy TCP - Overlapping requests lost offline mid-flight. A later guard saw the HF_HUB_OFFLINE that an earlier one had set and took the no-op branch, so when the earlier guard exited it restored the constants and sessions while the later request was still resolving hub files, dropping it back onto the retry path. Each guard now holds its own reference on the refcounted force_hf_offline window. A user-supplied offline variable is still left untouched, told apart via force_hf_offline_active(). - The socket-timeout fallback trusted a TCP handshake to the proxy, which only proves the proxy is up, not that it can reach the hub. A live proxy with a blackholed upstream therefore read as reachable. With a proxy configured the timeout now stays unreachable; the TCP check is only evidence when connecting to the endpoint directly. Verified: second guard engages and offline survives the first guard's exit, state fully restored after both; dead-upstream proxy reads unreachable while a slow direct endpoint still reads reachable; 9 concurrent metadata requests against an unreachable hub all return 200 in 5.1s total. --- studio/backend/core/inference/llama_cpp.py | 58 ++++++------- .../tests/test_offline_gguf_cache_fallback.py | 82 +++++++++++++++++++ studio/backend/utils/transformers_version.py | 7 +- studio/backend/utils/utils.py | 11 +++ 4 files changed, 123 insertions(+), 35 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 611184655e..0b29f42501 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -540,45 +540,35 @@ def _hf_unreachable() -> bool: @contextlib.contextmanager def _hf_offline_if_unreachable(): - """Set HF_HUB_OFFLINE for this block only when the Hub is unreachable; - restores env on exit so a transient hiccup can't quarantine the process. - No-op if the user already set it.""" - if "HF_HUB_OFFLINE" in os.environ: - yield False - return - if not _hf_unreachable(): - yield False - return + """Force HF offline for this block only when the Hub is unreachable; restores on exit + so a transient hiccup can't quarantine the process. No-op if the USER set the env var. - logger.warning("Hugging Face endpoint unreachable; using local HF cache for this load.") - - # Env vars alone do not take effect mid-process: huggingface_hub and transformers read - # their offline constants at import, so the calls below would still retry. - force_ctx = None + Env vars alone do not take effect mid-process (huggingface_hub and transformers read + their offline constants at import), so this holds a refcounted force_hf_offline + window. Overlapping requests each take their own reference: no-opping because another + guard already set HF_HUB_OFFLINE would let that guard's exit restore the network + mid-operation and drop this request back onto the retry path. + """ try: - from utils.utils import force_hf_offline - force_ctx = force_hf_offline() - force_ctx.__enter__() + from utils.utils import force_hf_offline, force_hf_offline_active except Exception: - force_ctx = None + yield False + return - transformers_was_set = "TRANSFORMERS_OFFLINE" in os.environ - if force_ctx is None: - os.environ["HF_HUB_OFFLINE"] = "1" - if not transformers_was_set: - os.environ["TRANSFORMERS_OFFLINE"] = "1" - try: + ours = force_hf_offline_active() + # A user-set offline var is theirs: don't probe it, don't touch it. + if "HF_HUB_OFFLINE" in os.environ and not ours: + yield False + return + # Already forced by an in-flight guard: reuse that verdict and hold a reference. + if not ours and not _hf_unreachable(): + yield False + return + + if not ours: + logger.warning("Hugging Face endpoint unreachable; using local HF cache for this load.") + with force_hf_offline(): yield True - finally: - if force_ctx is not None: - try: - force_ctx.__exit__(None, None, None) - except Exception: - pass - else: - os.environ.pop("HF_HUB_OFFLINE", None) - if not transformers_was_set: - os.environ.pop("TRANSFORMERS_OFFLINE", None) def _hf_offline_if_unreachable_for(model_name): diff --git a/studio/backend/tests/test_offline_gguf_cache_fallback.py b/studio/backend/tests/test_offline_gguf_cache_fallback.py index c92391f2f0..b8929ab243 100644 --- a/studio/backend/tests/test_offline_gguf_cache_fallback.py +++ b/studio/backend/tests/test_offline_gguf_cache_fallback.py @@ -1146,6 +1146,88 @@ class TestSlowLinkIsNotOffline: assert hf_tcp_reachable(1, "http://127.0.0.1:9") is True +class TestConcurrentGuardsHoldTheirOwnReference: + """Overlapping requests must each hold a reference. If a later guard no-ops because an + earlier one already set HF_HUB_OFFLINE, the earlier guard's exit restores the network + while the later request is still resolving hub files.""" + + def test_second_guard_engages_and_offline_survives_first_exit( + self, monkeypatch, clean_offline_env + ): + import threading + + import core.inference.llama_cpp as lc + + monkeypatch.setattr(lc, "_hf_unreachable", lambda: True) + + a_in, b_in, a_done = threading.Event(), threading.Event(), threading.Event() + seen: dict = {} + + def worker_a(): + with lc._hf_offline_if_unreachable(): + a_in.set() + b_in.wait(5) + a_done.set() + + def worker_b(): + a_in.wait(5) + with lc._hf_offline_if_unreachable() as engaged: + seen["engaged"] = engaged + b_in.set() + a_done.wait(5) + seen["offline_after_a_exit"] = hf_constants.HF_HUB_OFFLINE + seen["env_after_a_exit"] = os.environ.get("HF_HUB_OFFLINE") + + ta, tb = threading.Thread(target = worker_a), threading.Thread(target = worker_b) + ta.start(); tb.start(); ta.join(20); tb.join(20) + + assert seen.get("engaged") is True, "second guard no-opped instead of taking a reference" + assert seen.get("offline_after_a_exit") is True + assert seen.get("env_after_a_exit") == "1" + # Both windows closed -> fully restored. + assert hf_constants.HF_HUB_OFFLINE is False + assert "HF_HUB_OFFLINE" not in os.environ + + def test_user_set_offline_is_still_a_noop(self, monkeypatch, clean_offline_env): + import core.inference.llama_cpp as lc + + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + monkeypatch.setattr(lc, "_hf_unreachable", lambda: True) + with lc._hf_offline_if_unreachable() as engaged: + assert engaged is False + assert os.environ["HF_HUB_OFFLINE"] == "1" + + +class TestProxyTimeoutIsNotExcused: + """A live proxy proves the proxy is up, not that it can reach the hub.""" + + def _probe_timing_out(self, monkeypatch): + import urllib.error + import urllib.request + + monkeypatch.setattr( + urllib.request, + "urlopen", + lambda *a, **k: (_ for _ in ()).throw(urllib.error.URLError(TimeoutError("slow"))), + ) + from utils.transformers_version import hf_endpoint_unreachable + + return hf_endpoint_unreachable + + def test_timeout_through_a_proxy_stays_unreachable(self, monkeypatch): + probe = self._probe_timing_out(monkeypatch) + monkeypatch.setattr("utils.utils.hf_proxy_configured", lambda: True) + # Even if the proxy itself accepts TCP, the hub behind it may be blackholed. + monkeypatch.setattr("utils.utils.hf_tcp_reachable", lambda *a, **k: True) + assert probe(timeout = 1) is True + + def test_timeout_without_a_proxy_still_uses_tcp(self, monkeypatch): + probe = self._probe_timing_out(monkeypatch) + monkeypatch.setattr("utils.utils.hf_proxy_configured", lambda: False) + monkeypatch.setattr("utils.utils.hf_tcp_reachable", lambda *a, **k: True) + assert probe(timeout = 1) is False + + class TestEndpointNormalisation: """An empty or whitespace HF_ENDPOINT must fall back to the default hub in BOTH the DNS shortcut and the HTTP probe, or the two stages disagree.""" diff --git a/studio/backend/utils/transformers_version.py b/studio/backend/utils/transformers_version.py index 02e29677d5..8af30a2d1b 100644 --- a/studio/backend/utils/transformers_version.py +++ b/studio/backend/utils/transformers_version.py @@ -131,7 +131,12 @@ def hf_endpoint_unreachable(timeout: int = 3, *, gateway_errors_offline: bool = # A slow server still completes the TCP handshake; a blackholed route does not. # Bounded separately so the whole probe stays within a predictable deadline. try: - from utils.utils import hf_tcp_reachable + from utils.utils import hf_proxy_configured, hf_tcp_reachable + if hf_proxy_configured(): + # Through a proxy the handshake only proves the proxy is up, not that it + # can reach the hub, so a timeout stays unreachable rather than being + # excused by a live proxy with a dead upstream. + return True return not hf_tcp_reachable(min(timeout, 2.0), endpoint) except Exception: return True diff --git a/studio/backend/utils/utils.py b/studio/backend/utils/utils.py index 1f9f1e9580..f536fdaf9c 100644 --- a/studio/backend/utils/utils.py +++ b/studio/backend/utils/utils.py @@ -247,6 +247,17 @@ _OFFLINE_CONSTANTS = ( ) +def force_hf_offline_active() -> bool: + """True while a force_hf_offline window is open anywhere in this process. + + Lets a concurrent caller tell our own forced offline apart from one the user set, so + it can hold its own reference instead of no-opping and losing offline when the first + window exits. + """ + with _force_offline_lock: + return _force_offline_depth > 0 + + @contextmanager def force_hf_offline(): """Force HF offline for this block, in-process. From 93dafbd429f9131a9dca69d14175b0e68a28dd75 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 08:50:27 +0000 Subject: [PATCH 10/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/tests/test_offline_gguf_cache_fallback.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/studio/backend/tests/test_offline_gguf_cache_fallback.py b/studio/backend/tests/test_offline_gguf_cache_fallback.py index b8929ab243..61a30ebaed 100644 --- a/studio/backend/tests/test_offline_gguf_cache_fallback.py +++ b/studio/backend/tests/test_offline_gguf_cache_fallback.py @@ -1179,7 +1179,10 @@ class TestConcurrentGuardsHoldTheirOwnReference: seen["env_after_a_exit"] = os.environ.get("HF_HUB_OFFLINE") ta, tb = threading.Thread(target = worker_a), threading.Thread(target = worker_b) - ta.start(); tb.start(); ta.join(20); tb.join(20) + ta.start() + tb.start() + ta.join(20) + tb.join(20) assert seen.get("engaged") is True, "second guard no-opped instead of taking a reference" assert seen.get("offline_after_a_exit") is True