From 8b9ee5facb9858d728a283c931c5aca8e9926710 Mon Sep 17 00:00:00 2001 From: Souravrajvi0 <144546710+Souravrajvi0@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:29:48 +0530 Subject: [PATCH] avoid Hub metadata probe when loading tokenizers with local_files_only (#7482) * fix: keep offline GGUF export off the Hub for VLM tokenizers (#7481) Resolve cached snapshot directories before loading PreTrainedTokenizerFast during VLM processor fallback so transformers does not call is_base_mistral() -> model_info() when HF_HUB_OFFLINE is set. Also probe the local cache in _has_tokenizer_model instead of model_info when offline. Fixes unslothai/unsloth#7481 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * test: add real-cache offline GGUF integration checks for #7481 Download unsloth/gemma-3-270m-it-bnb-4bit (~430MB) and verify offline snapshot resolution and tokenizer load with network blocked. Full unsloth import tests remain GPU-gated. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix: address Codex review on offline GGUF tokenizer paths (#7481) - Only rewrite Hub repo ids to cached snapshot dirs when offline - Copy tokenizer.model from cache offline in preserve_sentencepiece - Do not cache negative offline tokenizer.model probe results - Add regression tests for all three review items * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix: probe HF cache before model_info for local-only GGUF saves (#7481) Always resolve tokenizer.model from the local Hub cache before calling model_info, and skip Hub metadata when the tokenizer was loaded with local_files_only or offline env vars. Fixes Codex review on PR #7482. * Fix lint blocker, false-green tests and offline defaults for PR #7482 Drop the two unused _env_says_offline imports that fail the Source lint import-hoist check. test_has_tokenizer_model_offline_skips_model_info and its local_files_only twin set model_info.side_effect = AssertionError, but _has_tokenizer_model wraps that call in "except Exception: return False", so the AssertionError was swallowed and both passed on the merge base with the fix absent. Assert model_info.call_count == 0 instead; both now fail on the base with assert 1 == 0. The real-cache integration tests called hf_hub_download and PreTrainedTokenizerFast directly, so they exercised plain huggingface_hub and passed identically on both trees. Route them through the resolver this PR adds, and gate the file at module level since importing unsloth needs a GPU host either way. _resolve_hub_repo_local_dir and _resolve_hub_repo_cached_file defaulted to local_files_only = False, so a helper named "resolve local dir" would download with backoff retries when called without the flag. Every caller already passes it explicitly, so default it closed. Use tempfile.gettempdir() rather than a hardcoded /tmp, which silently skipped both files on Windows, the platform in the bug report. Patch socket.socket connect rather than replacing the class, which broke isinstance checks. Wire the unit tests into the Bucket-A CI list; Repo tests (CPU) ignores tests/saving, so none of these ran anywhere. * docs: note transformers 4.57.2-5.5.4 window for local tokenizer resolve Name the version range where from_pretrained still probes model_info under local_files_only, and point at the 5.6.0 upstream fix so the helper can be removed once the supported floor moves past it. * fix: enable real-cache suite in offline GGUF integration runner Pass UNSLOTH_INTEGRATION_IMPORT=1 into the pytest subprocess so the documented runner actually executes the real-cache tests instead of reporting success after only the fake-cache unit file runs. * docs: note integration runner enables UNSLOTH_INTEGRATION_IMPORT Document that the runner sets the gate itself and still needs a host that can import unsloth. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Keep an explicit local_files_only load local-only at save time transformers takes local_files_only as an explicit from_pretrained parameter, so it never lands in tokenizer.init_kwargs, and _offline_aware_load restores HF_HUB_OFFLINE / TRANSFORMERS_OFFLINE as soon as the load window closes. A VLM loaded with local_files_only = True but no offline env var therefore came back with the Hub repo id in name_or_path and nothing recording the request, so _tokenizer_wants_local_only returned False on the later save and _has_tokenizer_model fell through to HfApi.model_info - and then _preserve_sentencepiece_tokenizer_assets fetched tokenizer.model from the Hub with local_files_only = False. On a disconnected host that is a network wait before the export gives up. Stamp the load's local-only mode onto the returned processor and its tokenizer inside the forced-offline window, and honour that stamp in _tokenizer_wants_local_only, so the save path inherits the load's contract. Verified against a real hub-cache layout whose snapshot has tokenizer metadata but no tokenizer.model: before, one model_info call plus an hf_hub_download with local_files_only = False; after, zero model_info calls and cache probes only. Two tests added to tests/saving/test_offline_gguf_vlm_tokenizer_7481.py; both fail with the loader_utils hunk reverted and pass with it in place. * Carry the load's cache_dir through to saving for PR #7482 The local-only stamp added in e7b7400de preserved only the boolean. Saving still derived its cache from HF_HUB_CACHE or HF_HOME, which does not see a caller-supplied cache_dir, and FastBaseModel.from_pretrained threads one all the way down. So a local_files_only load against a custom cache missed on the probe, and the stamp then stopped the Hub fallback that used to cover it, and tokenizer.model was silently left out of the GGUF staging directory. Stamp the cache_dir alongside the local-only marker and prefer it at both sites in save.py that derive one from the environment. Reverting save.py alone, with the helper still present, fails the new test on behaviour. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Merge main and drop an unused import for PR #7482 Brings the branch up to date with main, which clears the stale Source lint blocker inherited from #7476 by taking studio/backend/utils/hardware/__init__.py out of this PR's changed-file set. pytest was imported in the new test file and never used, which the import-hoist check flags in its own right. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Daniel Han --- .github/workflows/consolidated-tests-ci.yml | 6 +- tests/saving/run_offline_gguf_integration.py | 67 ++++ ...est_offline_gguf_real_cache_integration.py | 122 +++++++ .../test_offline_gguf_vlm_tokenizer_7481.py | 336 ++++++++++++++++++ unsloth/models/loader_utils.py | 191 +++++++++- unsloth/models/vision.py | 30 +- unsloth/save.py | 71 +++- 7 files changed, 801 insertions(+), 22 deletions(-) create mode 100644 tests/saving/run_offline_gguf_integration.py create mode 100644 tests/saving/test_offline_gguf_real_cache_integration.py create mode 100644 tests/saving/test_offline_gguf_vlm_tokenizer_7481.py diff --git a/.github/workflows/consolidated-tests-ci.yml b/.github/workflows/consolidated-tests-ci.yml index d1bea819eb..489ee4ca08 100644 --- a/.github/workflows/consolidated-tests-ci.yml +++ b/.github/workflows/consolidated-tests-ci.yml @@ -7,7 +7,7 @@ # # Why a separate workflow: # - studio-backend-ci.yml's "Repo tests (CPU)" job already auto-discovers -# tests/ minus tests/qlora, tests/saving, tests/utils, tests/sh. The 16 +# tests/ minus tests/qlora, tests/saving, tests/utils, tests/sh. The 17 # Bucket-A tests below live inside those --ignore dirs (CPU-runnable but # historically excluded with their GPU siblings); pulling them out into # a sibling job keeps the existing 760-passed baseline stable while we @@ -274,6 +274,7 @@ jobs: tests/saving/test_export_dispatch.py \ tests/saving/test_imatrix_export.py \ tests/saving/test_gguf_single_pass_export.py \ + tests/saving/test_offline_gguf_vlm_tokenizer_7481.py \ tests/utils/test_attention_masks.py \ tests/utils/test_trunc_normal_patch.py \ tests/python/test_fast_language_model_text_only.py @@ -365,6 +366,7 @@ jobs: tests/saving/test_export_dispatch.py \ tests/saving/test_imatrix_export.py \ tests/saving/test_gguf_single_pass_export.py \ + tests/saving/test_offline_gguf_vlm_tokenizer_7481.py \ tests/utils/test_attention_masks.py \ tests/utils/test_trunc_normal_patch.py \ tests/python/test_fast_language_model_text_only.py \ @@ -2129,7 +2131,7 @@ jobs: pip show unsloth_zoo echo "::endgroup::" echo "Consolidated job done. Coverage:" - echo " - 16 unsloth Bucket-A tests under tests/saving/ + tests/utils/" + echo " - 17 unsloth Bucket-A tests under tests/saving/ + tests/utils/" echo " - unsloth_zoo @ ${UNSLOTH_ZOO_REF} pytest tests/ (5 GPU cases deselected)" echo " - unsloth_zoo.compiler.test_apply_fused_lm_head" diff --git a/tests/saving/run_offline_gguf_integration.py b/tests/saving/run_offline_gguf_integration.py new file mode 100644 index 0000000000..eabc5f96e6 --- /dev/null +++ b/tests/saving/run_offline_gguf_integration.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Download real Gemma weights and run offline integration tests for #7481. + +Sets ``UNSLOTH_INTEGRATION_IMPORT=1`` for the pytest subprocess so the +real-cache suite is not silently skipped. Requires a host that can import +unsloth (typically GPU). + +Example: + python tests/saving/run_offline_gguf_integration.py + python tests/saving/run_offline_gguf_integration.py --download-only +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +REPO = "unsloth/gemma-3-270m-it-bnb-4bit" +CACHE_ROOT = Path( + os.environ.get("HF_HOME") or os.path.join(tempfile.gettempdir(), "hf_offline_test_cache") +) + + +def download(): + from huggingface_hub import snapshot_download + + os.environ.setdefault("HF_HOME", str(CACHE_ROOT)) + path = snapshot_download(REPO, cache_dir = str(CACHE_ROOT / "hub")) + print("cached at", path) + + +def run_tests(): + os.environ.setdefault("HF_HOME", str(CACHE_ROOT)) + # Real-cache suite is gated on this; without it every integration test skips + # and the runner reports success after only the fake-cache unit file ran. + env = os.environ.copy() + env["UNSLOTH_INTEGRATION_IMPORT"] = "1" + cmd = [ + sys.executable, + "-m", + "pytest", + "tests/saving/test_offline_gguf_vlm_tokenizer_7481.py", + "tests/saving/test_offline_gguf_real_cache_integration.py", + "-q", + ] + raise SystemExit(subprocess.call(cmd, cwd = str(Path(__file__).resolve().parents[2]), env = env)) + + +def main(): + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--download-only", action = "store_true") + args = parser.parse_args() + download() + if not args.download_only: + run_tests() + + +if __name__ == "__main__": + main() diff --git a/tests/saving/test_offline_gguf_real_cache_integration.py b/tests/saving/test_offline_gguf_real_cache_integration.py new file mode 100644 index 0000000000..fb71ed6603 --- /dev/null +++ b/tests/saving/test_offline_gguf_real_cache_integration.py @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Integration tests for #7481 using real cached Gemma weights. + +Requires a one-time online download into ``$HF_HOME`` (defaults to a +``hf_offline_test_cache`` directory under the platform temp dir): + + HF_HOME= python -c \\ + "from huggingface_hub import snapshot_download; snapshot_download('unsloth/gemma-3-270m-it-bnb-4bit', cache_dir='/hub')" + +Every test here drives unsloth's own resolver. Resolving through +``hf_hub_download`` directly would pass with the fix reverted, since that is +plain huggingface_hub behaviour rather than anything this change touches. + +Importing unsloth pulls the whole package graph, which CPU-only hosts cannot +do, so the suite is gated behind ``UNSLOTH_INTEGRATION_IMPORT=1``. +""" + +from __future__ import annotations + +import os +import socket +import tempfile +from pathlib import Path + +import pytest + +REPO = "unsloth/gemma-3-270m-it-bnb-4bit" +CACHE_ROOT = Path( + os.environ.get("HF_HOME") or os.path.join(tempfile.gettempdir(), "hf_offline_test_cache") +) + +pytestmark = [ + pytest.mark.integration, + pytest.mark.skipif( + os.environ.get("UNSLOTH_INTEGRATION_IMPORT") != "1", + reason = "full unsloth import needs a GPU host; set UNSLOTH_INTEGRATION_IMPORT=1 to enable", + ), +] + + +def _require_cached_repo(): + from huggingface_hub import scan_cache_dir + + cache_dir = CACHE_ROOT / "hub" + if not cache_dir.exists(): + pytest.skip(f"cache missing at {cache_dir}; run snapshot_download for {REPO}") + repos = [r.repo_id for r in scan_cache_dir(str(cache_dir)).repos] + if REPO not in repos: + pytest.skip(f"{REPO} not in {cache_dir}") + + +def _block_network(monkeypatch): + def _guard(*args, **kwargs): + raise OSError("network blocked for offline integration test") + + # Patch the method, not the class: replacing socket.socket itself breaks any + # isinstance(x, socket.socket) in the stack under test. + monkeypatch.setattr(socket.socket, "connect", _guard) + monkeypatch.setattr(socket, "create_connection", _guard) + monkeypatch.setattr(socket, "getaddrinfo", _guard) + + +def _offline_env(monkeypatch): + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + monkeypatch.setenv("TRANSFORMERS_OFFLINE", "1") + monkeypatch.setenv("HF_HOME", str(CACHE_ROOT)) + + +def test_real_cached_snapshot_resolves_offline(monkeypatch): + _require_cached_repo() + _offline_env(monkeypatch) + _block_network(monkeypatch) + + from unsloth.models.loader_utils import _resolve_hub_repo_local_dir + + snap = Path( + _resolve_hub_repo_local_dir( + REPO, + cache_dir = str(CACHE_ROOT / "hub"), + local_files_only = True, + ) + ) + assert (snap / "tokenizer.json").is_file() + assert (snap / "tokenizer.model").is_file() + + +def test_real_cached_tokenizer_loads_from_snapshot_not_repo_id(monkeypatch): + """The #7481 fix: the loader hands transformers a snapshot dir, not a repo id.""" + _require_cached_repo() + _offline_env(monkeypatch) + _block_network(monkeypatch) + + from unsloth.models.loader_utils import _load_pretrained_tokenizer_fast + + tok = _load_pretrained_tokenizer_fast( + REPO, + local_files_only = True, + cache_dir = str(CACHE_ROOT / "hub"), + ) + assert tok.vocab_size > 0 + # A repo id here means the Hub metadata probe was reached, which is the bug. + assert tok.name_or_path != REPO + assert Path(tok.name_or_path).is_dir() + + +def test_real_cached_unsloth_helpers_offline(monkeypatch): + _require_cached_repo() + _offline_env(monkeypatch) + _block_network(monkeypatch) + + from unsloth.models.loader_utils import _load_pretrained_tokenizer_fast + from unsloth.save import _has_tokenizer_model + + tok = _load_pretrained_tokenizer_fast( + REPO, + local_files_only = True, + cache_dir = str(CACHE_ROOT / "hub"), + ) + assert tok.vocab_size > 0 + assert _has_tokenizer_model(tok) is True diff --git a/tests/saving/test_offline_gguf_vlm_tokenizer_7481.py b/tests/saving/test_offline_gguf_vlm_tokenizer_7481.py new file mode 100644 index 0000000000..9a15e362fc --- /dev/null +++ b/tests/saving/test_offline_gguf_vlm_tokenizer_7481.py @@ -0,0 +1,336 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Offline GGUF export must not probe the Hub for VLM tokenizer metadata (issue #7481). + +Regression for ``PreTrainedTokenizerFast.from_pretrained`` on a repo id calling +``is_base_mistral()`` -> ``model_info()`` even with ``TRANSFORMERS_OFFLINE=1``. +Pure CPU, no network, no GPU. +""" + +import json +import os +from types import SimpleNamespace +from unittest.mock import patch + +from unsloth.models import loader_utils as L + + +_REPO = "llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic" +_COMMIT = "5964fe4c7339c5974e879baba8982a09616f68ca" + + +def _write_gemma4_cache( + root, + repo_id = _REPO, + commit = _COMMIT, +): + """Minimal cached snapshot matching the reporter's layout.""" + org, name = repo_id.split("/") + repo_root = root / f"models--{org}--{name}" + snap = repo_root / "snapshots" / commit + snap.mkdir(parents = True) + refs = repo_root / "refs" + refs.mkdir(parents = True, exist_ok = True) + (refs / "main").write_text(commit, encoding = "utf-8") + (snap / "tokenizer_config.json").write_text( + json.dumps({"tokenizer_class": "GemmaTokenizer", "model_max_length": 8192}), + encoding = "utf-8", + ) + (snap / "tokenizer.json").write_text( + json.dumps( + { + "version": "1.0", + "truncation": None, + "padding": None, + "added_tokens": [], + "normalizer": None, + "pre_tokenizer": None, + "post_processor": None, + "decoder": None, + "model": {"type": "BPE", "vocab": {"": 0}, "merges": []}, + } + ), + encoding = "utf-8", + ) + (snap / "processor_config.json").write_text("{}", encoding = "utf-8") + (snap / "config.json").write_text( + json.dumps({"model_type": "gemma4"}), + encoding = "utf-8", + ) + return snap + + +def _offline_env(monkeypatch, cache_root): + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + monkeypatch.setenv("TRANSFORMERS_OFFLINE", "1") + monkeypatch.setenv("HF_HUB_CACHE", str(cache_root)) + + +def test_resolve_hub_repo_cached_file_finds_tokenizer_model(tmp_path, monkeypatch): + snap = _write_gemma4_cache(tmp_path) + (snap / "tokenizer.model").write_bytes(b"sp-model") + _offline_env(monkeypatch, tmp_path) + + got = L._resolve_hub_repo_cached_file( + _REPO, + "tokenizer.model", + local_files_only = True, + cache_dir = str(tmp_path), + ) + assert got == str(snap / "tokenizer.model") + + +def test_resolve_hub_repo_local_dir_from_cached_snapshot(tmp_path, monkeypatch): + snap = _write_gemma4_cache(tmp_path) + _offline_env(monkeypatch, tmp_path) + + got = L._resolve_hub_repo_local_dir(_REPO, local_files_only = True, cache_dir = str(tmp_path)) + assert got == str(snap) + + +def test_hub_repo_or_local_path_prefers_snapshot_over_repo_id(tmp_path, monkeypatch): + snap = _write_gemma4_cache(tmp_path) + _offline_env(monkeypatch, tmp_path) + + got = L._hub_repo_or_local_path(_REPO, local_files_only = True, cache_dir = str(tmp_path)) + assert got == str(snap) + assert got != _REPO + + +def test_hub_repo_or_local_path_keeps_repo_id_online(tmp_path, monkeypatch): + snap = _write_gemma4_cache(tmp_path) + monkeypatch.delenv("HF_HUB_OFFLINE", raising = False) + monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False) + monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path)) + + got = L._hub_repo_or_local_path(_REPO, local_files_only = False, cache_dir = str(tmp_path)) + assert got == _REPO + assert got != str(snap) + + +def test_has_tokenizer_model_offline_does_not_cache_negative(tmp_path, monkeypatch): + from unsloth.save import _TOKENIZER_MODEL_CACHE, _has_tokenizer_model + + snap = _write_gemma4_cache(tmp_path) + _offline_env(monkeypatch, tmp_path) + _TOKENIZER_MODEL_CACHE.clear() + + tok = SimpleNamespace(name_or_path = _REPO) + assert _has_tokenizer_model(tok, token = None) is False + assert _REPO not in _TOKENIZER_MODEL_CACHE + + (snap / "tokenizer.model").write_bytes(b"sp-model") + assert _has_tokenizer_model(tok, token = None) is True + + +def test_preserve_sentencepiece_offline_copies_cached_model(tmp_path, monkeypatch): + from unsloth.save import _TOKENIZER_MODEL_CACHE, _preserve_sentencepiece_tokenizer_assets + + snap = _write_gemma4_cache(tmp_path) + (snap / "tokenizer.model").write_bytes(b"cached-sp-model") + _offline_env(monkeypatch, tmp_path) + _TOKENIZER_MODEL_CACHE.clear() + + save_dir = tmp_path / "export" + save_dir.mkdir() + (save_dir / "tokenizer_config.json").write_text("{}", encoding = "utf-8") + tok = SimpleNamespace(name_or_path = _REPO) + + _preserve_sentencepiece_tokenizer_assets(tok, str(save_dir)) + + assert (save_dir / "tokenizer.model").read_bytes() == b"cached-sp-model" + + +def test_load_pretrained_tokenizer_fast_passes_snapshot_not_repo_id(tmp_path, monkeypatch): + snap = _write_gemma4_cache(tmp_path) + _offline_env(monkeypatch, tmp_path) + + seen_paths = [] + + class _FakeFast: + @classmethod + def from_pretrained(cls, path, **kwargs): + seen_paths.append(path) + assert kwargs.get("local_files_only") is True + return SimpleNamespace(name_or_path = path) + + monkeypatch.setattr( + "transformers.PreTrainedTokenizerFast", + _FakeFast, + raising = False, + ) + + with patch("huggingface_hub.HfApi.model_info") as model_info: + model_info.side_effect = AssertionError("model_info must not run offline") + tok = L._load_pretrained_tokenizer_fast(_REPO, cache_dir = str(tmp_path)) + + assert seen_paths == [str(snap)] + assert tok.name_or_path == str(snap) + + +def test_has_tokenizer_model_offline_skips_model_info(tmp_path, monkeypatch): + from unsloth.save import _TOKENIZER_MODEL_CACHE, _has_tokenizer_model + + _write_gemma4_cache(tmp_path) + _offline_env(monkeypatch, tmp_path) + _TOKENIZER_MODEL_CACHE.clear() + + tok = SimpleNamespace(name_or_path = _REPO) + + # A raising side_effect proves nothing: _has_tokenizer_model wraps the call + # in `except Exception: return False`, so it passes with the fix reverted. + with patch("huggingface_hub.HfApi.model_info") as model_info: + assert _has_tokenizer_model(tok, token = None) is False + assert model_info.call_count == 0 + + +def test_has_tokenizer_model_probes_cache_before_model_info(tmp_path, monkeypatch): + from unsloth.save import _TOKENIZER_MODEL_CACHE, _has_tokenizer_model + + snap = _write_gemma4_cache(tmp_path) + (snap / "tokenizer.model").write_bytes(b"sp-model") + monkeypatch.delenv("HF_HUB_OFFLINE", raising = False) + monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False) + monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path)) + _TOKENIZER_MODEL_CACHE.clear() + + tok = SimpleNamespace(name_or_path = _REPO) + + with patch("huggingface_hub.HfApi.model_info") as model_info: + model_info.side_effect = AssertionError("model_info must not run when cache hit") + assert _has_tokenizer_model(tok, token = None) is True + + +def test_offline_aware_load_persists_local_only_for_saving(tmp_path, monkeypatch): + """An explicit ``local_files_only = True`` load must still be local-only at save time. + + ``transformers`` takes ``local_files_only`` as an explicit ``from_pretrained`` + parameter, so it never reaches ``tokenizer.init_kwargs``, and + ``_offline_aware_load`` restores the offline env vars once the load returns. + Without the stamp the request is invisible by the time we save. + """ + from unsloth.save import _TOKENIZER_MODEL_CACHE, _has_tokenizer_model + + # Snapshot has tokenizer metadata but deliberately no tokenizer.model, so the + # cache probe misses and only the local-only stamp can stop the Hub request. + _write_gemma4_cache(tmp_path) + monkeypatch.delenv("HF_HUB_OFFLINE", raising = False) + monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False) + monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path)) + _TOKENIZER_MODEL_CACHE.clear() + + @L._offline_aware_load + def _load(model_name, **kwargs): + assert os.environ.get("HF_HUB_OFFLINE") == "1" + # A processor keeps the Hub repo id and carries no local_files_only. + return object(), SimpleNamespace( + tokenizer = SimpleNamespace(name_or_path = model_name, init_kwargs = {}), + ) + + _model, processor = _load(_REPO, local_files_only = True) + + assert os.environ.get("HF_HUB_OFFLINE") is None + assert processor.tokenizer.init_kwargs.get("local_files_only") is None + assert L._tokenizer_wants_local_only(processor.tokenizer) is True + + with patch("huggingface_hub.HfApi.model_info") as model_info: + model_info.return_value = SimpleNamespace( + siblings = [SimpleNamespace(rfilename = "tokenizer.model")], + ) + assert _has_tokenizer_model(processor, token = None) is False + assert model_info.call_count == 0 + + +def test_preserve_sentencepiece_after_local_only_load_never_downloads(tmp_path, monkeypatch): + """The save path inherits the load's local-only mode: no metadata probe, no download.""" + import huggingface_hub + + from unsloth.save import _TOKENIZER_MODEL_CACHE, _preserve_sentencepiece_tokenizer_assets + + _write_gemma4_cache(tmp_path) + monkeypatch.delenv("HF_HUB_OFFLINE", raising = False) + monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False) + monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path)) + _TOKENIZER_MODEL_CACHE.clear() + + @L._offline_aware_load + def _load(model_name, **kwargs): + return object(), SimpleNamespace( + tokenizer = SimpleNamespace(name_or_path = model_name, init_kwargs = {}), + ) + + _model, processor = _load(_REPO, local_files_only = True) + + save_dir = tmp_path / "export" + save_dir.mkdir() + (save_dir / "tokenizer_config.json").write_text("{}", encoding = "utf-8") + + real_download = huggingface_hub.hf_hub_download + seen_local_files_only = [] + + def _recording_download(*args, **kwargs): + seen_local_files_only.append(kwargs.get("local_files_only")) + return real_download(*args, **kwargs) + + monkeypatch.setattr("huggingface_hub.hf_hub_download", _recording_download) + + with patch("huggingface_hub.HfApi.model_info") as model_info: + model_info.return_value = SimpleNamespace( + siblings = [SimpleNamespace(rfilename = "tokenizer.model")], + ) + _preserve_sentencepiece_tokenizer_assets(processor, str(save_dir), token = None) + + assert model_info.call_count == 0 + # Every hf_hub_download here must be a cache probe, never a Hub fetch. + assert seen_local_files_only and all(seen_local_files_only) + assert not (save_dir / "tokenizer.model").exists() + + +def test_has_tokenizer_model_local_files_only_skips_model_info(tmp_path, monkeypatch): + from unsloth.save import _TOKENIZER_MODEL_CACHE, _has_tokenizer_model + + _write_gemma4_cache(tmp_path) + monkeypatch.delenv("HF_HUB_OFFLINE", raising = False) + monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False) + monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path)) + _TOKENIZER_MODEL_CACHE.clear() + + tok = SimpleNamespace( + name_or_path = _REPO, + init_kwargs = {"local_files_only": True}, + ) + + with patch("huggingface_hub.HfApi.model_info") as model_info: + assert _has_tokenizer_model(tok, token = None) is False + assert model_info.call_count == 0 + + +def test_custom_cache_dir_survives_to_saving(tmp_path, monkeypatch): + """A local-only load with a caller-supplied cache_dir that no env var points + at. Saving derives its cache from HF_HUB_CACHE / HF_HOME, so without the + stamp it probes the wrong place, and the local-only marker then stops it + falling back to the Hub, silently dropping tokenizer.model.""" + from unsloth.save import _TOKENIZER_MODEL_CACHE, _has_tokenizer_model + + custom_cache = tmp_path / "caller_cache" + custom_cache.mkdir() + snap = _write_gemma4_cache(custom_cache) + (snap / "tokenizer.model").write_bytes(b"sp-model") + + # The environment points somewhere else entirely. + monkeypatch.delenv("HF_HUB_OFFLINE", raising = False) + monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising = False) + monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path / "unrelated")) + _TOKENIZER_MODEL_CACHE.clear() + + @L._offline_aware_load + def _load(**kwargs): + return SimpleNamespace(name_or_path = _REPO) + + tok = _load(local_files_only = True, cache_dir = str(custom_cache)) + + assert L._tokenizer_cache_dir(tok) == str(custom_cache) + with patch("huggingface_hub.HfApi.model_info") as model_info: + assert _has_tokenizer_model(tok, token = None) is True + assert model_info.call_count == 0 diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 7aae75fe4f..e8d0fa657c 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -875,6 +875,55 @@ def _get_effective_local_files_only(kwargs): return _env_says_offline() +# Attribute stamped on a tokenizer/processor that was loaded local-only, so a later +# save still knows. transformers takes local_files_only as an explicit from_pretrained +# parameter and never copies it into tokenizer.init_kwargs, and _offline_aware_load +# restores the offline env vars when the load window closes, so without this stamp an +# explicit local_files_only = True load is invisible by the time we save (issue #7481). +_LOCAL_FILES_ONLY_ATTR = "_unsloth_local_files_only" +# The load's cache_dir travels with it too: saving derives one from HF_HUB_CACHE / +# HF_HOME, which does not see a caller-supplied cache. +_LOADED_CACHE_DIR_ATTR = "_unsloth_loaded_cache_dir" + + +def _mark_loaded_local_files_only(result, cache_dir = None): + """Stamp a load's local-only mode and cache_dir onto the returned objects.""" + for obj in result if isinstance(result, (tuple, list)) else (result,): + try: + # A processor keeps the tokenizer that _has_tokenizer_model unwraps to, + # so stamp both (a wrapped model can raise from its own __getattr__). + targets = (obj, getattr(obj, "tokenizer", None)) + except Exception: + targets = (obj,) + for target in targets: + if target is None: + continue + # Objects that reject new attributes (__slots__) are skipped. + try: + setattr(target, _LOCAL_FILES_ONLY_ATTR, True) + if cache_dir: + setattr(target, _LOADED_CACHE_DIR_ATTR, str(cache_dir)) + except Exception: + pass + return result + + +def _tokenizer_cache_dir(tokenizer): + """The cache_dir the load used, when it was not the environment's.""" + tokenizer = tokenizer.tokenizer if hasattr(tokenizer, "tokenizer") else tokenizer + return getattr(tokenizer, _LOADED_CACHE_DIR_ATTR, None) + + +def _tokenizer_wants_local_only(tokenizer): + """True when Hub metadata probes should be skipped for this tokenizer.""" + if _env_says_offline(): + return True + if getattr(tokenizer, _LOCAL_FILES_ONLY_ATTR, False): + return True + init_kwargs = getattr(tokenizer, "init_kwargs", None) or {} + return bool(init_kwargs.get("local_files_only")) + + def _is_offline_related_error(exc): """True if exc (or its cause/context chain) is a lost-connection error, not a missing file. Plain FileNotFoundError propagates; LocalEntryNotFoundError is offline.""" @@ -1099,7 +1148,9 @@ def _offline_aware_load(fn): if _get_effective_local_files_only(kwargs): kwargs["local_files_only"] = True with _force_hf_offline(): - return fn(*args, **kwargs) + # Stamp inside the window: the env vars are restored on exit, so the + # request has to travel on the objects themselves to reach saving. + return _mark_loaded_local_files_only(fn(*args, **kwargs), kwargs.get("cache_dir")) _pb_were_disabled = _progress_bars_were_disabled() # restore before any retry try: return fn(*args, **kwargs) @@ -1158,6 +1209,144 @@ def _has_local_processor_files(path): ) +def _resolve_hub_repo_local_dir( + repo_id, + *, + token = None, + cache_dir = None, + # Default closed: a "resolve local dir" helper must not download. False here + # means five filenames each retried with backoff before it gives up. + local_files_only = True, + filenames = ( + "tokenizer_config.json", + "config.json", + "tokenizer.json", + "preprocessor_config.json", + "processor_config.json", + ), +): + """Return a local snapshot directory for a Hub repo id when files are cached. + + On transformers 4.57.2 through 5.5.4, ``PreTrainedTokenizerFast.from_pretrained`` + on a repo id can still call ``model_info()`` when ``local_files_only=True`` and + no offline env var is set. Loading from the resolved snapshot dir avoids that + Hub probe. Upstream fixed this in transformers 5.6.0 (huggingface/transformers#43603); + this helper can be removed once the supported floor is past that version. + """ + if not isinstance(repo_id, str) or not repo_id: + return None + if os.path.isdir(repo_id): + return repo_id + if cache_dir is None: + cache_dir = os.environ.get("HF_HUB_CACHE") + from huggingface_hub import hf_hub_download + + for filename in filenames: + try: + path = hf_hub_download( + repo_id = repo_id, + filename = filename, + token = token, + cache_dir = cache_dir, + local_files_only = local_files_only, + ) + if path and os.path.isfile(path): + return os.path.dirname(path) + except Exception: + continue + return None + + +def _resolve_hub_repo_cached_file( + repo_id, + filename, + *, + token = None, + cache_dir = None, + local_files_only = True, +): + """Return a cached file path under a Hub snapshot, or None if absent.""" + local_dir = _resolve_hub_repo_local_dir( + repo_id, + token = token, + cache_dir = cache_dir, + local_files_only = local_files_only, + filenames = (filename,), + ) + if local_dir is None: + return None + path = os.path.join(local_dir, filename) + return path if os.path.isfile(path) else None + + +def _hub_repo_or_local_path( + repo_id, + *, + token = None, + cache_dir = None, + local_files_only = False, + filenames = None, +): + """Prefer a cached snapshot path over a Hub repo id when offline or ``local_files_only``.""" + if isinstance(repo_id, str) and os.path.isdir(repo_id): + return repo_id + lfo = bool(local_files_only) or _env_says_offline() + if not lfo: + return repo_id + local_dir = _resolve_hub_repo_local_dir( + repo_id, + token = token, + cache_dir = cache_dir, + local_files_only = True, + filenames = filenames + or ( + "tokenizer_config.json", + "config.json", + "tokenizer.json", + "preprocessor_config.json", + "processor_config.json", + ), + ) + return local_dir if local_dir is not None else repo_id + + +def _load_pretrained_tokenizer_fast( + tokenizer_name, + *, + padding_side = "left", + token = None, + trust_remote_code = False, + cache_dir = None, + local_files_only = False, +): + """Load ``PreTrainedTokenizerFast`` without Hub metadata probes when cached/offline. + + Needed on transformers 4.57.2-5.5.4; redundant once the floor is past 5.6.0. + """ + from transformers import PreTrainedTokenizerFast + + lfo = bool(local_files_only) or _env_says_offline() + load_path = _hub_repo_or_local_path( + tokenizer_name, + token = token, + cache_dir = cache_dir, + local_files_only = lfo, + filenames = ( + "tokenizer_config.json", + "tokenizer.json", + "tokenizer.model", + ), + ) + return PreTrainedTokenizerFast.from_pretrained( + load_path, + padding_side = padding_side, + token = token, + trust_remote_code = trust_remote_code, + cache_dir = cache_dir, + local_files_only = lfo, + ) + + def _resolve_checkpoint_tokenizer_name( old_model_name, kwargs, diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index dc8032e7ba..0d55f272e3 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -631,7 +631,9 @@ def unsloth_base_fast_generate(self, *args, **kwargs): # Offline helpers live in loader_utils.py (shared canonical source). from .loader_utils import ( _get_effective_local_files_only, + _hub_repo_or_local_path, _is_offline_related_error, + _load_pretrained_tokenizer_fast, _offline_aware_load, ) @@ -667,20 +669,27 @@ def _construct_vlm_processor_fallback( tell an offline failure (retry from cache) from a genuine one.""" _fb_err = None try: - from transformers import AutoImageProcessor, PreTrainedTokenizerFast, AutoConfig + from transformers import AutoImageProcessor, AutoConfig from transformers.models.auto.processing_auto import PROCESSOR_MAPPING_NAMES import json + load_path = _hub_repo_or_local_path( + tokenizer_name, + token = token, + cache_dir = cache_dir, + local_files_only = local_files_only, + ) # Load image processor image_processor = AutoImageProcessor.from_pretrained( - tokenizer_name, + load_path, token = token, trust_remote_code = trust_remote_code, cache_dir = cache_dir, local_files_only = local_files_only, ) - # Load tokenizer via PreTrainedTokenizerFast (bypasses tokenizer_class check) - tok = PreTrainedTokenizerFast.from_pretrained( + # Load tokenizer via PreTrainedTokenizerFast (bypasses tokenizer_class check). + # Resolve the cached snapshot first so transformers does not call model_info (#7481). + tok = _load_pretrained_tokenizer_fast( tokenizer_name, padding_side = "left", token = token, @@ -740,7 +749,7 @@ def _construct_vlm_processor_fallback( # Try the top-level config.model_type which often has the processor mapping. try: config = AutoConfig.from_pretrained( - tokenizer_name, + load_path, token = token, trust_remote_code = trust_remote_code, cache_dir = cache_dir, @@ -1653,9 +1662,15 @@ class FastBaseModel: # Last resort: AutoTokenizer, then PreTrainedTokenizerFast (raise on network failure to retry). def _last_resort_tokenizer(lfo): from transformers import AutoTokenizer as _AutoTokenizer + load_path = _hub_repo_or_local_path( + tokenizer_name, + token = token, + cache_dir = kwargs.get("cache_dir"), + local_files_only = lfo, + ) try: return _AutoTokenizer.from_pretrained( - tokenizer_name, + load_path, padding_side = "left", token = token, trust_remote_code = trust_remote_code, @@ -1663,8 +1678,7 @@ class FastBaseModel: local_files_only = lfo, ) except Exception: - from transformers import PreTrainedTokenizerFast - return PreTrainedTokenizerFast.from_pretrained( + return _load_pretrained_tokenizer_fast( tokenizer_name, padding_side = "left", token = token, diff --git a/unsloth/save.py b/unsloth/save.py index 30ea18b066..9bd13bb4d5 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -52,7 +52,12 @@ import traceback import psutil import re from transformers.models.llama.modeling_llama import logger -from .models.loader_utils import get_model_name +from .models.loader_utils import ( + get_model_name, + _resolve_hub_repo_cached_file, + _tokenizer_cache_dir, + _tokenizer_wants_local_only, +) from .models._utils import _convert_torchao_model from .ollama_template_mappers import OLLAMA_TEMPLATES, MODEL_TO_OLLAMA_TEMPLATE_MAPPER from transformers import ProcessorMixin, PreTrainedTokenizerBase @@ -446,6 +451,27 @@ def _has_tokenizer_model(tokenizer, token = None): if source in _TOKENIZER_MODEL_CACHE: return _TOKENIZER_MODEL_CACHE[source] + # Hub repo id: probe local cache before model_info (issue #7481). + cache_dir = _tokenizer_cache_dir(tokenizer) or os.environ.get("HF_HUB_CACHE") + if not cache_dir: + hf_home = os.environ.get("HF_HOME") + if hf_home: + cache_dir = os.path.join(hf_home, "hub") + + cached_path = _resolve_hub_repo_cached_file( + source, + "tokenizer.model", + token = token, + local_files_only = True, + cache_dir = cache_dir, + ) + if cached_path is not None: + _TOKENIZER_MODEL_CACHE[source] = True + return True + + if _tokenizer_wants_local_only(tokenizer): + return False + try: repo_info = HfApi(token = token).model_info(source, files_metadata = False) except Exception: @@ -505,15 +531,33 @@ def _preserve_sentencepiece_tokenizer_assets( if os.path.isfile(local_path): downloaded_path = local_path else: - from huggingface_hub import hf_hub_download - try: - downloaded_path = hf_hub_download( - repo_id = source, - filename = "tokenizer.model", - token = token, - ) - except Exception: - downloaded_path = None + cache_dir = _tokenizer_cache_dir(tokenizer) or os.environ.get("HF_HUB_CACHE") + if not cache_dir: + hf_home = os.environ.get("HF_HOME") + if hf_home: + cache_dir = os.path.join(hf_home, "hub") + + cached_path = _resolve_hub_repo_cached_file( + source, + "tokenizer.model", + token = token, + local_files_only = True, + cache_dir = cache_dir, + ) + if cached_path is not None: + downloaded_path = cached_path + else: + from huggingface_hub import hf_hub_download + try: + downloaded_path = hf_hub_download( + repo_id = source, + filename = "tokenizer.model", + token = token, + local_files_only = _tokenizer_wants_local_only(tokenizer), + cache_dir = cache_dir, + ) + except Exception: + downloaded_path = None if not os.path.isfile(tokenizer_model) and downloaded_path is not None: shutil.copy2(downloaded_path, tokenizer_model) @@ -3793,7 +3837,12 @@ def unsloth_convert_lora_to_ggml_and_save_locally( return _unsloth_save_lora_gguf(self, tokenizer, save_directory, outtype = outtype) -from .models.loader_utils import get_model_name +from .models.loader_utils import ( + get_model_name, + _resolve_hub_repo_cached_file, + _tokenizer_cache_dir, + _tokenizer_wants_local_only, +) from unsloth_zoo.saving_utils import ( merge_and_overwrite_lora, prepare_saving,