diff --git a/tests/saving/test_offline_gguf_vlm_tokenizer_7481.py b/tests/saving/test_offline_gguf_vlm_tokenizer_7481.py new file mode 100644 index 0000000000..c22cbff470 --- /dev/null +++ b/tests/saving/test_offline_gguf_vlm_tokenizer_7481.py @@ -0,0 +1,119 @@ +"""Offline GGUF export must not probe the Hub for VLM tokenizer metadata (issue #7481). + +Regression for ``PreTrainedTokenizerFast.from_pretrained`` on a repo id calling +``is_base_mistral()`` -> ``model_info()`` even with ``TRANSFORMERS_OFFLINE=1``. +Pure CPU, no network, no GPU. +""" + +import json +import os +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from unsloth.models import loader_utils as L + + +_REPO = "llmfan46/gemma-4-E4B-it-ultra-uncensored-heretic" +_COMMIT = "5964fe4c7339c5974e879baba8982a09616f68ca" + + +def _write_gemma4_cache(root, repo_id=_REPO, commit=_COMMIT): + """Minimal cached snapshot matching the reporter's layout.""" + org, name = repo_id.split("/") + repo_root = root / f"models--{org}--{name}" + snap = repo_root / "snapshots" / commit + snap.mkdir(parents=True) + refs = repo_root / "refs" + refs.mkdir(parents=True, exist_ok=True) + (refs / "main").write_text(commit, encoding="utf-8") + (snap / "tokenizer_config.json").write_text( + json.dumps({"tokenizer_class": "GemmaTokenizer", "model_max_length": 8192}), + encoding="utf-8", + ) + (snap / "tokenizer.json").write_text( + json.dumps( + { + "version": "1.0", + "truncation": None, + "padding": None, + "added_tokens": [], + "normalizer": None, + "pre_tokenizer": None, + "post_processor": None, + "decoder": None, + "model": {"type": "BPE", "vocab": {"": 0}, "merges": []}, + } + ), + encoding="utf-8", + ) + (snap / "processor_config.json").write_text("{}", encoding="utf-8") + (snap / "config.json").write_text( + json.dumps({"model_type": "gemma4"}), + encoding="utf-8", + ) + return snap + + +def _offline_env(monkeypatch, cache_root): + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + monkeypatch.setenv("TRANSFORMERS_OFFLINE", "1") + monkeypatch.setenv("HF_HUB_CACHE", str(cache_root)) + + +def test_resolve_hub_repo_local_dir_from_cached_snapshot(tmp_path, monkeypatch): + snap = _write_gemma4_cache(tmp_path) + _offline_env(monkeypatch, tmp_path) + + got = L._resolve_hub_repo_local_dir(_REPO, local_files_only=True, cache_dir=str(tmp_path)) + assert got == str(snap) + + +def test_hub_repo_or_local_path_prefers_snapshot_over_repo_id(tmp_path, monkeypatch): + snap = _write_gemma4_cache(tmp_path) + _offline_env(monkeypatch, tmp_path) + + got = L._hub_repo_or_local_path(_REPO, local_files_only=True, cache_dir=str(tmp_path)) + assert got == str(snap) + assert got != _REPO + + +def test_load_pretrained_tokenizer_fast_passes_snapshot_not_repo_id(tmp_path, monkeypatch): + snap = _write_gemma4_cache(tmp_path) + _offline_env(monkeypatch, tmp_path) + + seen_paths = [] + + class _FakeFast: + @classmethod + def from_pretrained(cls, path, **kwargs): + seen_paths.append(path) + assert kwargs.get("local_files_only") is True + return SimpleNamespace(name_or_path=path) + + monkeypatch.setattr( + "transformers.PreTrainedTokenizerFast", + _FakeFast, + raising=False, + ) + + with patch("huggingface_hub.HfApi.model_info") as model_info: + model_info.side_effect = AssertionError("model_info must not run offline") + tok = L._load_pretrained_tokenizer_fast(_REPO, cache_dir=str(tmp_path)) + + assert seen_paths == [str(snap)] + assert tok.name_or_path == str(snap) + + +def test_has_tokenizer_model_offline_skips_model_info(tmp_path, monkeypatch): + from unsloth.save import _has_tokenizer_model + + _write_gemma4_cache(tmp_path) + _offline_env(monkeypatch, tmp_path) + + tok = SimpleNamespace(name_or_path=_REPO, tokenizer=None) + + with patch("huggingface_hub.HfApi.model_info") as model_info: + model_info.side_effect = AssertionError("model_info must not run offline") + assert _has_tokenizer_model(tok, token=None) is False diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 7661b0d714..3f7eca8fb3 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -1126,6 +1126,113 @@ def _has_local_processor_files(path): ) +def _resolve_hub_repo_local_dir( + repo_id, + *, + token = None, + cache_dir = None, + local_files_only = False, + filenames = ( + "tokenizer_config.json", + "config.json", + "tokenizer.json", + "preprocessor_config.json", + "processor_config.json", + ), +): + """Return a local snapshot directory for a Hub repo id when files are cached. + + ``PreTrainedTokenizerFast.from_pretrained`` on a repo id can call + ``is_base_mistral()`` -> ``model_info()`` even with ``local_files_only=True`` + (issue #7481). Loading from the resolved snapshot dir avoids that probe. + """ + if not isinstance(repo_id, str) or not repo_id: + return None + if os.path.isdir(repo_id): + return repo_id + if cache_dir is None: + cache_dir = os.environ.get("HF_HUB_CACHE") + from huggingface_hub import hf_hub_download + + for filename in filenames: + try: + path = hf_hub_download( + repo_id = repo_id, + filename = filename, + token = token, + cache_dir = cache_dir, + local_files_only = local_files_only, + ) + if path and os.path.isfile(path): + return os.path.dirname(path) + except Exception: + continue + return None + + +def _hub_repo_or_local_path( + repo_id, + *, + token = None, + cache_dir = None, + local_files_only = False, + filenames = None, +): + """Prefer a cached snapshot path over a Hub repo id when offline or ``local_files_only``.""" + lfo = bool(local_files_only) or _env_says_offline() + if not lfo and os.path.isdir(repo_id): + return repo_id + local_dir = _resolve_hub_repo_local_dir( + repo_id, + token = token, + cache_dir = cache_dir, + local_files_only = lfo, + filenames = filenames + or ( + "tokenizer_config.json", + "config.json", + "tokenizer.json", + "preprocessor_config.json", + "processor_config.json", + ), + ) + return local_dir if local_dir is not None else repo_id + + +def _load_pretrained_tokenizer_fast( + tokenizer_name, + *, + padding_side = "left", + token = None, + trust_remote_code = False, + cache_dir = None, + local_files_only = False, +): + """Load ``PreTrainedTokenizerFast`` without Hub metadata probes when cached/offline.""" + from transformers import PreTrainedTokenizerFast + + lfo = bool(local_files_only) or _env_says_offline() + load_path = _hub_repo_or_local_path( + tokenizer_name, + token = token, + cache_dir = cache_dir, + local_files_only = lfo, + filenames = ( + "tokenizer_config.json", + "tokenizer.json", + "tokenizer.model", + ), + ) + return PreTrainedTokenizerFast.from_pretrained( + load_path, + padding_side = padding_side, + token = token, + trust_remote_code = trust_remote_code, + cache_dir = cache_dir, + local_files_only = lfo, + ) + + def _resolve_checkpoint_tokenizer_name( old_model_name, kwargs, diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 729c191e83..374f847b0f 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -529,7 +529,9 @@ def unsloth_base_fast_generate(self, *args, **kwargs): # Offline helpers live in loader_utils.py (shared canonical source). from .loader_utils import ( _get_effective_local_files_only, + _hub_repo_or_local_path, _is_offline_related_error, + _load_pretrained_tokenizer_fast, _offline_aware_load, ) @@ -565,20 +567,27 @@ def _construct_vlm_processor_fallback( tell an offline failure (retry from cache) from a genuine one.""" _fb_err = None try: - from transformers import AutoImageProcessor, PreTrainedTokenizerFast, AutoConfig + from transformers import AutoImageProcessor, AutoConfig from transformers.models.auto.processing_auto import PROCESSOR_MAPPING_NAMES import json + load_path = _hub_repo_or_local_path( + tokenizer_name, + token = token, + cache_dir = cache_dir, + local_files_only = local_files_only, + ) # Load image processor image_processor = AutoImageProcessor.from_pretrained( - tokenizer_name, + load_path, token = token, trust_remote_code = trust_remote_code, cache_dir = cache_dir, local_files_only = local_files_only, ) - # Load tokenizer via PreTrainedTokenizerFast (bypasses tokenizer_class check) - tok = PreTrainedTokenizerFast.from_pretrained( + # Load tokenizer via PreTrainedTokenizerFast (bypasses tokenizer_class check). + # Resolve the cached snapshot first so transformers does not call model_info (#7481). + tok = _load_pretrained_tokenizer_fast( tokenizer_name, padding_side = "left", token = token, @@ -638,7 +647,7 @@ def _construct_vlm_processor_fallback( # Try the top-level config.model_type which often has the processor mapping. try: config = AutoConfig.from_pretrained( - tokenizer_name, + load_path, token = token, trust_remote_code = trust_remote_code, cache_dir = cache_dir, @@ -1551,9 +1560,16 @@ class FastBaseModel: # Last resort: AutoTokenizer, then PreTrainedTokenizerFast (raise on network failure to retry). def _last_resort_tokenizer(lfo): from transformers import AutoTokenizer as _AutoTokenizer + + load_path = _hub_repo_or_local_path( + tokenizer_name, + token = token, + cache_dir = kwargs.get("cache_dir"), + local_files_only = lfo, + ) try: return _AutoTokenizer.from_pretrained( - tokenizer_name, + load_path, padding_side = "left", token = token, trust_remote_code = trust_remote_code, @@ -1561,8 +1577,7 @@ class FastBaseModel: local_files_only = lfo, ) except Exception: - from transformers import PreTrainedTokenizerFast - return PreTrainedTokenizerFast.from_pretrained( + return _load_pretrained_tokenizer_fast( tokenizer_name, padding_side = "left", token = token, diff --git a/unsloth/save.py b/unsloth/save.py index 0e2650b174..c98da2a630 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -51,7 +51,7 @@ import subprocess import psutil import re from transformers.models.llama.modeling_llama import logger -from .models.loader_utils import get_model_name +from .models.loader_utils import get_model_name, _env_says_offline, _resolve_hub_repo_local_dir from .models._utils import _convert_torchao_model from .ollama_template_mappers import OLLAMA_TEMPLATES, MODEL_TO_OLLAMA_TEMPLATE_MAPPER from transformers import ProcessorMixin, PreTrainedTokenizerBase @@ -445,6 +445,21 @@ def _has_tokenizer_model(tokenizer, token = None): if source in _TOKENIZER_MODEL_CACHE: return _TOKENIZER_MODEL_CACHE[source] + # Offline: probe the local cache instead of model_info (issue #7481). + if _env_says_offline(): + local_dir = _resolve_hub_repo_local_dir( + source, + token = token, + local_files_only = True, + filenames = ("tokenizer.model", "tokenizer.json", "tokenizer_config.json"), + cache_dir = os.environ.get("HF_HUB_CACHE"), + ) + if local_dir is not None: + has_tokenizer_model = os.path.isfile(os.path.join(local_dir, "tokenizer.model")) + _TOKENIZER_MODEL_CACHE[source] = has_tokenizer_model + return has_tokenizer_model + return False + try: repo_info = HfApi(token = token).model_info(source, files_metadata = False) except Exception: @@ -3773,7 +3788,7 @@ def unsloth_convert_lora_to_ggml_and_save_locally( return _unsloth_save_lora_gguf(self, tokenizer, save_directory, outtype = outtype) -from .models.loader_utils import get_model_name +from .models.loader_utils import get_model_name, _env_says_offline, _resolve_hub_repo_local_dir from unsloth_zoo.saving_utils import ( merge_and_overwrite_lora, prepare_saving,