0f00bc1e2a* Fix Studio Python, Gemma 4 Unified sidecar, and worker crash messages * Clean up Gemma 4 sidecar test patch contexts * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Polish inference worker crash message * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Address transformers tier review feedback * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Route Gemma 4 assistant models to transformers 5.10 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
535 lines
21 KiB
Python
535 lines
21 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-only
|
|
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
|
|
|
"""Tests for transformers version detection with local checkpoint fallbacks."""
|
|
|
|
import json
|
|
import pytest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# The studio backend uses relative-style imports (``from utils.…``), so
|
|
# add the backend directory to *sys.path* if not already present.
|
|
# ---------------------------------------------------------------------------
|
|
import sys
|
|
|
|
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
|
|
if _BACKEND_DIR not in sys.path:
|
|
sys.path.insert(0, _BACKEND_DIR)
|
|
|
|
# Stub the custom logger before import so ``from loggers import
|
|
# get_logger`` doesn't fail.
|
|
import types as _types
|
|
|
|
_loggers_stub = _types.ModuleType("loggers")
|
|
_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
|
|
sys.modules.setdefault("loggers", _loggers_stub)
|
|
|
|
from utils.transformers_version import (
|
|
_resolve_base_model,
|
|
_check_tokenizer_config_needs_v5,
|
|
_check_config_needs_510,
|
|
_check_config_needs_550,
|
|
_config_json_cache,
|
|
_tokenizer_class_cache,
|
|
_config_needs_510_cache,
|
|
_config_needs_550_cache,
|
|
needs_transformers_5,
|
|
get_transformers_tier,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _resolve_base_model — config.json fallback
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestResolveBaseModel:
|
|
"""Tests for _resolve_base_model() local config fallbacks."""
|
|
|
|
def test_adapter_config_takes_priority(self, tmp_path: Path):
|
|
"""adapter_config.json should be preferred over config.json."""
|
|
adapter_cfg = {"base_model_name_or_path": "meta-llama/Llama-3-8B"}
|
|
config_cfg = {"_name_or_path": "different/model"}
|
|
(tmp_path / "adapter_config.json").write_text(json.dumps(adapter_cfg))
|
|
(tmp_path / "config.json").write_text(json.dumps(config_cfg))
|
|
|
|
result = _resolve_base_model(str(tmp_path))
|
|
assert result == "meta-llama/Llama-3-8B"
|
|
|
|
def test_config_json_fallback_model_name(self, tmp_path: Path):
|
|
"""config.json model_name should resolve when no adapter_config."""
|
|
config_cfg = {"model_name": "Qwen/Qwen3.5-9B"}
|
|
(tmp_path / "config.json").write_text(json.dumps(config_cfg))
|
|
|
|
result = _resolve_base_model(str(tmp_path))
|
|
assert result == "Qwen/Qwen3.5-9B"
|
|
|
|
def test_config_json_fallback_name_or_path(self, tmp_path: Path):
|
|
"""config.json _name_or_path should resolve as secondary fallback."""
|
|
config_cfg = {"_name_or_path": "Qwen/Qwen3.5-9B"}
|
|
(tmp_path / "config.json").write_text(json.dumps(config_cfg))
|
|
|
|
result = _resolve_base_model(str(tmp_path))
|
|
assert result == "Qwen/Qwen3.5-9B"
|
|
|
|
def test_model_name_takes_priority_over_name_or_path(self, tmp_path: Path):
|
|
"""model_name should be preferred over _name_or_path."""
|
|
config_cfg = {
|
|
"model_name": "Qwen/Qwen3.5-9B",
|
|
"_name_or_path": "some/other-model",
|
|
}
|
|
(tmp_path / "config.json").write_text(json.dumps(config_cfg))
|
|
|
|
result = _resolve_base_model(str(tmp_path))
|
|
assert result == "Qwen/Qwen3.5-9B"
|
|
|
|
def test_config_json_skips_self_referencing(self, tmp_path: Path):
|
|
"""config.json should be ignored if model_name == the checkpoint path."""
|
|
config_cfg = {"model_name": str(tmp_path)}
|
|
(tmp_path / "config.json").write_text(json.dumps(config_cfg))
|
|
|
|
result = _resolve_base_model(str(tmp_path))
|
|
# Falls through; does not return the self-referencing path.
|
|
assert result == str(tmp_path)
|
|
|
|
def test_no_config_files(self, tmp_path: Path):
|
|
"""Returns original name when no config files are present."""
|
|
result = _resolve_base_model(str(tmp_path))
|
|
assert result == str(tmp_path)
|
|
|
|
def test_plain_hf_id_passthrough(self):
|
|
"""Plain HuggingFace model IDs pass through unchanged."""
|
|
result = _resolve_base_model("meta-llama/Llama-3-8B")
|
|
assert result == "meta-llama/Llama-3-8B"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _check_tokenizer_config_needs_v5 — local file check
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCheckTokenizerConfigNeedsV5:
|
|
"""Tests for local tokenizer_config.json fallback."""
|
|
|
|
def setup_method(self):
|
|
_tokenizer_class_cache.clear()
|
|
|
|
def test_local_tokenizer_config_v5(self, tmp_path: Path):
|
|
"""Local tokenizer_config.json with v5 tokenizer should return True."""
|
|
tc = {"tokenizer_class": "TokenizersBackend"}
|
|
(tmp_path / "tokenizer_config.json").write_text(json.dumps(tc))
|
|
|
|
result = _check_tokenizer_config_needs_v5(str(tmp_path))
|
|
assert result is True
|
|
|
|
def test_local_tokenizer_config_v4(self, tmp_path: Path):
|
|
"""Local tokenizer_config.json with standard tokenizer should return False."""
|
|
tc = {"tokenizer_class": "LlamaTokenizerFast"}
|
|
(tmp_path / "tokenizer_config.json").write_text(json.dumps(tc))
|
|
|
|
result = _check_tokenizer_config_needs_v5(str(tmp_path))
|
|
assert result is False
|
|
|
|
def test_local_file_skips_network(self, tmp_path: Path):
|
|
"""When local file exists, no network request should be made."""
|
|
tc = {"tokenizer_class": "LlamaTokenizerFast"}
|
|
(tmp_path / "tokenizer_config.json").write_text(json.dumps(tc))
|
|
|
|
with patch("urllib.request.urlopen") as mock_urlopen:
|
|
result = _check_tokenizer_config_needs_v5(str(tmp_path))
|
|
mock_urlopen.assert_not_called()
|
|
assert result is False
|
|
|
|
def test_result_is_cached(self, tmp_path: Path):
|
|
"""Subsequent calls should use the cache."""
|
|
tc = {"tokenizer_class": "TokenizersBackend"}
|
|
(tmp_path / "tokenizer_config.json").write_text(json.dumps(tc))
|
|
|
|
key = str(tmp_path)
|
|
_check_tokenizer_config_needs_v5(key)
|
|
assert key in _tokenizer_class_cache
|
|
assert _tokenizer_class_cache[key] is True
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# needs_transformers_5 — integration-level
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestNeedsTransformers5:
|
|
"""Integration tests for the top-level needs_transformers_5() function."""
|
|
|
|
def setup_method(self):
|
|
_tokenizer_class_cache.clear()
|
|
|
|
def test_qwen35_substring(self):
|
|
assert needs_transformers_5("Qwen/Qwen3.5-9B") is True
|
|
|
|
def test_qwen3_30b_a3b_substring(self):
|
|
assert needs_transformers_5("Qwen/Qwen3-30B-A3B-Instruct-2507") is True
|
|
|
|
def test_ministral_substring(self):
|
|
assert needs_transformers_5("mistralai/Ministral-3-8B-Instruct-2512") is True
|
|
|
|
def test_llama_does_not_need_v5(self):
|
|
"""Standard models should not trigger v5."""
|
|
# Patch network call to avoid a real fetch.
|
|
with patch(
|
|
"utils.transformers_version._check_tokenizer_config_needs_v5",
|
|
return_value = False,
|
|
):
|
|
assert needs_transformers_5("meta-llama/Llama-3-8B") is False
|
|
|
|
def test_local_checkpoint_resolved_via_config(self, tmp_path: Path):
|
|
"""Local checkpoint with config.json pointing to Qwen3.5 needs v5."""
|
|
config_cfg = {"model_name": "Qwen/Qwen3.5-9B"}
|
|
(tmp_path / "config.json").write_text(json.dumps(config_cfg))
|
|
|
|
# needs_transformers_5 only does substring matching, so test the
|
|
# full resolution chain via _resolve_base_model here.
|
|
resolved = _resolve_base_model(str(tmp_path))
|
|
assert needs_transformers_5(resolved) is True
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _check_config_needs_550 — config.json architecture/model_type check
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCheckConfigNeeds550:
|
|
"""Tests for _check_config_needs_550() local config.json checks."""
|
|
|
|
def setup_method(self):
|
|
_config_json_cache.clear()
|
|
_config_needs_550_cache.clear()
|
|
|
|
def test_gemma4_architecture(self, tmp_path: Path):
|
|
"""config.json with Gemma4ForConditionalGeneration should return True."""
|
|
cfg = {
|
|
"architectures": ["Gemma4ForConditionalGeneration"],
|
|
"model_type": "gemma4",
|
|
}
|
|
(tmp_path / "config.json").write_text(json.dumps(cfg))
|
|
|
|
assert _check_config_needs_550(str(tmp_path)) is True
|
|
|
|
def test_gemma4_model_type_only(self, tmp_path: Path):
|
|
"""config.json with model_type=gemma4 (no architectures) should return True."""
|
|
cfg = {"model_type": "gemma4"}
|
|
(tmp_path / "config.json").write_text(json.dumps(cfg))
|
|
|
|
assert _check_config_needs_550(str(tmp_path)) is True
|
|
|
|
def test_llama_architecture(self, tmp_path: Path):
|
|
"""config.json with LlamaForCausalLM should return False."""
|
|
cfg = {"architectures": ["LlamaForCausalLM"], "model_type": "llama"}
|
|
(tmp_path / "config.json").write_text(json.dumps(cfg))
|
|
|
|
assert _check_config_needs_550(str(tmp_path)) is False
|
|
|
|
def test_no_config_json(self, tmp_path: Path):
|
|
"""Missing config.json should return False (fail-open)."""
|
|
# Patch network call to avoid a real fetch.
|
|
with patch("urllib.request.urlopen") as mock_urlopen:
|
|
mock_urlopen.side_effect = Exception("no network")
|
|
assert _check_config_needs_550(str(tmp_path)) is False
|
|
|
|
def test_result_is_cached(self, tmp_path: Path):
|
|
"""Subsequent calls should use the cache."""
|
|
cfg = {"architectures": ["Gemma4ForConditionalGeneration"]}
|
|
(tmp_path / "config.json").write_text(json.dumps(cfg))
|
|
|
|
key = str(tmp_path)
|
|
_check_config_needs_550(key)
|
|
assert key in _config_needs_550_cache
|
|
assert _config_needs_550_cache[key] is True
|
|
|
|
def test_local_file_skips_network(self, tmp_path: Path):
|
|
"""When local config.json exists, no network request should be made."""
|
|
cfg = {"architectures": ["LlamaForCausalLM"]}
|
|
(tmp_path / "config.json").write_text(json.dumps(cfg))
|
|
|
|
with patch("urllib.request.urlopen") as mock_urlopen:
|
|
_check_config_needs_550(str(tmp_path))
|
|
mock_urlopen.assert_not_called()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _check_config_needs_510 — config.json architecture/model_type check
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCheckConfigNeeds510:
|
|
"""Tests for _check_config_needs_510() local config.json checks."""
|
|
|
|
def setup_method(self):
|
|
_config_json_cache.clear()
|
|
_config_needs_510_cache.clear()
|
|
|
|
def test_gemma4_unified_architecture(self, tmp_path: Path):
|
|
"""config.json with Gemma4UnifiedForConditionalGeneration should return True."""
|
|
cfg = {
|
|
"architectures": ["Gemma4UnifiedForConditionalGeneration"],
|
|
"model_type": "gemma4_unified",
|
|
}
|
|
(tmp_path / "config.json").write_text(json.dumps(cfg))
|
|
|
|
assert _check_config_needs_510(str(tmp_path)) is True
|
|
|
|
def test_gemma4_unified_model_type_only(self, tmp_path: Path):
|
|
"""config.json with model_type=gemma4_unified should return True."""
|
|
cfg = {"model_type": "gemma4_unified"}
|
|
(tmp_path / "config.json").write_text(json.dumps(cfg))
|
|
|
|
assert _check_config_needs_510(str(tmp_path)) is True
|
|
|
|
def test_gemma4_unified_assistant_architecture(self, tmp_path: Path):
|
|
"""Assistant Gemma 4 Unified configs should return True."""
|
|
cfg = {
|
|
"architectures": ["Gemma4UnifiedAssistantForCausalLM"],
|
|
"model_type": "gemma4_unified_assistant",
|
|
}
|
|
(tmp_path / "config.json").write_text(json.dumps(cfg))
|
|
|
|
assert _check_config_needs_510(str(tmp_path)) is True
|
|
|
|
def test_gemma4_unified_assistant_model_type_only(self, tmp_path: Path):
|
|
"""Assistant Gemma 4 Unified model_type should return True."""
|
|
cfg = {"model_type": "gemma4_unified_assistant"}
|
|
(tmp_path / "config.json").write_text(json.dumps(cfg))
|
|
|
|
assert _check_config_needs_510(str(tmp_path)) is True
|
|
|
|
def test_gemma4_assistant_architecture(self, tmp_path: Path):
|
|
"""Assistant Gemma 4 configs should return True."""
|
|
cfg = {
|
|
"architectures": ["Gemma4AssistantForCausalLM"],
|
|
"model_type": "gemma4_assistant",
|
|
}
|
|
(tmp_path / "config.json").write_text(json.dumps(cfg))
|
|
|
|
assert _check_config_needs_510(str(tmp_path)) is True
|
|
|
|
def test_gemma4_assistant_model_type_only(self, tmp_path: Path):
|
|
"""Assistant Gemma 4 model_type should return True."""
|
|
cfg = {"model_type": "gemma4_assistant"}
|
|
(tmp_path / "config.json").write_text(json.dumps(cfg))
|
|
|
|
assert _check_config_needs_510(str(tmp_path)) is True
|
|
|
|
def test_gemma4_non_unified_returns_false(self, tmp_path: Path):
|
|
"""Older Gemma 4 config should stay on the 550 tier."""
|
|
cfg = {
|
|
"architectures": ["Gemma4ForConditionalGeneration"],
|
|
"model_type": "gemma4",
|
|
}
|
|
(tmp_path / "config.json").write_text(json.dumps(cfg))
|
|
|
|
assert _check_config_needs_510(str(tmp_path)) is False
|
|
|
|
def test_no_config_json(self, tmp_path: Path):
|
|
"""Missing config.json should return False (fail-open)."""
|
|
# Patch network call to avoid real fetch
|
|
with patch("urllib.request.urlopen") as mock_urlopen:
|
|
mock_urlopen.side_effect = Exception("no network")
|
|
assert _check_config_needs_510(str(tmp_path)) is False
|
|
|
|
def test_result_is_cached(self, tmp_path: Path):
|
|
"""Subsequent calls should use the cache."""
|
|
cfg = {"architectures": ["Gemma4UnifiedForConditionalGeneration"]}
|
|
(tmp_path / "config.json").write_text(json.dumps(cfg))
|
|
|
|
key = str(tmp_path)
|
|
_check_config_needs_510(key)
|
|
assert key in _config_needs_510_cache
|
|
assert _config_needs_510_cache[key] is True
|
|
|
|
def test_local_file_skips_network(self, tmp_path: Path):
|
|
"""When local config.json exists, no network request should be made."""
|
|
cfg = {"architectures": ["LlamaForCausalLM"]}
|
|
(tmp_path / "config.json").write_text(json.dumps(cfg))
|
|
|
|
with patch("urllib.request.urlopen") as mock_urlopen:
|
|
_check_config_needs_510(str(tmp_path))
|
|
mock_urlopen.assert_not_called()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# get_transformers_tier — tier detection
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestGetTransformersTier:
|
|
"""Tests for get_transformers_tier() tiered version detection."""
|
|
|
|
def setup_method(self):
|
|
_tokenizer_class_cache.clear()
|
|
_config_json_cache.clear()
|
|
_config_needs_510_cache.clear()
|
|
_config_needs_550_cache.clear()
|
|
|
|
def test_gemma4_substring_returns_550(self):
|
|
assert get_transformers_tier("google/gemma-4-E2B-it") == "550"
|
|
|
|
def test_gemma4_12b_substring_returns_510(self):
|
|
assert get_transformers_tier("unsloth/gemma-4-12b-it") == "510"
|
|
|
|
def test_gemma4_assistant_substring_returns_510(self):
|
|
assert get_transformers_tier("google/gemma-4-E2B-it-assistant") == "510"
|
|
|
|
def test_gemma4_alt_substring_returns_550(self):
|
|
assert get_transformers_tier("unsloth/gemma4-E4B-it") == "550"
|
|
|
|
def test_gemma4_config_json_returns_550(self, tmp_path: Path):
|
|
"""Local checkpoint with Gemma4 architecture → 550."""
|
|
cfg = {
|
|
"architectures": ["Gemma4ForConditionalGeneration"],
|
|
"model_type": "gemma4",
|
|
}
|
|
(tmp_path / "config.json").write_text(json.dumps(cfg))
|
|
|
|
assert get_transformers_tier(str(tmp_path)) == "550"
|
|
|
|
def test_gemma4_unified_config_json_returns_510(self, tmp_path: Path):
|
|
"""Local checkpoint with Gemma4 Unified architecture → 510."""
|
|
cfg = {
|
|
"architectures": ["Gemma4UnifiedForConditionalGeneration"],
|
|
"model_type": "gemma4_unified",
|
|
}
|
|
(tmp_path / "config.json").write_text(json.dumps(cfg))
|
|
|
|
assert get_transformers_tier(str(tmp_path)) == "510"
|
|
|
|
def test_gemma4_assistant_config_json_returns_510(self, tmp_path: Path):
|
|
"""Local checkpoint with Gemma4 Assistant architecture → 510."""
|
|
cfg = {
|
|
"architectures": ["Gemma4AssistantForCausalLM"],
|
|
"model_type": "gemma4_assistant",
|
|
}
|
|
(tmp_path / "config.json").write_text(json.dumps(cfg))
|
|
|
|
assert get_transformers_tier(str(tmp_path)) == "510"
|
|
|
|
def test_local_config_json_short_circuits_path_substrings(self, tmp_path: Path):
|
|
"""Local config.json should prevent false matches from parent directory names."""
|
|
model_dir = tmp_path / "gemma-4-12b-experiment" / "llama-checkpoint"
|
|
model_dir.mkdir(parents = True)
|
|
(model_dir / "config.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"architectures": ["LlamaForCausalLM"],
|
|
"model_type": "llama",
|
|
}
|
|
)
|
|
)
|
|
(model_dir / "tokenizer_config.json").write_text(
|
|
json.dumps({"tokenizer_class": "LlamaTokenizerFast"})
|
|
)
|
|
|
|
with patch("urllib.request.urlopen") as mock_urlopen:
|
|
assert get_transformers_tier(str(model_dir)) == "default"
|
|
mock_urlopen.assert_not_called()
|
|
|
|
def test_remote_config_json_is_fetched_once_for_config_tiers(self):
|
|
"""510 and 550 slow-path checks should share one config.json fetch."""
|
|
|
|
class _Response:
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc, tb):
|
|
return False
|
|
|
|
def read(self):
|
|
return json.dumps(
|
|
{
|
|
"architectures": ["Gemma4ForConditionalGeneration"],
|
|
"model_type": "gemma4",
|
|
}
|
|
).encode()
|
|
|
|
with patch("urllib.request.urlopen", return_value = _Response()) as mock_urlopen:
|
|
assert get_transformers_tier("org/no-fast-substring-model") == "550"
|
|
|
|
assert mock_urlopen.call_count == 1
|
|
|
|
def test_qwen35_returns_530(self):
|
|
with (
|
|
patch(
|
|
"utils.transformers_version._check_config_needs_550",
|
|
return_value = False,
|
|
),
|
|
patch(
|
|
"utils.transformers_version._check_config_needs_510",
|
|
return_value = False,
|
|
),
|
|
):
|
|
assert get_transformers_tier("Qwen/Qwen3.5-9B") == "530"
|
|
|
|
def test_ministral_returns_530(self):
|
|
with (
|
|
patch(
|
|
"utils.transformers_version._check_config_needs_550",
|
|
return_value = False,
|
|
),
|
|
patch(
|
|
"utils.transformers_version._check_config_needs_510",
|
|
return_value = False,
|
|
),
|
|
):
|
|
assert get_transformers_tier("mistralai/Ministral-3-8B-Instruct-2512") == "530"
|
|
|
|
def test_llama_returns_default(self):
|
|
with (
|
|
patch(
|
|
"utils.transformers_version._check_config_needs_550",
|
|
return_value = False,
|
|
),
|
|
patch(
|
|
"utils.transformers_version._check_config_needs_510",
|
|
return_value = False,
|
|
),
|
|
patch(
|
|
"utils.transformers_version._check_tokenizer_config_needs_v5",
|
|
return_value = False,
|
|
),
|
|
):
|
|
assert get_transformers_tier("meta-llama/Llama-3-8B") == "default"
|
|
|
|
def test_550_checked_before_530(self):
|
|
"""5.5.0 is checked before 5.3.0 - a model matching both gets 550."""
|
|
assert get_transformers_tier("gemma-4-model") == "550"
|
|
|
|
def test_needs_transformers_5_compat(self):
|
|
"""needs_transformers_5 should return True for 510, 530, and 550 models."""
|
|
assert needs_transformers_5("unsloth/gemma-4-12b-it") is True
|
|
assert needs_transformers_5("google/gemma-4-E2B-it") is True
|
|
with (
|
|
patch(
|
|
"utils.transformers_version._check_config_needs_550",
|
|
return_value = False,
|
|
),
|
|
patch(
|
|
"utils.transformers_version._check_config_needs_510",
|
|
return_value = False,
|
|
),
|
|
):
|
|
assert needs_transformers_5("Qwen/Qwen3.5-9B") is True
|
|
with (
|
|
patch(
|
|
"utils.transformers_version._check_config_needs_550",
|
|
return_value = False,
|
|
),
|
|
patch(
|
|
"utils.transformers_version._check_config_needs_510",
|
|
return_value = False,
|
|
),
|
|
patch(
|
|
"utils.transformers_version._check_tokenizer_config_needs_v5",
|
|
return_value = False,
|
|
),
|
|
):
|
|
assert needs_transformers_5("meta-llama/Llama-3-8B") is False
|