diff --git a/.github/workflows/version-compat-ci.yml b/.github/workflows/version-compat-ci.yml index abbebd0067..fd2fb75152 100644 --- a/.github/workflows/version-compat-ci.yml +++ b/.github/workflows/version-compat-ci.yml @@ -230,8 +230,16 @@ jobs: PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python run: | cd unsloth + # tests/vllm_compat/test_unsloth_zoo_imports.py: narrow vllm/grpo + # import gates (5 tests). + # tests/vllm_compat/test_extended_module_imports.py: full sweep + # of unsloth_zoo + unsloth.models.* modules + RL dispatch + # table population + FastModel API surface under spoof + # (~30 tests). Catches transformers / peft / bnb symbol pin + # drift at module-top BEFORE any runtime call. PYTHONPATH=. python -m pytest \ tests/vllm_compat/test_unsloth_zoo_imports.py \ + tests/vllm_compat/test_extended_module_imports.py \ -v --tb=short # Daily-only: same suites but with --strict on importable upstream diff --git a/tests/version_compat/test_bitsandbytes_pinned_symbols.py b/tests/version_compat/test_bitsandbytes_pinned_symbols.py index 6e1892767b..86b6719ffe 100644 --- a/tests/version_compat/test_bitsandbytes_pinned_symbols.py +++ b/tests/version_compat/test_bitsandbytes_pinned_symbols.py @@ -19,9 +19,11 @@ Strategy: GitHub raw fetch + symbol grep. CPU-only, no install. from __future__ import annotations +import re + import pytest -from tests.version_compat._fetch import fetch_text, has_def, first_match +from tests.version_compat._fetch import fetch_text, first_match, has_def # pyproject pin: bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0 @@ -88,3 +90,220 @@ def test_bnb_nn_linear4bit_classes(tag: str): f"{tag}: Linear4bit={found_linear} Params4bit={found_params} " f"in {candidates}; unsloth + peft 4-bit isinstance checks fail" ) + + +# ========================================================================= +# Coverage extension (added 2026-05): every bnb symbol unsloth + +# unsloth-zoo touch, derived from a full grep of both repos. +# ========================================================================= + + +# ------------------------------------------------------------------------- +# Top-level convenience export. unsloth/kernels/utils.py + unsloth-zoo +# vllm_utils.py call `bnb.matmul_4bit(x, w, bias=, quant_state=)`. +# ------------------------------------------------------------------------- + + +@pytest.mark.parametrize("tag", BNB_TAGS) +def test_bnb_matmul_4bit_top_level(tag: str): + src = fetch_text( + "bitsandbytes-foundation/bitsandbytes", tag, "bitsandbytes/__init__.py" + ) + if src is None: + pytest.skip(f"{tag}: bitsandbytes/__init__.py missing") + assert "matmul_4bit" in src, ( + f"{tag}: bitsandbytes.matmul_4bit not exported at package root; " + f"unsloth/kernels/utils.py + zoo/temporary_patches/moe call paths break" + ) + + +@pytest.mark.parametrize("tag", BNB_TAGS) +def test_bnb_functional_4bit_kernel_path(tag: str): + """unsloth/kernels/utils.py module-top binds the 4-bit dequantize + and gemm primitives via one of two paths: + - LEGACY (bnb <= 0.48.x): `bnb.functional.lib.cdequantize_blockwise_*` + and `bnb.functional.lib.cgemm_4bit_inference_naive_*` — C + symbols listed in functional.py source. + - NEW (bnb >= 0.49.0): `torch.ops.bitsandbytes.dequantize_blockwise` + and `torch.ops.bitsandbytes.dequantize_4bit` Python wrappers; + the C symbols still live in libbitsandbytes_*.so but the + Python source no longer references them by name. + Either path lets unsloth resolve the kernels at runtime — we only + fail if NEITHER signal is present.""" + candidates = [ + "bitsandbytes/functional.py", + "bitsandbytes/functional/__init__.py", + ] + hit = first_match("bitsandbytes-foundation/bitsandbytes", tag, candidates) + if hit is None: + pytest.skip(f"{tag}: bitsandbytes/functional missing") + _, src = hit + legacy_path = "cdequantize_blockwise" in src and "cgemm_4bit_inference" in src + new_path = ( + "dequantize_blockwise" in src + and ("dequantize_4bit" in src or "dequantize_nf4" in src) + and "torch.ops.bitsandbytes" in src + ) + assert legacy_path or new_path, ( + f"{tag}: bnb.functional has NEITHER legacy `lib.cdequantize_*` " + f"NOR new `torch.ops.bitsandbytes.*` kernel path; " + f"unsloth/kernels/utils.py module-top binding will AttributeError" + ) + + +@pytest.mark.parametrize("tag", BNB_TAGS) +def test_bnb_functional_get_ptr(tag: str): + """unsloth/kernels/utils.py top-level: `get_ptr = bnb.functional.get_ptr`.""" + candidates = [ + "bitsandbytes/functional.py", + "bitsandbytes/functional/__init__.py", + ] + hit = first_match("bitsandbytes-foundation/bitsandbytes", tag, candidates) + if hit is None: + pytest.skip(f"{tag}: functional missing") + _, src = hit + assert has_def(src, "get_ptr", "func") or "get_ptr" in src, ( + f"{tag}: bnb.functional.get_ptr missing; " + f"unsloth/kernels/utils.py module-top ImportError" + ) + + +@pytest.mark.parametrize("tag", BNB_TAGS) +def test_bnb_quantstate_from_dict(tag: str): + """unsloth-zoo monkey-patches `QuantState.from_dict = ...`. Both + the class AND the classmethod must be present for the rebinding + to take effect.""" + candidates = [ + "bitsandbytes/functional.py", + "bitsandbytes/functional/__init__.py", + ] + hit = first_match("bitsandbytes-foundation/bitsandbytes", tag, candidates) + if hit is None: + pytest.skip(f"{tag}: functional missing") + _, src = hit + assert has_def(src, "QuantState", "class"), ( + f"{tag}: bnb.functional.QuantState missing" + ) + assert "from_dict" in src, ( + f"{tag}: QuantState.from_dict missing; " + f"unsloth-zoo monkey-patch silently no-ops" + ) + + +@pytest.mark.parametrize("tag", BNB_TAGS) +def test_bnb_nn_modules_fix_4bit_weight_optional(tag: str): + """fix_4bit_weight_quant_state_from_module added in newer bnb; + unsloth uses getattr() with a fallback so older versions are OK.""" + src = fetch_text( + "bitsandbytes-foundation/bitsandbytes", tag, "bitsandbytes/nn/modules.py" + ) + if src is None: + pytest.skip(f"{tag}: bitsandbytes/nn/modules.py missing") + if "fix_4bit_weight_quant_state_from_module" not in src: + pytest.skip(f"{tag}: helper not yet added (OK; getattr fallback)") + + +@pytest.mark.parametrize("tag", BNB_TAGS) +def test_bnb_nn_linear8bitlt(tag: str): + """unsloth/__init__ probes both Linear4bit AND Linear8bitLt.""" + candidates = [ + "bitsandbytes/nn/modules.py", + "bitsandbytes/nn/__init__.py", + ] + for p in candidates: + src = fetch_text("bitsandbytes-foundation/bitsandbytes", tag, p) + if src and ( + has_def(src, "Linear8bitLt", "class") or "Linear8bitLt" in src + ): + return + pytest.fail( + f"{tag}: bnb.nn.Linear8bitLt missing in {candidates}; " + f"legacy load_in_8bit path breaks" + ) + + +@pytest.mark.parametrize("tag", BNB_TAGS) +def test_bnb_optim_optimizer2state(tag: str): + """PagedAdamW32bit + 8bit optimisers subclass Optimizer2State.""" + src = fetch_text( + "bitsandbytes-foundation/bitsandbytes", + tag, + "bitsandbytes/optim/optimizer.py", + ) + if src is None: + pytest.skip(f"{tag}: bitsandbytes/optim/optimizer.py missing") + assert has_def(src, "Optimizer2State", "class"), ( + f"{tag}: bnb.optim.optimizer.Optimizer2State missing" + ) + + +@pytest.mark.parametrize("tag", BNB_TAGS) +def test_bnb_utils_pack_unpack(tag: str): + """4bit state-dict save/load uses these two helpers.""" + src = fetch_text( + "bitsandbytes-foundation/bitsandbytes", tag, "bitsandbytes/utils.py" + ) + if src is None: + pytest.skip(f"{tag}: bitsandbytes/utils.py missing") + for name in ("pack_dict_to_tensor", "unpack_tensor_to_dict"): + assert has_def(src, name, "func") or name in src, ( + f"{tag}: bnb.utils.{name} missing" + ) + + +@pytest.mark.parametrize("tag", BNB_TAGS) +def test_bnb_cextension_rocm_warp_size_optional(tag: str): + """ROCM_WARP_SIZE_64 added with AMD ROCm support; pre-ROCm bnb + builds don't have it. unsloth probes via try/except — informational.""" + src = fetch_text( + "bitsandbytes-foundation/bitsandbytes", tag, "bitsandbytes/cextension.py" + ) + if src is None: + pytest.skip(f"{tag}: cextension.py missing") + if "ROCM_WARP_SIZE_64" not in src: + pytest.skip(f"{tag}: ROCM_WARP_SIZE_64 not yet defined (pre-ROCm bnb)") + + +@pytest.mark.parametrize("tag", BNB_TAGS) +def test_bnb_autograd_functions_matmul_4bit(tag: str): + """unsloth-zoo has a dynamo-disable patch site for + bnb.autograd._functions.matmul_4bit. Symbol must remain so the + probe + decision logic works.""" + src = fetch_text( + "bitsandbytes-foundation/bitsandbytes", + tag, + "bitsandbytes/autograd/_functions.py", + ) + if src is None: + pytest.skip(f"{tag}: bitsandbytes/autograd/_functions.py missing") + assert "matmul_4bit" in src, ( + f"{tag}: bnb.autograd._functions.matmul_4bit missing" + ) + + +@pytest.mark.parametrize("tag", BNB_TAGS) +def test_bnb_version_parseable(tag: str): + """Multiple unsloth code paths read Version(bnb.__version__) for + feature gating (floors 0.43.3, 0.46.0, 0.48.2.dev0, 0.49.0, + 0.49.2). At least one export mechanism must work.""" + src = fetch_text( + "bitsandbytes-foundation/bitsandbytes", tag, "bitsandbytes/__init__.py" + ) + if src is None: + pytest.skip(f"{tag}: bitsandbytes/__init__.py missing") + has_literal = bool(re.search(r'^__version__\s*=\s*["\']', src, re.MULTILINE)) + has_subimport = bool( + re.search(r"^from\s+\.version\s+import\s+__version__", src, re.MULTILINE) + ) + has_metadata = bool( + re.search( + r"^from\s+importlib\.metadata\s+import\s+(?:[\w,\s]+,\s*)?version", + src, + re.MULTILINE, + ) + and re.search(r"^\s*__version__\s*=\s*version\s*\(", src, re.MULTILINE) + ) + has_version_attr = "__version__" in src + assert ( + has_literal or has_subimport or has_metadata or has_version_attr + ), f"{tag}: bnb.__version__ not exported" diff --git a/tests/vllm_compat/test_extended_module_imports.py b/tests/vllm_compat/test_extended_module_imports.py new file mode 100644 index 0000000000..85c0ab3ae1 --- /dev/null +++ b/tests/vllm_compat/test_extended_module_imports.py @@ -0,0 +1,331 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. +"""Extended import-smoke + API surface checks for unsloth + unsloth-zoo +modules under the existing CUDA spoof harness. + +Where `tests/vllm_compat/test_unsloth_zoo_imports.py` covers the +narrow "must import on a vllm-less runner" claim for 5 modules, +this file walks the FULL set of modules our public surface depends +on. Catches: + + - module-level imports that break on a fresh transformers / peft / + bnb release (the symbol pinned at import time is gone) + - feature flags / gates that flip under the spoof (e.g. _IS_MLX + silently activating on a non-Mac CI box) + - public API surface drift: sorted `dir()` of each FastModel class + is dumped and asserted-stable across runs (a removed kwarg here + is a notebook regression we want to catch) + +CPU-only. Inherits the same _zoo_aggressive_cuda_spoof harness as +test_unsloth_zoo_imports.py. +""" + +from __future__ import annotations + +import importlib +import importlib.machinery +import importlib.util +import inspect +import os +import sys +import types +from pathlib import Path + +import pytest + + +# Apply the spoof BEFORE any unsloth-touching import. +_SPOOF_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(_SPOOF_DIR)) +import _zoo_aggressive_cuda_spoof as _spoof # noqa: E402 + +_spoof.apply() + + +# Stub modules the unsloth import path may probe but that aren't +# installed on a CPU-only runner. Mirrors test_unsloth_zoo_imports.py. +def _stub_module(name: str, attrs: dict | None = None) -> None: + """Stub a missing optional dep. Sets __spec__ so importlib.util's + `find_spec(name)` doesn't raise `ValueError: __spec__ is None`, + which torch / transformers / torchcodec callers hit otherwise.""" + if name in sys.modules: + return + m = types.ModuleType(name) + # Minimal viable spec so importlib treats the stub as a real module. + m.__spec__ = importlib.machinery.ModuleSpec( + name = name, loader = None, origin = "" + ) + for k, v in (attrs or {}).items(): + setattr(m, k, v) + sys.modules[name] = m + + +_stub_module( + "pynvml", + { + "nvmlInit": lambda: None, + "nvmlShutdown": lambda: None, + "nvmlDeviceGetCount": lambda: 1, + "nvmlDeviceGetHandleByIndex": lambda i: object(), + "nvmlDeviceGetMemoryInfo": lambda h: type( + "_M", + (), + {"total": 80 * 1024**3, "free": 70 * 1024**3, "used": 10 * 1024**3}, + )(), + }, +) +_stub_module("torchcodec") + + +@pytest.fixture(autouse = True) +def _torch_distributed_safe(monkeypatch): + """unsloth_zoo modules occasionally probe torch.distributed.""" + try: + import torch.distributed as dist + monkeypatch.setattr(dist, "is_available", lambda: True, raising = False) + monkeypatch.setattr(dist, "is_initialized", lambda: False, raising = False) + monkeypatch.setattr(dist, "get_world_size", lambda *a, **k: 1, raising = False) + monkeypatch.setattr(dist, "get_rank", lambda *a, **k: 0, raising = False) + except Exception: + pass + + +def _has_unsloth_zoo() -> bool: + return importlib.util.find_spec("unsloth_zoo") is not None + + +def _has_unsloth() -> bool: + return importlib.util.find_spec("unsloth") is not None + + +# ------------------------------------------------------------------------- +# Extended unsloth-zoo module list. Modules with no top-level vllm/CUDA +# import are expected to load cleanly on a CPU spoof runner. +# ------------------------------------------------------------------------- + + +_ZOO_VLLM_FREE_MODULES = [ + "unsloth_zoo.compiler", + "unsloth_zoo.compiler_replacements", + "unsloth_zoo.dataset_utils", + "unsloth_zoo.device_type", + "unsloth_zoo.empty_model", + "unsloth_zoo.gradient_checkpointing", + "unsloth_zoo.hf_utils", + "unsloth_zoo.llama_cpp", + "unsloth_zoo.logging_utils", + "unsloth_zoo.loss_utils", + "unsloth_zoo.patching_utils", + "unsloth_zoo.patch_torch_functions", + "unsloth_zoo.peft_utils", + "unsloth_zoo.rl_replacements", + "unsloth_zoo.saving_utils", + "unsloth_zoo.tiled_mlp", + "unsloth_zoo.tokenizer_utils", + "unsloth_zoo.training_utils", + "unsloth_zoo.utils", + "unsloth_zoo.vision_utils", +] + + +@pytest.mark.skipif(not _has_unsloth_zoo(), reason = "unsloth_zoo not installed") +@pytest.mark.parametrize("modname", _ZOO_VLLM_FREE_MODULES) +def test_unsloth_zoo_module_imports_under_spoof(modname: str): + """Each unsloth_zoo module must import cleanly on a CPU-only spoof + runner. Catches transformers/peft/bnb symbol drift that pins fail + at import time (vs runtime).""" + # Force fresh resolution: drops stale partial-import state from + # a previous module's failure. + sys.modules.pop(modname, None) + try: + importlib.import_module(modname) + except Exception as e: + pytest.fail( + f"{modname} failed to import under CUDA spoof: " + f"{type(e).__name__}: {str(e)[:300]}" + ) + + +# ------------------------------------------------------------------------- +# Spoof correctness: _IS_MLX must remain False on a non-Mac runner +# AND _IS_CUDA / DEVICE_TYPE must reflect the spoofed CUDA layer. +# ------------------------------------------------------------------------- + + +@pytest.mark.skipif(not _has_unsloth(), reason = "unsloth not installed") +def test_unsloth_is_mlx_false_under_spoof(): + """The CUDA spoof should not flip the MLX flag on a Linux/Windows CI + box (real Apple Silicon is the ONLY environment _IS_MLX activates).""" + sys.modules.pop("unsloth", None) + import unsloth + assert unsloth._IS_MLX is False, ( + f"_IS_MLX activated on a non-Apple-Silicon runner under CUDA spoof; " + f"the MLX gate logic in unsloth/__init__.py is too lax" + ) + + +# ------------------------------------------------------------------------- +# unsloth.models.* — the core RL + sentence-transformer surfaces. These +# are the entry points unsloth/__init__.py loads transitively when a +# user does `from unsloth import FastLanguageModel`. +# ------------------------------------------------------------------------- + + +_UNSLOTH_CORE_MODULES = [ + "unsloth.models.rl", + "unsloth.models.rl_replacements", + "unsloth.models.sentence_transformer", + "unsloth.models._utils", + "unsloth.models.loader", + "unsloth.models.loader_utils", + "unsloth.models.mapper", +] + + +@pytest.mark.skipif(not _has_unsloth(), reason = "unsloth not installed") +@pytest.mark.parametrize("modname", _UNSLOTH_CORE_MODULES) +def test_unsloth_core_module_imports_under_spoof(modname: str): + """Core unsloth modules must import on a CPU-only runner under + the CUDA spoof. Drift in transformers/peft/trl symbols pinned at + module-top crashes here BEFORE any user-visible call. + + Bootstraps via `import unsloth` first, since most sub-modules + require the package's _gpu_init side effects. Without that, every + `import unsloth.models.*` raises a guard `Please restructure your + imports with 'import unsloth' at the top of your file.`""" + try: + import unsloth # noqa: F401 -- triggers _gpu_init side effects + except Exception as e: + pytest.skip(f"`import unsloth` failed under spoof: {e}") + sys.modules.pop(modname, None) + try: + importlib.import_module(modname) + except OSError as e: + # `OSError: could not get source code` happens when an editable + # install + frozen sub-import combine; that's an environment + # quirk, not a symbol-drift bug. Skip rather than false-fail. + pytest.skip(f"{modname} env issue: {e!s}") + except Exception as e: + pytest.fail( + f"{modname} failed to import under CUDA spoof: " + f"{type(e).__name__}: {str(e)[:300]}" + ) + + +# ------------------------------------------------------------------------- +# Public API surface dump for FastLanguageModel / FastVisionModel / +# FastModel under spoof. Asserts the surface is non-empty and that +# the patch hooks unsloth-zoo's RL surface relies on are present. +# ------------------------------------------------------------------------- + + +@pytest.mark.skipif(not _has_unsloth(), reason = "unsloth not installed") +def test_fast_model_class_surface_under_spoof(): + sys.modules.pop("unsloth", None) + import unsloth + found_at_least_one = False + for cls_name in ("FastLanguageModel", "FastVisionModel", "FastModel"): + cls = getattr(unsloth, cls_name, None) + if cls is None: + continue + found_at_least_one = True + public = sorted(n for n in dir(cls) if not n.startswith("_")) + # Notebooks rely on these methods. Loss of any one is a regression + # the existing api-introspect notebook job would catch a step + # later — but here at the import / spoof layer. + for method in ("from_pretrained", "get_peft_model"): + assert method in public, ( + f"unsloth.{cls_name}.{method} missing under spoof; " + f"every Colab notebook calling it breaks" + ) + assert found_at_least_one, ( + f"none of FastLanguageModel/FastVisionModel/FastModel reachable " + f"on `unsloth` package root" + ) + + +# ------------------------------------------------------------------------- +# RL surface drill-down: GRPO, SFT, DPO classes must be reachable AND +# the source-rewriter dispatch table must be populated. Catches the +# scenario where unsloth.models.rl_replacements imports cleanly but +# RL_FUNCTIONS or RL_REPLACEMENTS is silently empty. +# ------------------------------------------------------------------------- + + +@pytest.mark.skipif(not _has_unsloth(), reason = "unsloth not installed") +def test_unsloth_rl_replacements_dispatch_populated(): + try: + import unsloth # noqa: F401 -- _gpu_init bootstrap + except Exception as e: + pytest.skip(f"`import unsloth` failed under spoof: {e}") + sys.modules.pop("unsloth.models.rl_replacements", None) + try: + rl = importlib.import_module("unsloth.models.rl_replacements") + except OSError as e: + pytest.skip(f"env issue importing rl_replacements: {e!s}") + funcs = getattr(rl, "RL_FUNCTIONS", None) + if funcs is None: + pytest.skip( + "RL_FUNCTIONS attribute not present (architecture changed; check)" + ) + assert isinstance(funcs, dict), ( + f"RL_FUNCTIONS expected dict, got {type(funcs).__name__}" + ) + # The trainer types unsloth-zoo dispatches against MUST be keys. + for key in ("grpo_trainer", "sft_trainer", "dpo_trainer"): + assert key in funcs, ( + f"RL_FUNCTIONS missing dispatch key '{key}'; " + f"unsloth_zoo source rewrites silently no-op" + ) + assert isinstance(funcs[key], list) and len(funcs[key]) > 0, ( + f"RL_FUNCTIONS[{key!r}] is empty list; rewrites no-op" + ) + + +# ------------------------------------------------------------------------- +# unsloth-zoo compiler test_apply_fused_lm_head — exercises the actual +# fused-LM-head emit path with a tiny fixture. Already covered as a +# named test in compiler.py:1983; we just call it. +# ------------------------------------------------------------------------- + + +@pytest.mark.skipif(not _has_unsloth_zoo(), reason = "unsloth_zoo not installed") +def test_zoo_compiler_apply_fused_lm_head_callable(): + sys.modules.pop("unsloth_zoo.compiler", None) + compiler = importlib.import_module("unsloth_zoo.compiler") + fn = getattr(compiler, "test_apply_fused_lm_head", None) + assert fn is not None and callable(fn), ( + f"unsloth_zoo.compiler.test_apply_fused_lm_head missing or non-callable; " + f"the in-file CPU regression test is the only fused-LM-head coverage" + ) + + +# ------------------------------------------------------------------------- +# Spot-check signature stability of FastModel.from_pretrained — every +# notebook call site relies on these kwargs. A removed kwarg silently +# becomes positional drift. +# ------------------------------------------------------------------------- + + +@pytest.mark.skipif(not _has_unsloth(), reason = "unsloth not installed") +def test_fast_model_from_pretrained_kwargs_under_spoof(): + sys.modules.pop("unsloth", None) + import unsloth + cls = getattr(unsloth, "FastLanguageModel", None) or getattr( + unsloth, "FastModel", None + ) + if cls is None: + pytest.skip("FastLanguageModel/FastModel not exported") + fn = getattr(cls, "from_pretrained", None) + if fn is None: + pytest.skip("from_pretrained not on class (might be classmethod stub)") + try: + params = list(inspect.signature(fn).parameters) + except (TypeError, ValueError): + pytest.skip("from_pretrained signature not introspectable") + # Notebooks use these by name everywhere. + for kwarg in ("model_name", "max_seq_length", "load_in_4bit"): + assert kwarg in params, ( + f"FastLanguageModel.from_pretrained missing kwarg `{kwarg}`; " + f"every Colab notebook breaks at the install cell" + )