diff --git a/.github/workflows/consolidated-tests-ci.yml b/.github/workflows/consolidated-tests-ci.yml index bc7ab8a992..93152cc6e3 100644 --- a/.github/workflows/consolidated-tests-ci.yml +++ b/.github/workflows/consolidated-tests-ci.yml @@ -52,14 +52,44 @@ permissions: jobs: consolidated: - name: Consolidated tests (unsloth Bucket-A + unsloth_zoo@main + test_apply_fused_lm_head) + # Matrix: three (transformers, TRL) combos cover the failure surface the + # PR cares about: + # 1. transformers==4.57.6 + TRL latest <1.0.0 (the just-before-5.x line) + # 2. transformers latest 5.x + TRL latest 1.x (the absolute upstream tip; + # currently 5.8.0 + 1.3.0, both BEYOND the unsloth/unsloth_zoo + # <=5.5.0 / <=0.24.0 caps -- the cell exists explicitly to surface + # drift signal) + # 3. transformers + TRL pinned by pyproject.toml's dependency entries + # (resolved dynamically at job time via tomllib) + # fail-fast: false so each cell runs independently. continue-on-error + # at the job level keeps the PR unblocked while we observe the matrix. + strategy: + fail-fast: false + matrix: + combo: + - id: t4576-trl0latest + label: "T 4.57.6 + TRL <1" + transformers_spec: "transformers==4.57.6" + trl_spec: "trl>=0.18.2,<1.0.0" + - id: tlatest5-trl1latest + label: "T latest 5.x + TRL latest 1.x" + transformers_spec: "transformers>=5,<6" + trl_spec: "trl>=1,<2" + - id: pyproject + label: "pyproject.toml pins (dynamic)" + transformers_spec: "__from_pyproject__" + trl_spec: "__from_pyproject__" + name: "Consolidated CPU (${{ matrix.combo.label }})" runs-on: ubuntu-latest timeout-minutes: 35 # First-pass posture: surface results in the PR check UI without blocking - # merge. Flip to false (or delete this line) once one full run is green. + # merge. Flip to false (or delete this line) once every cell is green. continue-on-error: true env: UNSLOTH_ZOO_REF: ${{ inputs.unsloth_zoo_ref || 'main' }} + MATRIX_TRANSFORMERS_SPEC: ${{ matrix.combo.transformers_spec }} + MATRIX_TRL_SPEC: ${{ matrix.combo.trl_spec }} + MATRIX_COMBO_ID: ${{ matrix.combo.id }} # Hoisted to job-level so every step (Sanity, Bucket-A, unsloth_zoo # pytest, test_apply_fused_lm_head) inherits it. transformers' bundled # *_pb2.py was generated against an older protoc; the C++ protobuf @@ -95,6 +125,46 @@ jobs: - name: Install uv (some unsloth_zoo dev tooling expects it on PATH) run: pip install uv + - name: Resolve matrix specs (handle __from_pyproject__ sentinel) + # The pyproject cell uses a sentinel; resolve the real `transformers` + # and `trl` constraints from the project's pyproject.toml at job time. + # unsloth's pyproject puts the LLM stack pins in + # [project.optional-dependencies] under the `huggingfacenotorch` + # extra (top-level [project.dependencies] is just typer/pydantic/etc.), + # so we walk every optional extra and pick the first matching spec. + # Other cells pass their spec through unchanged. + run: | + set -euxo pipefail + python <<'PY' >> "$GITHUB_ENV" + import os, re, tomllib + spec_t = os.environ["MATRIX_TRANSFORMERS_SPEC"] + spec_r = os.environ["MATRIX_TRL_SPEC"] + + def _pkg_name(spec: str) -> str: + m = re.match(r"\s*([A-Za-z0-9_.-]+)", spec) + return (m.group(1).lower() if m else "") + + if spec_t == "__from_pyproject__" or spec_r == "__from_pyproject__": + with open("pyproject.toml", "rb") as f: + doc = tomllib.load(f) + proj = doc.get("project", {}) + # Try top-level deps first, then all optional extras. + all_deps: list[str] = list(proj.get("dependencies", [])) + for _name, dep_list in proj.get("optional-dependencies", {}).items(): + all_deps.extend(dep_list) + + if spec_t == "__from_pyproject__": + spec_t = next((x for x in all_deps if _pkg_name(x) == "transformers"), + "transformers") + if spec_r == "__from_pyproject__": + spec_r = next((x for x in all_deps if _pkg_name(x) == "trl"), + "trl") + print(f"RESOLVED_TRANSFORMERS_SPEC={spec_t}") + print(f"RESOLVED_TRL_SPEC={spec_r}") + PY + # Echo to logs so the matrix cell label maps cleanly to a spec. + grep RESOLVED_ "$GITHUB_ENV" || true + - name: Install runtime deps (mirrors studio-backend-ci.yml + mlx-ci.yml) # The shape matches studio-backend-ci.yml's "Repo tests (CPU)" install # so we inherit the same CPU-spoof harness in tests/conftest.py and @@ -110,6 +180,7 @@ jobs: # `import triton`. The triton PyPI wheel installs cleanly on # Linux x86_64 even without CUDA (the import succeeds; runtime # GPU work is what would fail, which we never do here). + # transformers + trl are matrix-parameterized. run: | set -euxo pipefail python -m pip install --upgrade pip @@ -120,17 +191,25 @@ jobs: 'numpy<3' pytest==9.0.3 pytest-asyncio httpx \ protobuf sentencepiece triton \ psutil packaging tqdm safetensors datasets \ - 'peft>=0.18,<0.20' 'accelerate>=0.34,<2' 'trl>=0.13,<1' + 'peft>=0.18,<0.20' 'accelerate>=0.34,<2' # torchvision: unsloth_zoo.vision_utils imports it at module scope. pip install --index-url https://download.pytorch.org/whl/cpu \ 'torch>=2.4,<2.11' 'torchvision<0.26' - pip install 'transformers>=4.51,<5.5' + # transformers + trl from the matrix combo. + pip install "$RESOLVED_TRANSFORMERS_SPEC" + pip install "$RESOLVED_TRL_SPEC" # bitsandbytes: hard import in unsloth/models/_utils.py. Recent # versions ship a CPU build that imports cleanly on Linux. pip install 'bitsandbytes>=0.45' # unsloth itself, editable, no-deps so pip does not fight the # explicit torch CPU-index install above. pip install -e . --no-deps + echo "::group::Installed transformers + trl + torch + unsloth versions" + pip show transformers + pip show trl + pip show torch + pip show unsloth + echo "::endgroup::" - name: Clone unsloth_zoo @ ${{ env.UNSLOTH_ZOO_REF }} # We need the repository tree (the wheel does not ship tests/), so @@ -210,6 +289,12 @@ jobs: # Auto-generated by .github/workflows/consolidated-tests-ci.yml. # Wraps unsloth_zoo.compiler.test_apply_fused_lm_head so that # tests/conftest.py's GPU-spoof harness applies before the import. + # _zoo_aggressive_cuda_spoof extends conftest's harness with deeper + # patches (see tests/_zoo_aggressive_cuda_spoof.py). + import sys, pathlib + sys.path.insert(0, str(pathlib.Path(__file__).parent)) + import _zoo_aggressive_cuda_spoof as _spoof + _spoof.apply() from unsloth_zoo.compiler import test_apply_fused_lm_head as _zoo_test def test_zoo_apply_fused_lm_head_runs(): _zoo_test() @@ -314,6 +399,10 @@ jobs: # tests/conftest.py GPU-spoof harness applies. continue-on-error # at the workflow level catches per-patch failures; this shim only # asserts that the two `required` helpers are reachable. + import sys, pathlib + sys.path.insert(0, str(pathlib.Path(__file__).parent)) + import _zoo_aggressive_cuda_spoof as _spoof + _spoof.apply() import importlib, inspect MODULES = [ @@ -392,6 +481,10 @@ jobs: set -euxo pipefail cat > tests/_tiled_mlp_check_shim.py <<'PY' # Auto-generated by .github/workflows/consolidated-tests-ci.yml. + import sys, pathlib + sys.path.insert(0, str(pathlib.Path(__file__).parent)) + import _zoo_aggressive_cuda_spoof as _spoof + _spoof.apply() import torch import torch.nn as nn from unsloth_zoo.tiled_mlp import patch_tiled_mlp, patch_mlp diff --git a/tests/_zoo_aggressive_cuda_spoof.py b/tests/_zoo_aggressive_cuda_spoof.py new file mode 100644 index 0000000000..c5cd47f139 --- /dev/null +++ b/tests/_zoo_aggressive_cuda_spoof.py @@ -0,0 +1,173 @@ +# Auto-generated by .github/workflows/consolidated-tests-ci.yml. +# Aggressive CUDA spoof for the consolidated CPU-only CI job. Extends +# tests/conftest.py:84-141's import-time harness with deeper patches that +# unblock more patch_* functions and unsloth_zoo init paths on a GPU-less +# runner. Imported by every shim test file in this workflow before any +# unsloth / unsloth_zoo / transformers import. +# +# Design: only no-op or value-returning patches. We do NOT replace tensor +# allocators. The single exception is `pin_memory=True` kwarg dropping, +# which converts a hard CUDA-required call into a CPU-OK call -- the +# intent of pin_memory is a CUDA-host fast-copy, which simply has no +# meaning on this runner; downgrading silently is the right behavior here. + +from __future__ import annotations + +import sys +import types +from typing import Any + + +def apply() -> None: + """Apply the spoof. Idempotent: calling again has no effect.""" + import torch + + if getattr(torch.cuda, "_unsloth_consolidated_spoof", False): + return + + # ----- device probes (cheap, value-returning) ------------------------- + torch.cuda.is_available = lambda: True + torch.cuda.device_count = lambda: 1 + torch.cuda.current_device = lambda: 0 + torch.cuda.is_initialized = lambda: True + torch.cuda.set_device = lambda *a, **k: None + torch.cuda.synchronize = lambda *a, **k: None + torch.cuda.empty_cache = lambda *a, **k: None + torch.cuda.get_device_name = lambda *a, **k: "NVIDIA A100-SPOOFED" + torch.cuda.get_device_capability = lambda *a, **k: (8, 0) + torch.cuda.is_bf16_supported = lambda *a, **k: True + torch.cuda._is_in_bad_fork = lambda *a, **k: False # type: ignore[attr-defined] + + class _Props: + name = "NVIDIA A100-SPOOFED" + major = 8 + minor = 0 + total_memory = 80 * 1024**3 + multi_processor_count = 108 + is_integrated = False + is_multi_gpu_board = False + + torch.cuda.get_device_properties = lambda *a, **k: _Props() # type: ignore[assignment] + + # ----- cudart() wrapper ----------------------------------------------- + class _CudaRt: + @staticmethod + def cudaMemGetInfo(device: int = 0): + return (0, 80 * 1024**3) + + @staticmethod + def cudaGetDeviceCount(*_a, **_k): + return 0 # Not used on the spoof path + + @staticmethod + def cudaSetDevice(*_a, **_k): + return 0 + + torch.cuda.cudart = lambda: _CudaRt() # type: ignore[assignment] + + # ----- memory module -------------------------------------------------- + try: + import torch.cuda.memory as _cuda_memory # type: ignore + + _cuda_memory.mem_get_info = lambda *a, **k: (0, 80 * 1024**3) + _cuda_memory.memory_stats = lambda *a, **k: {} + _cuda_memory.memory_allocated = lambda *a, **k: 0 + _cuda_memory.max_memory_allocated = lambda *a, **k: 0 + _cuda_memory.memory_reserved = lambda *a, **k: 0 + _cuda_memory.max_memory_reserved = lambda *a, **k: 0 + _cuda_memory.reset_peak_memory_stats = lambda *a, **k: None + except Exception: + pass + + # ----- nvtx no-op stub ------------------------------------------------ + nvtx_stub = types.ModuleType("torch.cuda.nvtx") + nvtx_stub.range_push = lambda *a, **k: None # type: ignore[attr-defined] + nvtx_stub.range_pop = lambda *a, **k: None # type: ignore[attr-defined] + nvtx_stub.mark = lambda *a, **k: None # type: ignore[attr-defined] + sys.modules.setdefault("torch.cuda.nvtx", nvtx_stub) + torch.cuda.nvtx = nvtx_stub # type: ignore[attr-defined] + + # ----- random API ---------------------------------------------------- + torch.cuda.manual_seed = lambda seed: torch.manual_seed(seed) # type: ignore[assignment] + torch.cuda.manual_seed_all = lambda seed: torch.manual_seed(seed) # type: ignore[assignment] + torch.cuda.get_rng_state = lambda *a, **k: torch.get_rng_state() # type: ignore[assignment] + torch.cuda.set_rng_state = lambda state, *a, **k: torch.set_rng_state(state) # type: ignore[assignment] + torch.cuda.get_rng_state_all = lambda *a, **k: [torch.get_rng_state()] # type: ignore[attr-defined] + torch.cuda.set_rng_state_all = lambda states, *a, **k: torch.set_rng_state(states[0]) if states else None # type: ignore[attr-defined] + + # ----- Stream / Event no-op classes ----------------------------------- + class _NoopStream: + def __init__(self, *a, **k): ... + def __enter__(self): return self + def __exit__(self, *a): return False + def synchronize(self, *a, **k): ... + def wait_stream(self, *a, **k): ... + def query(self): return True + + class _NoopEvent: + def __init__(self, *a, **k): ... + def record(self, *a, **k): ... + def wait(self, *a, **k): ... + def query(self): return True + def synchronize(self, *a, **k): ... + def elapsed_time(self, *a, **k): return 0.0 + + torch.cuda.Stream = _NoopStream # type: ignore[assignment] + torch.cuda.Event = _NoopEvent # type: ignore[assignment] + torch.cuda.stream = lambda s: s if s is not None else _NoopStream() # type: ignore[assignment] + torch.cuda.current_stream = lambda *a, **k: _NoopStream() # type: ignore[assignment] + torch.cuda.default_stream = lambda *a, **k: _NoopStream() # type: ignore[assignment] + + # ----- pin_memory drop ------------------------------------------------- + # `torch.empty(..., pin_memory=True)` and friends raise on a CPU-only + # build. Strip the kwarg — pin_memory has no meaning here. + for _name in ( + "empty", "zeros", "ones", + "empty_like", "zeros_like", "ones_like", + "rand", "randn", "randint", + ): + _orig = getattr(torch, _name, None) + if _orig is None: + continue + + def _wrap(*args: Any, _orig=_orig, **kwargs: Any): + kwargs.pop("pin_memory", None) + return _orig(*args, **kwargs) + + setattr(torch, _name, _wrap) + + # Tensor.pin_memory() instance method: also a no-op (return self). + if hasattr(torch.Tensor, "pin_memory"): + torch.Tensor.pin_memory = lambda self, *a, **k: self # type: ignore[assignment] + if hasattr(torch.Tensor, "is_pinned"): + torch.Tensor.is_pinned = lambda self, *a, **k: False # type: ignore[assignment] + + # ----- amp.GradScaler: use the real one if torch ships a CPU-friendly + # path, else stub. Newer torch ships torch.amp.GradScaler that handles + # CPU; torch.cuda.amp.GradScaler is a wrapper. Both should work; just + # guard against import error. + try: + import torch.cuda.amp # type: ignore + except Exception: + cuda_amp = types.ModuleType("torch.cuda.amp") + class _StubScaler: + def __init__(self, *a, **k): ... + def scale(self, x): return x + def step(self, opt): opt.step() + def update(self, *a, **k): ... + def unscale_(self, *a, **k): ... + def get_scale(self): return 1.0 + def is_enabled(self): return False + def state_dict(self): return {} + def load_state_dict(self, *a, **k): ... + cuda_amp.GradScaler = _StubScaler # type: ignore[attr-defined] + sys.modules.setdefault("torch.cuda.amp", cuda_amp) + torch.cuda.amp = cuda_amp # type: ignore[attr-defined] + + # ----- Sentinel ------------------------------------------------------ + torch.cuda._unsloth_consolidated_spoof = True # type: ignore[attr-defined] + + +if __name__ == "__main__": + apply() + print("CUDA spoof applied.")