# SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. # # Cross-repo notebook validator. Lives in unslothai/unsloth (this repo) # and inspects every notebook in unslothai/notebooks at HEAD (or the # ref dispatched in via repository_dispatch). # # Catches the bug classes that landed in: # - unslothai/notebooks#258 Colab torchao 0.10 vs peft 0.19 floor # - unslothai/notebooks#260 DONT_UPDATE_EXCEPTIONS coverage drift # - unslothai/notebooks#261 torch/torchcodec ABI; --no-deps tokenizers # - unslothai/notebooks#264 --no-deps transformers + Colab tokenizers drift # - unslothai/notebooks#221 git+ HEAD installs in install cells # - unslothai/notebooks commit 51b1462 template/notebook drift # # CPU-only by design. Layer 2 (api-introspect) reuses the existing # tests/_zoo_aggressive_cuda_spoof.py harness so `import unsloth` # succeeds on a GPU-less ubuntu-latest runner. name: Notebooks CI on: pull_request: paths: - 'unsloth/**' - 'scripts/notebook_validator.py' - 'scripts/notebook_to_python.py' - 'scripts/data/colab_pip_freeze.gpu.txt' - 'scripts/data/colab_to_cpu_pin.json' - 'tests/notebooks/**' - 'tests/_zoo_aggressive_cuda_spoof.py' - '.github/workflows/notebooks-ci.yml' schedule: # Daily 06:17 UTC. Catches Colab preinstall bumps (the upstream image # is rebuilt roughly weekly) without us waiting on a PR. Off the # :00/:30 fleet-collision spots. - cron: '17 6 * * *' workflow_dispatch: inputs: notebooks_ref: description: 'unslothai/notebooks ref to lint (branch / SHA / tag)' default: 'main' include_smoke: description: 'Also run the install-cell smoke matrix (longer)' type: boolean default: false repository_dispatch: # Fired by a tiny companion workflow on unslothai/notebooks. types: [notebooks_pr_opened, notebooks_main_pushed] concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true permissions: contents: read env: NOTEBOOKS_REF: >- ${{ github.event.inputs.notebooks_ref || github.event.client_payload.ref || 'main' }} jobs: static: name: static (drift + lint + exceptions) runs-on: ubuntu-latest timeout-minutes: 10 steps: # Validate the dispatched ref before it reaches actions/checkout's `ref:` # input. Reading via env (NOT direct ${{ ... }} interpolation in the # regex test) closes the GitHub-Actions-injection class where a # client_payload.ref like `main"; rm -rf / #` would be embedded into the # shell command. NOTEBOOKS_REF defaults to 'main' on non-dispatch # events, but only repository_dispatch can supply attacker-controlled # values, so we gate this check on that event type. - name: Validate client_payload.ref shape if: github.event_name == 'repository_dispatch' env: NOTEBOOKS_REF: ${{ github.event.client_payload.ref }} run: | if ! printf '%s' "$NOTEBOOKS_REF" | grep -Eq '^[A-Za-z0-9._/-]+$'; then echo "::error::client_payload.ref contains disallowed characters" >&2 exit 1 fi - name: Checkout unsloth (this PR) uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: path: unsloth persist-credentials: false - name: Checkout unslothai/notebooks @ ${{ env.NOTEBOOKS_REF }} uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: unslothai/notebooks ref: ${{ env.NOTEBOOKS_REF }} path: notebooks fetch-depth: 0 # drift check needs git status / diff persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.12' cache: 'pip' - name: Install validator deps run: | python -m pip install --upgrade pip # nbformat + nbconvert come from the converter's requirements; # spellchecker + huggingface_hub are imported at module top of # update_all_notebooks.py. pip install \ 'nbformat>=5.10' 'nbconvert>=7.16' 'pyspellchecker>=0.8' \ 'huggingface_hub>=0.34' 'tqdm>=4.66' - name: Refresh Colab pip-freeze (best-effort; falls back to snapshot) run: | python unsloth/scripts/notebook_validator.py refresh-colab \ --out unsloth/scripts/data/colab_pip_freeze.gpu.txt \ || echo "::warning::refresh-colab failed; using committed snapshot" - name: Diff Colab oracle vs committed snapshots (advisory) # Pulls pip-freeze.gpu.txt + apt-list-gpu.txt + os-info-gpu.txt # from googlecolab/backend-info and prints NEW / REMOVED / # CHANGED entries against scripts/data/colab_*.txt. Non-blocking # on PRs; the daily cron job below runs the same step with # --strict so upstream rotations surface within ~24h. continue-on-error: true working-directory: ${{ github.workspace }} run: | python unsloth/scripts/notebook_validator.py colab-diff \ --snapshot-dir unsloth/scripts/data - name: Drift check (re-run update_all_notebooks.py + git diff) working-directory: ${{ github.workspace }} # Reported as non-blocking until the upstream `unslothai/notebooks` # tree is regenerated. The first run on @main surfaces ~463 files # of drift (7359 / 9634 line delta), which is a real backlog the # notebooks-side maintainers need to clear in their own repo -- # this PR's role is to surface the count, not auto-fix it. continue-on-error: true run: | python unsloth/scripts/notebook_validator.py drift \ --notebooks-dir notebooks - name: Convert sanity (every nb / kaggle / original_template -> .py) # Same rationale as Drift: a handful of upstream notebooks fail # the converter (custom magics, malformed JSON, etc). Surface # the count without blocking; the team triages in unslothai/notebooks. continue-on-error: true run: | python unsloth/scripts/notebook_validator.py convert \ --notebooks-dir notebooks \ --out _converted - name: Lint (install cells + AST scan, env-scoped) # Reported as non-blocking (continue-on-error: true) until the # backlog of pre-existing findings on unslothai/notebooks@main is # cleared. Same pattern PR #5298 used for biome:check on the # frontend. As of this commit the live tree surfaces 27 errors + # 6 warnings, all real (peft/torchao floor missing in 6 nb/ # notebooks, 14 git+ HEAD installs in hand-tuned exception # notebooks, 6 torch/torchcodec ABI mismatches, 1 # transformers/tokenizers --no-deps drift). The count surfaces # in the PR check UI. Drop continue-on-error once it hits zero. continue-on-error: true run: | python unsloth/scripts/notebook_validator.py lint \ --notebooks-dir notebooks \ --colab-pin unsloth/scripts/data/colab_pip_freeze.gpu.txt \ --no-pypi # --no-pypi skips R-INST-002 (transitive resolve via PyPI metadata). # Layer 1 keeps PR-time wall-clock predictable; the daily cron run # below drops --no-pypi and refreshes the cache. - name: DONT_UPDATE_EXCEPTIONS coverage run: | python unsloth/scripts/notebook_validator.py exceptions \ --notebooks-dir notebooks static-with-pypi: name: static + transitive resolve (cron / dispatch only) if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} runs-on: ubuntu-latest timeout-minutes: 15 steps: # See `static.Validate client_payload.ref shape` for rationale. This # job's `if:` excludes repository_dispatch today, so the validation # step is a defence-in-depth no-op until that gate ever relaxes. - name: Validate client_payload.ref shape if: github.event_name == 'repository_dispatch' env: NOTEBOOKS_REF: ${{ github.event.client_payload.ref }} run: | if ! printf '%s' "$NOTEBOOKS_REF" | grep -Eq '^[A-Za-z0-9._/-]+$'; then echo "::error::client_payload.ref contains disallowed characters" >&2 exit 1 fi - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false path: unsloth - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: unslothai/notebooks ref: ${{ env.NOTEBOOKS_REF }} path: notebooks persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: { python-version: '3.12', cache: 'pip' } - name: Install run: pip install -U pip - name: Refresh Colab oracle run: | python unsloth/scripts/notebook_validator.py refresh-colab \ --out unsloth/scripts/data/colab_pip_freeze.gpu.txt - name: Diff Colab oracle vs committed snapshots (--strict on cron) # Cron-only escalation of the advisory PR-time check. Fails if # any of pip-freeze.gpu.txt / apt-list-gpu.txt / os-info-gpu.txt # has drifted from scripts/data/colab_*.txt; refresh the # snapshots in this repo to acknowledge. run: | python unsloth/scripts/notebook_validator.py colab-diff \ --snapshot-dir unsloth/scripts/data --strict - name: Lint with live PyPI metadata run: | python unsloth/scripts/notebook_validator.py lint \ --notebooks-dir notebooks \ --colab-pin unsloth/scripts/data/colab_pip_freeze.gpu.txt api-introspect: name: api surface (under CUDA spoof) runs-on: ubuntu-latest timeout-minutes: 12 steps: - name: Validate client_payload.ref shape if: github.event_name == 'repository_dispatch' env: NOTEBOOKS_REF: ${{ github.event.client_payload.ref }} run: | if ! printf '%s' "$NOTEBOOKS_REF" | grep -Eq '^[A-Za-z0-9._/-]+$'; then echo "::error::client_payload.ref contains disallowed characters" >&2 exit 1 fi - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false path: unsloth - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: unslothai/notebooks ref: ${{ env.NOTEBOOKS_REF }} path: notebooks persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: { python-version: '3.12', cache: 'pip' } - name: Install CPU torch + pinned unsloth + trl + converter deps run: | python -m pip install --upgrade pip # CPU torch + torchvision. torchvision is required because # unsloth_zoo.vision_utils imports PIL at module top, and the # easiest way to get a torch-compatible PIL on a CPU runner is # to let torchvision pull the right Pillow version. pip install --index-url https://download.pytorch.org/whl/cpu \ 'torch>=2.8,<2.11' 'torchvision<0.26' # Pin to the same versions update_all_notebooks.py installs in # generated notebooks. Keep these in lockstep with PIN_TRL / # PIN_TRANSFORMERS in unslothai/notebooks/update_all_notebooks.py. # `triton` is added because unsloth/_gpu_init.py:232 does an # unconditional `import triton`; the PyPI wheel installs cleanly # on Linux x86_64 even without CUDA (same rationale as # consolidated-tests-ci.yml line 192-205). # Pillow is listed explicitly as a defensive belt-and-braces # next to torchvision (vision_utils crashes ModuleNotFoundError # if torchvision skipped its Pillow dep for any reason). pip install 'transformers>=4.56,<5.6' 'trl>=0.22,<0.26' 'accelerate>=1.0' \ 'datasets>=3.4,<5' 'peft>=0.15,<0.20' \ 'bitsandbytes>=0.43' 'sentencepiece' 'protobuf' triton \ Pillow safetensors tqdm packaging psutil # Converter deps (nbformat for notebook_to_python.py). pip install 'nbformat>=5.10' 'nbconvert>=7.16' # Install unsloth from the LOCAL checkout (the PR head), not PyPI. # The PR-time CI must validate the code in this PR; PyPI unsloth # may lag the in-repo CPU-torch fallback in unsloth/kernels/utils.py # (lines 162-170) that handles missing torch._C._cuda_getCurrentRawStream. pip install --no-deps unsloth_zoo pip install --no-deps -e ./unsloth - name: Convert notebooks for AST scan # Same upstream-conversion-error tolerance as the static job. continue-on-error: true run: | python unsloth/scripts/notebook_validator.py convert \ --notebooks-dir notebooks --out _converted - name: Dump unsloth + trl API surface (under CUDA spoof) run: | PYTHONPATH=unsloth/tests python -u - <<'PY' import sys, json, inspect import _zoo_aggressive_cuda_spoof as _spoof _spoof.apply() import unsloth import trl surface = {} for cls_name in ("FastLanguageModel", "FastVisionModel", "FastModel"): cls = getattr(unsloth, cls_name, None) if cls is None: continue surface[cls_name] = sorted(n for n in dir(cls) if not n.startswith("_")) surface["SFTConfig_kwargs"] = sorted(inspect.signature(trl.SFTConfig.__init__).parameters) json.dump(surface, open("_api_surface.json", "w"), indent=2) print("dumped surface for:", list(surface)) PY - name: Run API rule against converted notebooks run: | python unsloth/scripts/notebook_validator.py api \ --converted-dir _converted \ --surface _api_surface.json smoke-install: name: smoke install (Colab-shaped venv, opt-in) if: ${{ github.event.inputs.include_smoke == 'true' || github.event_name == 'schedule' }} runs-on: ubuntu-latest timeout-minutes: 25 strategy: fail-fast: false matrix: # One representative notebook per installation_*_content template. # Add rows when a new install template lands in update_all_notebooks.py. notebook: - 'nb/Llama3.1_(8B)-Alpaca.ipynb' # installation_content - 'nb/Gemma3_(4B)-Vision.ipynb' # installation_content + vision - 'nb/Llama3.1_(8B)-GRPO.ipynb' # installation_extra_grpo_content - 'nb/gpt-oss-(20B)-Fine-tuning.ipynb' # installation_gpt_oss_content - 'nb/Qwen3_5_(4B)_Vision.ipynb' # installation_qwen3_5_content - 'nb/Nemotron-3-Nano-30B-A3B_A100.ipynb' # installation_nemotron_nano_content - 'nb/Whisper.ipynb' # installation_whisper_content - 'nb/Synthetic_Data_Hackathon.ipynb' # installation_synthetic_data_content steps: - name: Validate client_payload.ref shape if: github.event_name == 'repository_dispatch' env: NOTEBOOKS_REF: ${{ github.event.client_payload.ref }} run: | if ! printf '%s' "$NOTEBOOKS_REF" | grep -Eq '^[A-Za-z0-9._/-]+$'; then echo "::error::client_payload.ref contains disallowed characters" >&2 exit 1 fi - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false path: unsloth - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: unslothai/notebooks ref: ${{ env.NOTEBOOKS_REF }} path: notebooks persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: { python-version: '3.12' } - name: Seed Colab-shaped venv from pip-freeze (CPU-mapped) run: | # Strip cu128 local versions, route torch/torchvision to the CPU # wheel index, drop CUDA-specific deps the runner can't use. python -u - <<'PY' > /tmp/seed_pins.txt import json, re mapping = json.load(open("unsloth/scripts/data/colab_to_cpu_pin.json")) rewrite = mapping["rewrite"] skip = set(mapping["skip"]) spoof = set(mapping["module_spoof"]) out = [] for line in open("unsloth/scripts/data/colab_pip_freeze.gpu.txt"): line = line.strip() if not line or line.startswith("#"): continue m = re.match(r"^([A-Za-z0-9._-]+)\s*==\s*(.+)$", line) if not m: continue name, ver = m.group(1).lower(), m.group(2) if name in skip: continue if name in spoof: continue if name in rewrite: ver = re.sub(r"[+\-].+$", "", ver) out.append(f"{name}=={ver}") else: ver = re.sub(r"[+\-].+$", "", ver) out.append(f"{name}=={ver}") print("\n".join(out)) PY head -5 /tmp/seed_pins.txt wc -l /tmp/seed_pins.txt - name: Install Colab-shaped venv run: | python -m pip install --upgrade pip # Best-effort: any single line that fails to resolve on CPU is # tolerated; the smoke contract is "the install cell + the unsloth # import works", not "the entire Colab venv reproduces." while IFS= read -r spec; do pip install "$spec" --index-url https://download.pytorch.org/whl/cpu \ --extra-index-url https://pypi.org/simple || \ echo "::warning::pin failed: $spec" done < /tmp/seed_pins.txt - name: Run install cell run: | python unsloth/scripts/notebook_validator.py convert \ --notebooks-dir notebooks --out _converted # Take the converted .py and run the install cell only. BASE="$(basename '${{ matrix.notebook }}' .ipynb | tr -d '()' | tr -c '[:alnum:]_' _)" PY="_converted/${BASE}.py" [ -f "$PY" ] || { echo "::error::$PY not found"; ls _converted | head; exit 1; } # Truncate at the first `from unsloth import` so we run install + # core imports only. awk '/^from unsloth import/ { print "import sys; sys.exit(0)"; exit } { print }' "$PY" > _smoke.py PYTHONPATH=unsloth/tests python -u - <<'PY' import _zoo_aggressive_cuda_spoof as _s; _s.apply() # Stub torchcodec for cells that import it — no CPU wheel exists. import sys, types if "torchcodec" not in sys.modules: sys.modules["torchcodec"] = types.ModuleType("torchcodec") exec(open("_smoke.py").read(), {"__name__": "__main__"}) PY - name: Verify imports under spoof run: | PYTHONPATH=unsloth/tests python -u - <<'PY' import sys, types if "torchcodec" not in sys.modules: sys.modules["torchcodec"] = types.ModuleType("torchcodec") import _zoo_aggressive_cuda_spoof as _s; _s.apply() import unsloth, peft, torch, torchao, transformers, tokenizers print("OK: imports pass under CUDA spoof") PY