From c5b4f5ee86e89914f2a0005fd17c0c9e5225a03f Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 28 Jul 2026 12:07:33 +0000 Subject: [PATCH 01/36] CI: prove the installer works on a machine with no developer toolchain No job has ever run the installer on a machine without one. studio-mac-install-matrix.yml is the only macOS installer job and it runs 'bash install.sh --local --no-torch' on runners that already have the Xcode CLT selected and setup-python preinstalled, so the CLT gate never fires there, and --local is precisely the mode that legitimately needs git. Repo-wide there was zero coverage of xcode-select or CommandLineTools outside install.sh itself. clean-machine-install-ci.yml runs the installer on a genuinely stripped machine. macOS legs move /var/db/xcode_select_link, /Library/Developer/CommandLineTools, /Applications/Xcode*.app and Homebrew aside, so xcode-select -p, git, cc and clang really do fail, and restore unconditionally afterwards. Removing the select-link alone is not enough: xcode-select falls through to a full Xcode.app and re-arms /usr/bin/git. Linux legs use containers, which are genuinely clean. Windows legs cover winget visible and masked, plus windows-11-arm. A WSL leg covers the 126 lines of WSL-specific install.sh logic that had no runtime test. Each macOS leg runs four deliveries: pipe (the advertised command, and the shape that turns an early exit into curl (56)), file (separates installer logic from pipe delivery), no-torch, and tauri (stdin closed, no tty, as the desktop app invokes it). One leg records every toolchain invocation and asserts the trace, which is the real deliverable: proof the installer never reached for a compiler rather than proof it happened to succeed. The asserts test that tools do NOT WORK rather than that they are absent from PATH. On a real virgin Mac /usr/bin/git and /usr/bin/cc exist as CLT stubs, so 'command -v git' succeeds and only running it tells the truth. desktop-app-clean-machine-ci.yml installs and launches the SHIPPED desktop app release on a stripped machine, covering Gatekeeper and quarantine on macOS, NSIS silent install on Windows, and Xvfb with WebKit2GTK on Linux. Known limit, stated plainly: hosted macOS runners are developer machines. Masking reproduces this bug and proves the installer does not invoke a toolchain, but it cannot prove no hidden dependency exists on a truly virgin Mac. An ephemeral-VM lane is the follow-up. --- .github/scripts/clean-machine-assert.sh | 127 +++++ .github/scripts/clean-machine-env.sh | 165 ++++++ .../workflows/clean-machine-install-ci.yml | 528 ++++++++++++++++++ .../desktop-app-clean-machine-ci.yml | 354 ++++++++++++ 4 files changed, 1174 insertions(+) create mode 100755 .github/scripts/clean-machine-assert.sh create mode 100755 .github/scripts/clean-machine-env.sh create mode 100644 .github/workflows/clean-machine-install-ci.yml create mode 100644 .github/workflows/desktop-app-clean-machine-ci.yml diff --git a/.github/scripts/clean-machine-assert.sh b/.github/scripts/clean-machine-assert.sh new file mode 100755 index 0000000000..d96a6c8540 --- /dev/null +++ b/.github/scripts/clean-machine-assert.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +# +# Assert the clean-machine contract after an install attempt. +# +# absent The toolchain really was absent for the whole run. Guards against a +# leg that "passed" only because masking silently failed, or because +# the installer quietly installed Xcode CLT behind our back. +# notools The trace recorded no compiler/git/brew invocation (trace mode). +# nobuild The install log shows no source build (no sdist, no cmake, no +# "Building wheel"). This is the wheels-only contract. +# +# Usage: bash .github/scripts/clean-machine-assert.sh absent notools nobuild +set -uo pipefail + +LOG="${INSTALL_LOG:-logs/install.log}" +TRACE="${UNSLOTH_TOOL_TRACE:-}" +rc=0 + +fail() { echo "::error::$*"; rc=1; } +ok() { echo "[assert] OK $*"; } + +for check in "$@"; do + case "$check" in + + absent) + # Deliberately NOT a `command -v` check. On a real virgin Mac /usr/bin/git and + # /usr/bin/cc EXIST as Xcode CLT stubs, so `command -v git` SUCCEEDS -- running + # it is what fails ("xcrun: error: invalid active developer path"). Asserting on + # `command -v` would therefore be unfaithful and would fail on a correctly masked + # runner. The honest invariant is: the tool must not WORK. + if xcode-select -p >/dev/null 2>&1; then + fail "xcode-select -p still resolves to $(xcode-select -p 2>/dev/null); not a clean Mac" + else + ok "xcode-select -p fails (the gate a virgin Mac hits)" + fi + for tool in git cc clang cmake; do + command -v "$tool" >/dev/null 2>&1 || { ok "$tool not on PATH"; continue; } + if "$tool" --version >/dev/null 2>&1; then + fail "toolchain still usable: '$tool --version' succeeded ($(command -v "$tool")); masking failed" + else + ok "$tool present but non-functional (CLT stub), as on a clean Mac" + fi + done + # brew is a plain binary with no stub, so absence from PATH is the right test. + if command -v brew >/dev/null 2>&1; then + fail "Homebrew still on PATH at $(command -v brew); masking failed" + else + ok "brew absent" + fi + ;; + + notools) + if [ -z "$TRACE" ] || [ ! -f "$TRACE" ]; then + fail "notools requested but no trace file (\$UNSLOTH_TOOL_TRACE=$TRACE)" + else + # git is legitimate under --local (it installs unsloth-zoo from a git URL); + # UNSLOTH_ALLOW_TOOLS lets that leg allow-list it explicitly. + allow="${UNSLOTH_ALLOW_TOOLS:-}" + hits="" + while IFS=$'\t' read -r tool rest; do + [ -n "$tool" ] || continue + case " $allow " in *" $tool "*) continue ;; esac + # `xcode-select -p` ASKS whether a toolchain is selected; it cannot build + # anything. The installer has to ask in order to tell the user whether a + # source build is available, and the whole point of the fix is that it then + # carries on without one. Treating the question as toolchain USE would fail + # the very leg that proves the toolchain was never used. `--install`, which + # pops the CLT installer, stays a hit. + if [ "$tool" = "xcode-select" ]; then + case "$rest" in + -p|--print-path|-v|--version|"") continue ;; + esac + fi + hits="$hits $tool" + done < "$TRACE" + if [ -n "$hits" ]; then + fail "installer invoked toolchain:$(echo "$hits" | tr ' ' '\n' | sort -u | tr '\n' ' ')" + echo "---- tool trace ----"; sort -u "$TRACE" | head -50 + else + ok "no compiler/git/brew invocation recorded" + fi + fi + ;; + + nobuild) + # "Built an sdist" is NOT the same as "needed a compiler". Four packages on the + # macOS path are sdist-only PURE PYTHON projects that build fine with no + # toolchain (verified by resolving each against cp313/macos-arm64): + # openai-whisper, argbind, randomname -- no version ever ships a wheel + # antlr4-python3-runtime==4.9.3 -- pinned below the 4.13.2 wheel + # Failing on those would be a false alarm, so the contract asserted here is + # "nothing that needs a COMPILER was built", with that allowlist subtracted. + # UNSLOTH_ALLOW_SDIST can extend it. + _allow="openai-whisper argbind randomname antlr4-python3-runtime ${UNSLOTH_ALLOW_SDIST:-}" + if [ ! -f "$LOG" ]; then + fail "nobuild requested but $LOG is missing" + else + _built="$(grep -oiE "building wheel for [a-z0-9._-]+" "$LOG" 2>/dev/null \ + | sed -E 's/.* for //' | tr 'A-Z' 'a-z' | sort -u || true)" + _bad="" + for pkg in $_built; do + case " $_allow " in *" $pkg "*) continue ;; esac + _bad="$_bad $pkg" + done + if [ -n "$_bad" ]; then + fail "built from source:$_bad -- these must resolve to wheels on a clean machine" + else + [ -n "$_built" ] && say_built="$(echo "$_built" | tr '\n' ' ')" || say_built="none" + ok "no non-allowlisted source build (built: $say_built)" + fi + # Independent of package names: a compiler error means a toolchain was needed. + if grep -qiE "error: command '(cc|gcc|clang|cl)' failed|no such file or directory: 'cc'|clang: error|cargo: not found|error: linker \`cc\` not found" "$LOG"; then + fail "compiler invocation appears in the install log" + grep -iE "error: command '(cc|gcc|clang|cl)' failed|clang: error" "$LOG" | head -10 + fi + fi + ;; + + *) + fail "unknown check '$check'" + ;; + esac +done + +exit "$rc" diff --git a/.github/scripts/clean-machine-env.sh b/.github/scripts/clean-machine-env.sh new file mode 100755 index 0000000000..c1e838d35a --- /dev/null +++ b/.github/scripts/clean-machine-env.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +# +# Simulate a virgin developer machine on a GitHub-hosted runner, so the installer +# is exercised the way a real user's brand-new Mac / PC exercises it. +# +# Two modes, because "the tool is absent" and "the installer never called the tool" +# CANNOT be simulated by the same mechanism: +# +# mask Make the toolchain genuinely ABSENT. Scrubs PATH down to the OS +# defaults and (with --remove) moves the real toolchain aside. After +# this, `command -v git` correctly FAILS, which is what a clean Mac does. +# A failing "poison shim" on PATH would do the opposite -- `command -v` +# finds it and reports the tool as present -- so shims are NOT used here. +# +# trace Leave the toolchain working, but route it through logging wrappers that +# record the invocation and then exec the real binary. Proves whether the +# installer ever REACHES for a compiler/git, without changing behaviour. +# +# Writes shell exports to $CLEAN_ENV_FILE (default ./clean-machine.env) for the +# caller to `source`. Nothing is exported globally, so other workflow steps +# (checkout, upload-artifact) keep a normal environment. +# +# Usage: +# bash .github/scripts/clean-machine-env.sh mask [--remove] +# bash .github/scripts/clean-machine-env.sh trace +# source ./clean-machine.env +set -uo pipefail + +MODE="${1:-}" +REMOVE=0 +[ "${2:-}" = "--remove" ] && REMOVE=1 + +case "$MODE" in + mask|trace) ;; + *) echo "usage: $0 {mask|trace} [--remove]" >&2; exit 2 ;; +esac + +OS="$(uname -s)" +WORK="${CLEAN_MACHINE_DIR:-$PWD/.clean-machine}" +ENV_FILE="${CLEAN_ENV_FILE:-$PWD/clean-machine.env}" +TRACE="$WORK/tool-invocations.log" +BIN="$WORK/bin" +RESTORE="$WORK/restore.sh" +mkdir -p "$BIN" +: > "$TRACE" +: > "$ENV_FILE" +printf '#!/usr/bin/env bash\n# Undo clean-machine-env.sh --remove. Safe to run twice.\nset -uo pipefail\n' > "$RESTORE" +chmod +x "$RESTORE" + +# The toolchain we care about: a consumer install must need none of it. +TOOLS="xcode-select xcrun clang clang++ cc c++ gcc g++ git cmake make brew ninja cargo rustc" + +note() { echo "[clean-machine] $*"; } + +# ── PATH scrub ──────────────────────────────────────────────────────────────── +# Keep only OS-default system dirs. Drops Homebrew, the hosted Python toolcache, +# setup-* shims, pipx, cargo, and every other preinstalled developer dir. +scrub_path() { + local keep out="" + if [ "$OS" = "Darwin" ]; then + keep="/usr/bin:/bin:/usr/sbin:/sbin" + else + keep="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + fi + local IFS=":" + for d in $keep; do + [ -d "$d" ] && out="${out:+$out:}$d" + done + echo "$out" +} + +# ── mask ────────────────────────────────────────────────────────────────────── +if [ "$MODE" = "mask" ]; then + NEWPATH="$(scrub_path)" + { + echo "export PATH='$NEWPATH'" + # DEVELOPER_DIR must be UNSET, not pointed at a fake path: `xcode-select -p` + # honours DEVELOPER_DIR and prints it verbatim with exit 0, so setting it to a + # nonexistent dir makes the probe SUCCEED -- the exact opposite of a clean Mac, + # where DEVELOPER_DIR is unset and the missing /var/db/xcode_select_link is what + # makes `xcode-select -p` fail. + echo "unset DEVELOPER_DIR || true" + echo "unset SDKROOT CC CXX CFLAGS CXXFLAGS LDFLAGS CMAKE_GENERATOR CMAKE_PREFIX_PATH || true" + echo "export HOMEBREW_NO_AUTO_UPDATE=1" + echo "export UNSLOTH_CLEAN_MACHINE=1" + } >> "$ENV_FILE" + + if [ "$REMOVE" = "1" ] && [ "$OS" = "Darwin" ]; then + # Best-effort real removal. Each step is independent and recorded in + # restore.sh so an `if: always()` step can put the runner back. + # /var/db/xcode_select_link is exactly what `xcode-select -p` reads, so + # removing it reproduces a virgin Mac's gate precisely. `xcode-select --reset` + # is NOT enough: it can reselect a full Xcode.app. + if [ -e /var/db/xcode_select_link ]; then + if sudo rm -f /var/db/xcode_select_link 2>/dev/null; then + note "removed /var/db/xcode_select_link" + echo "sudo xcode-select --switch /Library/Developer/CommandLineTools 2>/dev/null || true" >> "$RESTORE" + else + note "WARN could not remove /var/db/xcode_select_link" + fi + fi + # Moving the CLT dir aside turns /usr/bin/{cc,clang,git} into dead shims, so + # the run also proves the install needs no compiler at all. + if [ -d /Library/Developer/CommandLineTools ]; then + if sudo mv /Library/Developer/CommandLineTools /Library/Developer/CommandLineTools.masked 2>/dev/null; then + note "moved CommandLineTools aside" + echo "sudo mv /Library/Developer/CommandLineTools.masked /Library/Developer/CommandLineTools 2>/dev/null || true" >> "$RESTORE" + else + note "WARN could not move CommandLineTools" + fi + fi + # Xcode.app must go too. With the select link removed AND CommandLineTools moved, + # `xcode-select -p` does not fail -- it falls through to whatever Xcode bundle the + # runner image ships (observed: /Applications/Xcode_16.4.app/Contents/Developer), + # which re-arms /usr/bin/git and /usr/bin/cc and silently un-cleans the machine. + # A rename is instant regardless of bundle size: same filesystem, no copy. + for app in /Applications/Xcode*.app; do + [ -d "$app" ] || continue + if sudo mv "$app" "${app}.masked" 2>/dev/null; then + note "moved $(basename "$app") aside" + echo "sudo mv '${app}.masked' '$app' 2>/dev/null || true" >> "$RESTORE" + else + note "WARN could not move $app" + fi + done + for brewdir in /opt/homebrew /usr/local/Homebrew; do + if [ -d "$brewdir" ]; then + if sudo mv "$brewdir" "${brewdir}.masked" 2>/dev/null; then + note "moved $brewdir aside" + echo "sudo mv '${brewdir}.masked' '$brewdir' 2>/dev/null || true" >> "$RESTORE" + else + note "WARN could not move $brewdir" + fi + fi + done + fi +fi + +# ── trace ───────────────────────────────────────────────────────────────────── +if [ "$MODE" = "trace" ]; then + for tool in $TOOLS; do + real="$(command -v "$tool" 2>/dev/null || true)" + [ -n "$real" ] || continue + # Wrapper logs the call then execs the REAL binary, so behaviour is unchanged + # and the trace answers "did the installer reach for this?" honestly. + cat > "$BIN/$tool" <> "$TRACE" +exec "$real" "\$@" +WRAP + chmod +x "$BIN/$tool" + done + { + echo "export PATH='$BIN:$PATH'" + echo "export UNSLOTH_TOOL_TRACE='$TRACE'" + echo "export UNSLOTH_CLEAN_MACHINE=trace" + } >> "$ENV_FILE" +fi + +note "mode=$MODE remove=$REMOVE" +note "env file: $ENV_FILE" +note "trace: $TRACE" +note "restore: $RESTORE" diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml new file mode 100644 index 0000000000..acfe56e48f --- /dev/null +++ b/.github/workflows/clean-machine-install-ci.yml @@ -0,0 +1,528 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +# Proves Unsloth installs on a machine that has never seen a developer toolchain. +# +# Why this exists: studio-mac-install-matrix.yml runs `install.sh --local --no-torch` +# on runners that already have Xcode CLT selected AND actions/setup-python +# preinstalled, so the macOS dependency gate in install.sh never fires there -- and +# `--local` is precisely the mode that legitimately needs git. A brand-new Mac +# therefore hits a hard `exit 1` that no CI job covered. +# +# Hosted runners are developer machines, so each job simulates absence rather than +# being virgin. Two modes, because they answer different questions and cannot be +# done by the same mechanism (see .github/scripts/clean-machine-env.sh): +# mask -> the toolchain is genuinely unusable; does the install still work? +# trace -> the toolchain works but is logged; does the installer ever call it? +# Linux is the exception: containers are genuinely clean. + +name: Clean machine install + +on: + pull_request: + paths: + - 'install.sh' + - 'install.ps1' + - 'studio/setup.sh' + - 'studio/setup.ps1' + - 'studio/install_python_stack.py' + - '.github/scripts/clean-machine-*.sh' + - '.github/workflows/clean-machine-install-ci.yml' + push: + branches: [main] + paths: + - 'install.sh' + - 'install.ps1' + - '.github/workflows/clean-machine-install-ci.yml' + workflow_dispatch: + inputs: + installer_source: + description: 'published = curl unsloth.ai/install.sh, tree = the checked-out script' + type: choice + options: [tree, published] + default: tree + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + # Keep every install inside the workspace so a leg cannot inherit another's state. + UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home + # No wildcard bind -> no ifconfig.me / check-host.net calls on the startup path. + UNSLOTH_STUDIO_DISABLE_PUBLIC_CHECK: '1' + +jobs: + # ── macOS: the reported failure ──────────────────────────────────────────── + macos: + name: mac ${{ matrix.os }} / ${{ matrix.mode }} / ${{ matrix.delivery }}${{ matrix.flags && format(' {0}', matrix.flags) || '' }} + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + continue-on-error: ${{ matrix.experimental }} + # Explicit legs rather than a full cross-product: the interesting dimensions are + # (does the toolchain exist) x (how the script is delivered), not every pairing. + strategy: + fail-fast: false + matrix: + include: + # The reported failure, in the shape users run it. Default install (torch + # included) because that is what a consumer actually gets. + - {os: macos-14, mode: mask, delivery: pipe, flags: '', experimental: false} + - {os: macos-14, mode: mask, delivery: file, flags: '', experimental: false} + # What the desktop app runs: no tty, stdin closed, TAURI markers on. + - {os: macos-14, mode: mask, delivery: tauri, flags: '', experimental: false} + # Toolchain present but logged: does the installer ever reach for it? + - {os: macos-14, mode: trace, delivery: file, flags: '', experimental: false} + # --no-torch is the one macOS path that can still want a compiler + # (sentencepiece has no guaranteed cp313 arm64 wheel), so probe it apart + # from the default path instead of letting it mask the gate under test. + - {os: macos-14, mode: mask, delivery: file, flags: '--no-torch', experimental: true} + # OS-version dimension. + - {os: macos-15, mode: mask, delivery: pipe, flags: '', experimental: false} + - {os: macos-26, mode: mask, delivery: file, flags: '', experimental: true} + # Intel pins python 3.12, not 3.13 -- informational only. + - {os: macos-15-intel, mode: mask, delivery: file, flags: '', experimental: true} + + steps: + # checkout FIRST: it needs a working git, which masking then takes away. + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + # Deliberately no actions/setup-python: install.sh must bring its own + # uv-managed CPython, exactly as it must on a user's machine. + + - name: Record the pre-masking toolchain + run: | + { + echo "xcode-select -p : $(xcode-select -p 2>&1 || true)" + echo "git : $(command -v git || echo none)" + echo "brew : $(command -v brew || echo none)" + echo "cmake : $(command -v cmake || echo none)" + echo "python3 : $(command -v python3 || echo none)" + } | tee runner-baseline.txt + + - name: Simulate a clean machine (${{ matrix.mode }}) + run: | + mkdir -p logs + if [ "${{ matrix.mode }}" = "mask" ]; then + bash .github/scripts/clean-machine-env.sh mask --remove + else + bash .github/scripts/clean-machine-env.sh trace + fi + + - name: Verify the simulation actually took effect + if: matrix.mode == 'mask' + run: | + set -a; . ./clean-machine.env; set +a + bash .github/scripts/clean-machine-assert.sh absent + + - name: Install + id: install + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Withheld on PR: this step runs checked-out PR code. + HF_TOKEN: ${{ github.event_name != 'pull_request' && secrets.HF_TOKEN || '' }} + run: | + set -a; . ./clean-machine.env; set +a + set -o pipefail + rc=0 + FLAGS="${{ matrix.flags }}" + case "${{ matrix.delivery }}" in + file) + # Plain file execution: isolates "installer logic broken" from + # "curl-pipe delivery broken". + bash install.sh $FLAGS 2>&1 | tee logs/install.log || rc=$? + ;; + pipe) + # The shape users actually run. install.sh is ~150KB of top-level + # statements, so an early `exit` leaves the writer with a closed + # pipe -> `curl: (56)`. Piping a local file reproduces that + # faithfully without depending on unsloth.ai being current. + # On pull_request/push this input is empty, which correctly falls through to + # the checked-out ref -- only an explicit dispatch tests unsloth.ai. + if [ "${{ inputs.installer_source }}" = "published" ]; then + curl -fsSL https://unsloth.ai/install.sh | sh -s -- $FLAGS 2>&1 | tee logs/install.log || rc=$? + else + # `sh -s --` with no further args would pass an empty positional, + # so only add the separator when there are flags to pass. + if [ -n "$FLAGS" ]; then + cat install.sh | sh -s -- $FLAGS 2>&1 | tee logs/install.log || rc=$? + else + cat install.sh | sh 2>&1 | tee logs/install.log || rc=$? + fi + fi + ;; + tauri) + # Exactly how the desktop app invokes it: no tty, stdin closed. + # --tauri rejects a custom UNSLOTH_STUDIO_HOME outright (the desktop app + # still uses the legacy ~/.unsloth/studio root), so the workspace-scoped + # value every other leg relies on has to go here or the installer exits + # before it does any work. The runner is ephemeral, so the real home is + # as disposable as the override. + env -u UNSLOTH_STUDIO_HOME \ + bash install.sh --tauri $FLAGS < /dev/null 2>&1 | tee logs/install.log || rc=$? + ;; + esac + echo "install_rc=$rc" >> "$GITHUB_OUTPUT" + echo "installer exit code: $rc" + # The pipe legs are the ones that expose curl:(56); surface it explicitly + # rather than leaving it buried in a 4000-line log. + if grep -qE "curl: \(5[36]\)|Failure writing output to destination" logs/install.log; then + echo "::warning::curl reported a broken pipe -- an early exit killed the reader" + fi + exit "$rc" + + - name: Assert no source build and no toolchain use + if: always() && steps.install.outcome == 'success' + run: | + set -a; . ./clean-machine.env; set +a + checks="nobuild" + [ "${{ matrix.mode }}" = "trace" ] && checks="$checks notools" + bash .github/scripts/clean-machine-assert.sh $checks + + - name: Assert llama.cpp loads + if: steps.install.outcome == 'success' + run: | + set -a; . ./clean-machine.env; set +a + # The tauri leg cannot honour UNSLOTH_STUDIO_HOME (see the Install step), so + # it installed into the legacy root and that is where to look. + if [ "${{ matrix.delivery }}" = "tauri" ]; then + HOME_DIR="$HOME/.unsloth/studio" + else + HOME_DIR="$UNSLOTH_STUDIO_HOME" + fi + STUDIO_HOME="$HOME_DIR" bash .github/scripts/assert-llama-loads.sh + + - name: Restore the runner + if: always() + run: bash .clean-machine/restore.sh || true + + - name: Upload logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: clean-mac-${{ matrix.os }}-${{ matrix.mode }}-${{ matrix.delivery }} + path: | + logs/ + runner-baseline.txt + clean-machine.env + .clean-machine/tool-invocations.log + retention-days: 7 + if-no-files-found: warn + + # ── Linux: genuinely clean, via containers ──────────────────────────────── + linux: + name: linux ${{ matrix.label }} + runs-on: ${{ matrix.runner }} + container: ${{ matrix.image }} + timeout-minutes: 40 + continue-on-error: ${{ matrix.experimental }} + # Container jobs default to `sh -e`, which is dash: `set -o pipefail` is an + # "Illegal option" there and kills the step before the installer even starts. + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + include: + # Root + apt available: install.sh's _smart_apt_install should self-heal + # from a base image with no curl, git, gcc or cmake at all. + - label: ubuntu2404-root + image: ubuntu:24.04 + runner: ubuntu-latest + experimental: false + - label: ubuntu2404-arm-root + image: ubuntu:24.04 + runner: ubuntu-24.04-arm + experimental: false + # No elevation: today this hard-fails at install.sh:856-861. Expected + # failure -- the point is to pin the message and prove it is actionable + # rather than a bare `curl: (56)`. + - label: ubuntu2404-nonroot + image: ubuntu:24.04 + runner: ubuntu-latest + experimental: true + # Non-apt: today this hard-fails at install.sh:2034. Expected failure; + # forces the decision on whether dnf/pacman/zypper get supported. + - label: fedora41 + image: fedora:41 + runner: ubuntu-latest + experimental: true + + steps: + - name: Describe the container's starting state + run: | + for t in curl wget git gcc cc cmake make python3 sudo; do + printf '%-8s %s\n' "$t" "$(command -v $t 2>/dev/null || echo ABSENT)" + done | tee /tmp/container-baseline.txt + + # The advertised `curl | sh` cannot even start on an image without curl, so + # the bootstrap transport is provisioned separately from the installer's own + # dependencies. Everything else stays absent. + - name: Provision only the bootstrap transport + run: | + if command -v apt-get >/dev/null 2>&1; then + apt-get update -qq && apt-get install -y -qq --no-install-recommends ca-certificates curl + elif command -v dnf >/dev/null 2>&1; then + dnf install -y -q ca-certificates curl + fi + + # No actions/checkout here on purpose: it requires git, and a container with git + # preinstalled is not the clean machine under test. Fetch the two files we need + # over the transport provisioned above -- and fetch the INSTALLER from the same + # ref, so these legs can validate a fix instead of only the published script. + - name: Fetch installer + assert script for this ref + run: | + mkdir -p logs .github/scripts + raw="https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/${GITHUB_SHA}" + curl -fsSL "$raw/.github/scripts/clean-machine-assert.sh" -o .github/scripts/clean-machine-assert.sh + # On pull_request/push this input is empty, which correctly falls through to + # the checked-out ref -- only an explicit dispatch tests unsloth.ai. + if [ "${{ inputs.installer_source }}" = "published" ]; then + curl -fsSL https://unsloth.ai/install.sh -o install.sh + echo "installer: published (unsloth.ai)" + else + curl -fsSL "$raw/install.sh" -o install.sh + echo "installer: this ref (${GITHUB_SHA})" + fi + wc -l install.sh + + - name: Create an unprivileged user + if: matrix.label == 'ubuntu2404-nonroot' + run: | + useradd -m tester + # No sudo installed and not root -> exercises the "cannot elevate" branch. + chown -R tester logs install.sh + + - name: Install (root) + if: matrix.label != 'ubuntu2404-nonroot' + run: | + set -o pipefail + rc=0 + # Piped, because that is the advertised command and the shape that turns an + # early exit into curl:(56). + cat install.sh | sh 2>&1 | tee logs/install.log || rc=$? + echo "installer exit code: $rc" + exit "$rc" + + - name: Install (unprivileged, expected to fail cleanly) + if: matrix.label == 'ubuntu2404-nonroot' + run: | + set -o pipefail + rc=0 + su tester -c 'cat install.sh | sh' > logs/install.log 2>&1 || rc=$? + echo "installer exit code: $rc" + tail -40 logs/install.log + # It may legitimately fail; what must NOT happen is an unexplained exit or a + # bare broken-pipe error standing in for a real diagnosis. + if [ "$rc" != "0" ] && ! grep -qiE "sudo is not available|apt-get install|missing:|permission" logs/install.log; then + echo "::error::unprivileged install failed with no actionable message" + exit 1 + fi + + - name: Assert no source build + if: always() + run: | + if [ -f .github/scripts/clean-machine-assert.sh ]; then + INSTALL_LOG=logs/install.log bash .github/scripts/clean-machine-assert.sh nobuild + else + echo "::warning::assert script unavailable (fetch step did not run)" + fi + + - name: Upload logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: clean-linux-${{ matrix.label }} + path: | + logs/ + /tmp/container-baseline.txt + retention-days: 7 + if-no-files-found: warn + + # ── WSL ─────────────────────────────────────────────────────────────────── + # install.sh carries ~126 lines of WSL-specific logic (the `linux|wsl` dependency + # branch, UNSLOTH_WSL_REROUTED, the Strix Halo reroute to 24.04), and none of it + # had ever run in CI -- tests/sh/test_strixhalo_wsl_reroute.sh extracts functions + # and mocks the environment, which cannot catch anything about a real WSL. + # + # No third-party action: the official Ubuntu WSL rootfs plus `wsl --import` is + # deterministic, checksum-verifiable, and avoids adding a supply-chain dependency + # to a repo that audits its lockfiles. + wsl: + name: wsl ubuntu-24.04 + runs-on: windows-latest + timeout-minutes: 50 + continue-on-error: true + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Import a fresh Ubuntu 24.04 WSL distro + shell: pwsh + run: | + # WSL2 is present on windows-2022+ runner images; only a distro is missing. + wsl --set-default-version 2 + $url = 'https://cloud-images.ubuntu.com/wsl/releases/24.04/current/ubuntu-noble-wsl-amd64-24.04lts.rootfs.tar.gz' + $expected = '2a790896740b14d637dbdc583cce1ba081ac53b9e9cdb46dc09a2f73abbd9934' + New-Item -ItemType Directory -Force -Path wsl-dist, logs | Out-Null + Invoke-WebRequest -Uri $url -OutFile wsl-dist/rootfs.tar.gz -UseBasicParsing -TimeoutSec 900 + $actual = (Get-FileHash wsl-dist/rootfs.tar.gz -Algorithm SHA256).Hash.ToLower() + if ($actual -ne $expected) { + Write-Host "::error::rootfs checksum mismatch: got $actual" + exit 1 + } + wsl --import unsloth-ci "$PWD/wsl-dist/instance" "$PWD/wsl-dist/rootfs.tar.gz" --version 2 + wsl -d unsloth-ci -- uname -a + # A freshly imported rootfs is genuinely bare: no curl, no git, no compiler. + # That is the clean machine, not a simulation of one. + wsl -d unsloth-ci -- sh -c 'for t in curl wget git gcc cmake python3 sudo; do printf "%-8s %s\n" "$t" "$(command -v $t || echo ABSENT)"; done' + + - name: Install inside WSL, piped exactly as documented + shell: pwsh + run: | + # Only ca-certificates + curl, because the advertised one-liner cannot even + # start without a transport. Everything else must come from the installer. + wsl -d unsloth-ci -u root -- sh -c 'apt-get update -qq && apt-get install -y -qq --no-install-recommends ca-certificates curl' 2>&1 | Tee-Object -FilePath logs/wsl-bootstrap.log + # Copy the script in rather than reaching across /mnt/c: a DrvFs path brings + # Windows file permissions and CRLF risk with it, and neither is what a real + # WSL user's install looks like. + $wslPath = (wsl -d unsloth-ci -- wslpath -a "$($env:GITHUB_WORKSPACE -replace '\\','/')/install.sh").Trim() + Write-Host "installer source in WSL: $wslPath" + wsl -d unsloth-ci -u root -- cp "$wslPath" /root/install.sh + # Feed it through a pipe: same shape as `curl ... | sh`, so an early exit + # still exposes the broken-pipe problem, but the script under test is this + # ref rather than whatever production currently serves. + wsl -d unsloth-ci -u root -- sh -c 'cd /root && cat install.sh | sh' 2>&1 | Tee-Object -FilePath logs/wsl-install.log + Write-Host "installer exit: $LASTEXITCODE" + + - name: Did it detect WSL, and did it end up usable? + if: always() + shell: pwsh + run: | + # The platform line proves the wsl branch was taken rather than plain linux. + Select-String -Path logs/wsl-install.log -Pattern 'platform|\[TAURI:DIAG\]|wsl' -ErrorAction SilentlyContinue | + Select-Object -First 10 + wsl -d unsloth-ci -u root -- sh -c 'test -x "$HOME/.unsloth/studio/unsloth_studio/bin/unsloth" && "$HOME/.unsloth/studio/unsloth_studio/bin/unsloth" --version || echo "no CLI installed"' 2>&1 | + Tee-Object -FilePath logs/wsl-verify.log + + - name: Tear the distro down + if: always() + shell: pwsh + run: wsl --unregister unsloth-ci 2>&1 | Out-Null; exit 0 + + - name: Upload logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: clean-wsl-ubuntu2404 + path: logs/ + retention-days: 7 + if-no-files-found: warn + + # ── Windows ─────────────────────────────────────────────────────────────── + windows: + name: win ${{ matrix.os }} / winget=${{ matrix.winget }} + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + include: + - os: windows-latest + winget: 'visible' + experimental: false + # The no-winget path (LTSC / Server / managed corporate machines) falls + # back to python.org + astral.sh and is completely untested today. It is + # also the path where Ensure-VCRedist silently does not run, which leaves + # torch unable to load -- hence the explicit `import torch` assert below. + - os: windows-latest + winget: 'masked' + experimental: false + - os: windows-11-arm + winget: 'visible' + experimental: true + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + # No actions/setup-python here either: install.ps1 must bootstrap Python. + + - name: Simulate a clean machine + shell: pwsh + run: | + New-Item -ItemType Directory -Force -Path logs | Out-Null + # Drop preinstalled Python, git, CMake, VS/LLVM and the WindowsApps + # aliases from PATH. A full Visual Studio uninstall is not realistic in + # CI (registry + vswhere discovery, slow, may need a reboot), so PATH and + # env scrubbing is the honest approximation -- recorded as such. + $drop = @('hostedtoolcache\windows\Python', 'WindowsApps', '\Git\', + 'CMake', 'Microsoft Visual Studio', 'BuildTools', 'LLVM', + 'MSYS', 'mingw', 'Strawberry') + $kept = ($env:PATH -split ';') | Where-Object { + $p = $_; $p -and -not ($drop | Where-Object { $p -like "*$_*" }) + } + if ('${{ matrix.winget }}' -eq 'masked') { + $kept = $kept | Where-Object { $_ -notlike '*WinGet*' -and $_ -notlike '*Microsoft\WindowsApps*' } + } + $newPath = ($kept -join ';') + "PATH=$newPath" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + foreach ($v in 'VSINSTALLDIR','VCINSTALLDIR','WindowsSdkDir','INCLUDE','LIB','LIBPATH') { + "$v=" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + } + Write-Host "kept PATH entries: $($kept.Count)" + + - name: Verify the simulation took effect + shell: pwsh + run: | + foreach ($t in 'python','git','cmake','cl') { + $f = Get-Command $t -ErrorAction SilentlyContinue + Write-Host ("{0,-8} {1}" -f $t, $(if ($f) { $f.Source } else { 'ABSENT' })) + } + if ('${{ matrix.winget }}' -eq 'masked' -and (Get-Command winget -ErrorAction SilentlyContinue)) { + Write-Host '::error::winget still resolvable; masking failed' + exit 1 + } + + - name: Install + id: install + shell: pwsh + run: | + $ErrorActionPreference = 'Continue' + & ./install.ps1 -SkipTorch *>&1 | Tee-Object -FilePath logs/install.log + $rc = $LASTEXITCODE + Write-Host "installer exit code: $rc" + exit $rc + + - name: Assert torch loads (the VCRedist contract) + if: steps.install.outcome == 'success' + shell: pwsh + run: | + # The prebuilt llama-server and PyTorch both link the VC++ runtime. + # Ensure-VCRedist only runs when winget exists, so on the masked leg this + # is the assertion that catches a silently broken install. + $py = Join-Path $env:UNSLOTH_STUDIO_HOME 'unsloth_studio\Scripts\python.exe' + if (-not (Test-Path $py)) { $py = (Get-Command python -ErrorAction SilentlyContinue).Source } + if (-not $py) { Write-Host '::error::no python from the install'; exit 1 } + & $py -c "import ctypes.util, sys; print('VCRUNTIME140:', ctypes.util.find_library('vcruntime140'))" + & $py -c "import torch; print('torch', torch.__version__)" + if ($LASTEXITCODE -ne 0) { Write-Host '::error::torch failed to import (VC++ runtime missing?)'; exit 1 } + + - name: Upload logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: clean-win-${{ matrix.os }}-${{ matrix.winget }} + path: logs/ + retention-days: 7 + if-no-files-found: warn diff --git a/.github/workflows/desktop-app-clean-machine-ci.yml b/.github/workflows/desktop-app-clean-machine-ci.yml new file mode 100644 index 0000000000..246e423dcc --- /dev/null +++ b/.github/workflows/desktop-app-clean-machine-ci.yml @@ -0,0 +1,354 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +# Installs and launches the SHIPPED desktop app on a machine stripped of developer +# tooling, on all three platforms. +# +# studio-tauri-smoke.yml compiles the Tauri crate; release-desktop.yml produces the +# bundles. Neither takes a published artifact, puts it on a clean machine, and checks +# that it starts -- which is exactly the gap the reported failures fell through: both +# came from the packaged app running its bundled +# Contents/Resources/install.sh, a path no CI job exercised. +# +# What "runs" means here, given hosted runners have no interactive desktop session: +# - the bundle installs / mounts / extracts +# - the binary is present, of the right architecture, and passes the OS gatekeeper +# checks a user would hit (macOS quarantine + codesign, Windows installer exit) +# - the process starts and STAYS UP past its preflight (it does not exit or crash), +# which is where an unhappy app dies +# - it writes tauri.log, and that log shows the preflight disposition -- the same +# field that read `ManagedReady` over an unbootable venv in the bug report +# Linux gets the strongest check: a real webview under Xvfb. + +name: Desktop app clean machine + +on: + # Also on PRs that touch this job or the machine-stripping scripts. workflow_dispatch + # alone is not enough to validate a change to the job itself: dispatch resolves the + # workflow from the DEFAULT branch, so a new or edited file on a feature branch can + # never be dispatched, and the job would first run only after merging blind. + pull_request: + paths: + - '.github/workflows/desktop-app-clean-machine-ci.yml' + - '.github/scripts/clean-machine-env.sh' + - '.github/scripts/clean-machine-assert.sh' + workflow_dispatch: + inputs: + release_tag: + description: 'Release tag in the desktop release repo' + type: string + default: 'desktop-v0.1.50-beta' + release_repo: + description: 'owner/name hosting the desktop release' + type: string + default: 'unsloth-test/unsloth-test' + strip_toolchain: + description: 'Strip developer tooling before installing' + type: boolean + default: true + schedule: + # Nightly, so a broken published bundle is caught without anyone asking. + - cron: '17 5 * * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + REL_REPO: ${{ inputs.release_repo || 'unsloth-test/unsloth-test' }} + REL_TAG: ${{ inputs.release_tag || 'desktop-v0.1.50-beta' }} + UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home + UNSLOTH_STUDIO_DISABLE_PUBLIC_CHECK: '1' + +jobs: + # ── macOS: .dmg, Apple Silicon ──────────────────────────────────────────── + macos: + name: desktop macOS ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + include: + - {os: macos-14, experimental: false} + - {os: macos-15, experimental: false} + - {os: macos-26, experimental: true} + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Download the shipped .dmg + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + mkdir -p dl logs + gh release download "$REL_TAG" --repo "$REL_REPO" \ + --pattern '*aarch64.dmg' --dir dl + ls -la dl + + - name: Strip the developer toolchain + if: ${{ inputs.strip_toolchain != false }} + run: | + bash .github/scripts/clean-machine-env.sh mask --remove + set -a; . ./clean-machine.env; set +a + bash .github/scripts/clean-machine-assert.sh absent + + - name: Mount and install + run: | + DMG="$(ls dl/*.dmg | head -1)" + # A real download is quarantined; Gatekeeper treats it differently from a + # locally built bundle, and that difference is a genuine failure mode. + xattr -w com.apple.quarantine \ + "0081;$(printf %x $(date +%s));Safari;" "$DMG" 2>/dev/null || true + hdiutil attach "$DMG" -nobrowse -quiet -mountpoint /Volumes/UnslothCI + APP="$(ls -d /Volumes/UnslothCI/*.app | head -1)" + echo "app bundle: $APP" + cp -R "$APP" /Applications/ + hdiutil detach /Volumes/UnslothCI -quiet + ls -la /Applications | grep -i unsloth + + - name: Inspect the bundle (arch, signature, Gatekeeper) + run: | + APP="$(ls -d /Applications/*Unsloth*.app | head -1)" + BIN="$APP/Contents/MacOS/$(/usr/libexec/PlistBuddy -c 'Print :CFBundleExecutable' "$APP/Contents/Info.plist")" + file "$BIN" + lipo -archs "$BIN" || true + # Report rather than gate: an unnotarised beta is expected to fail + # assessment, but a user WILL hit this, so it must be visible. + codesign -dv --verbose=2 "$APP" 2>&1 | head -20 || true + spctl -a -vvv -t install "$APP" 2>&1 | head -5 || \ + echo "::warning::Gatekeeper assessment failed -- users see 'cannot be opened' unless notarised" + # The bundled installer is what actually failed for users. + test -f "$APP/Contents/Resources/install.sh" \ + && echo "bundled install.sh present" \ + || echo "::error::no bundled install.sh in the app" + + - name: Launch and prove it stays up + run: | + set -a; [ -f ./clean-machine.env ] && . ./clean-machine.env; set +a + APP="$(ls -d /Applications/*Unsloth*.app | head -1)" + BIN="$APP/Contents/MacOS/$(/usr/libexec/PlistBuddy -c 'Print :CFBundleExecutable' "$APP/Contents/Info.plist")" + "$BIN" > logs/app-stdout.log 2>&1 & + APP_PID=$! + # 90s: long enough to clear preflight and start the bundled installer. + for i in $(seq 1 90); do + kill -0 "$APP_PID" 2>/dev/null || break + sleep 1 + done + if kill -0 "$APP_PID" 2>/dev/null; then + echo "app still running after 90s (pid $APP_PID)" + kill -TERM "$APP_PID" 2>/dev/null || true + else + wait "$APP_PID" 2>/dev/null; rc=$? + echo "::error::desktop app exited early with rc=$rc" + tail -50 logs/app-stdout.log || true + exit 1 + fi + + - name: What did its own log say? + if: always() + run: | + for f in "$UNSLOTH_STUDIO_HOME/tauri.log" "$HOME/.unsloth/studio/tauri.log"; do + [ -f "$f" ] || continue + echo "=== $f ===" + cp "$f" logs/ 2>/dev/null || true + tail -60 "$f" + # The two fields the bug report turned on. + grep -E "disposition=|can_auto_repair=|Xcode Command Line|ModuleNotFoundError" "$f" || true + done + + - name: Restore the runner + if: always() + run: bash .clean-machine/restore.sh || true + + - name: Upload logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: desktop-macos-${{ matrix.os }} + path: logs/ + retention-days: 7 + if-no-files-found: warn + + # ── Linux: .deb and .AppImage, with a real webview under Xvfb ──────────── + linux: + name: desktop linux ${{ matrix.kind }} + runs-on: ubuntu-22.04 + timeout-minutes: 45 + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + include: + - {kind: deb, experimental: false} + - {kind: appimage, experimental: false} + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Download the shipped bundle + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + mkdir -p dl logs + pat='*.deb'; [ "${{ matrix.kind }}" = "appimage" ] && pat='*.AppImage' + gh release download "$REL_TAG" --repo "$REL_REPO" --pattern "$pat" --dir dl + ls -la dl + + - name: Install with NO dev tooling, only runtime libs + run: | + # Deliberately not build-essential/cmake/git: a user installing a .deb has + # none of that. WebKit + Xvfb are runtime requirements of the app itself, + # and apt pulls the .deb's declared deps -- if that list is wrong, this + # step is what catches it. + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends xvfb + if [ "${{ matrix.kind }}" = "deb" ]; then + sudo apt-get install -y ./dl/*.deb || { + echo "::error::the .deb does not declare its runtime dependencies correctly" + exit 1 + } + BIN="$(dpkg -L "$(dpkg-deb -f dl/*.deb Package)" | grep -E '/usr/bin/' | head -1)" + else + sudo apt-get install -y -qq --no-install-recommends libfuse2 \ + libwebkit2gtk-4.1-0 libgtk-3-0 libayatana-appindicator3-1 || true + chmod +x dl/*.AppImage + BIN="$(ls dl/*.AppImage | head -1)" + fi + echo "BIN=$BIN" >> "$GITHUB_ENV" + echo "binary: $BIN" + + - name: Launch under Xvfb and prove it stays up + run: | + # Linux is the one platform where a hosted runner can give the app a real + # display, so this is the strongest "does the UI actually come up" check + # available without self-hosted hardware. + xvfb-run -a --server-args="-screen 0 1440x900x24" \ + "$BIN" > logs/app-stdout.log 2>&1 & + APP_PID=$! + for i in $(seq 1 90); do + kill -0 "$APP_PID" 2>/dev/null || break + sleep 1 + done + if kill -0 "$APP_PID" 2>/dev/null; then + echo "app still running after 90s" + kill -TERM "$APP_PID" 2>/dev/null || true + else + wait "$APP_PID" 2>/dev/null; rc=$? + echo "::error::desktop app exited early with rc=$rc" + tail -60 logs/app-stdout.log || true + exit 1 + fi + + - name: What did its own log say? + if: always() + run: | + for f in "$UNSLOTH_STUDIO_HOME/tauri.log" "$HOME/.unsloth/studio/tauri.log"; do + [ -f "$f" ] || continue + echo "=== $f ==="; cp "$f" logs/ 2>/dev/null || true; tail -60 "$f" + grep -E "disposition=|can_auto_repair=|ModuleNotFoundError" "$f" || true + done + + - name: Upload logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: desktop-linux-${{ matrix.kind }} + path: logs/ + retention-days: 7 + if-no-files-found: warn + + # ── Windows: NSIS setup.exe, silent install ────────────────────────────── + windows: + name: desktop windows + runs-on: windows-latest + timeout-minutes: 45 + continue-on-error: true + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Download the shipped installer + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + mkdir -p dl logs + gh release download "$REL_TAG" --repo "$REL_REPO" --pattern '*setup.exe' --dir dl + ls -la dl + + - name: Strip developer tooling from PATH + if: ${{ inputs.strip_toolchain != false }} + shell: pwsh + run: | + $drop = @('hostedtoolcache\windows\Python', 'WindowsApps', '\Git\', 'CMake', + 'Microsoft Visual Studio', 'BuildTools', 'LLVM', 'MSYS', 'mingw') + $kept = ($env:PATH -split ';') | Where-Object { + $p = $_; $p -and -not ($drop | Where-Object { $p -like "*$_*" }) + } + "PATH=$($kept -join ';')" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + + - name: Silent install + shell: pwsh + run: | + $exe = (Get-ChildItem dl/*setup.exe | Select-Object -First 1).FullName + # /S is the NSIS silent switch. A user double-clicks, but an installer that + # cannot run unattended also cannot be scripted or MDM-deployed. + $p = Start-Process -FilePath $exe -ArgumentList '/S' -Wait -PassThru + Write-Host "installer exit: $($p.ExitCode)" + if ($p.ExitCode -ne 0) { Write-Host "::error::silent install failed"; exit 1 } + $found = Get-ChildItem -Path "$env:LOCALAPPDATA","$env:ProgramFiles" -Recurse ` + -Filter '*Unsloth*.exe' -ErrorAction SilentlyContinue | + Select-Object -First 1 + if (-not $found) { Write-Host '::error::no installed executable found'; exit 1 } + Write-Host "installed: $($found.FullName)" + "APP_EXE=$($found.FullName)" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + + - name: Launch and prove it stays up + shell: pwsh + run: | + $p = Start-Process -FilePath $env:APP_EXE -PassThru ` + -RedirectStandardOutput logs/app-stdout.log ` + -RedirectStandardError logs/app-stderr.log + for ($i = 0; $i -lt 90; $i++) { if ($p.HasExited) { break }; Start-Sleep -Seconds 1 } + if ($p.HasExited) { + Write-Host "::error::desktop app exited early with rc=$($p.ExitCode)" + Get-Content logs/app-stdout.log, logs/app-stderr.log -Tail 40 -ErrorAction SilentlyContinue + exit 1 + } + Write-Host "app still running after 90s" + Stop-Process -Id $p.Id -Force -ErrorAction SilentlyContinue + + - name: What did its own log say? + if: always() + shell: pwsh + run: | + foreach ($f in @("$env:UNSLOTH_STUDIO_HOME\tauri.log", + "$env:USERPROFILE\.unsloth\studio\tauri.log")) { + if (Test-Path $f) { + Write-Host "=== $f ===" + Copy-Item $f logs/ -ErrorAction SilentlyContinue + Get-Content $f -Tail 60 + Select-String -Path $f -Pattern 'disposition=|can_auto_repair=|ModuleNotFoundError' ` + -ErrorAction SilentlyContinue + } + } + + - name: Upload logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: desktop-windows + path: logs/ + retention-days: 7 + if-no-files-found: warn From d2ade8ad1e068856e277258ab0229487b111ef83 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 28 Jul 2026 12:23:35 +0000 Subject: [PATCH 02/36] Point the llama assert at the right root, and name the Intel limitation The tauri leg installs to the legacy root because --tauri refuses a custom UNSLOTH_STUDIO_HOME. Its install succeeds end to end, but llama.cpp lives at /llama.cpp while the venv is at /studio, so the assert was pointed one level too deep. On macos-15-intel /usr/bin/git keeps working once the CLT are gone, so it is not CLT-provided there and no masking can remove it, while cc and clang do become stubs. Calling that 'masking failed' was wrong. That leg allowlists git explicitly and says why, so the assert stays strict everywhere else. --- .github/scripts/clean-machine-assert.sh | 10 ++++++++++ .github/workflows/clean-machine-install-ci.yml | 13 ++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/scripts/clean-machine-assert.sh b/.github/scripts/clean-machine-assert.sh index d96a6c8540..9a281043d2 100755 --- a/.github/scripts/clean-machine-assert.sh +++ b/.github/scripts/clean-machine-assert.sh @@ -38,6 +38,16 @@ for check in "$@"; do for tool in git cc clang cmake; do command -v "$tool" >/dev/null 2>&1 || { ok "$tool not on PATH"; continue; } if "$tool" --version >/dev/null 2>&1; then + # On Intel runners /usr/bin/git keeps working once the CLT are gone, so it + # is not CLT-provided there and no masking can remove it. cc and clang do + # become stubs, and the consumer path needs no git on macOS, so report it + # rather than calling the simulation broken. + case " ${UNSLOTH_CLEAN_ALLOW_WORKING:-} " in + *" $tool "*) + echo "[assert] NOTE $tool still works ($(command -v "$tool")); allowed on this runner" + continue + ;; + esac fail "toolchain still usable: '$tool --version' succeeded ($(command -v "$tool")); masking failed" else ok "$tool present but non-functional (CLT stub), as on a clean Mac" diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index acfe56e48f..104bcfed3b 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -83,8 +83,9 @@ jobs: # OS-version dimension. - {os: macos-15, mode: mask, delivery: pipe, flags: '', experimental: false} - {os: macos-26, mode: mask, delivery: file, flags: '', experimental: true} - # Intel pins python 3.12, not 3.13 -- informational only. - - {os: macos-15-intel, mode: mask, delivery: file, flags: '', experimental: true} + # Intel pins python 3.12 and its /usr/bin/git is not CLT-provided, so it + # survives masking. Informational only. + - {os: macos-15-intel, mode: mask, delivery: file, flags: '', experimental: true, allow_working: 'git'} steps: # checkout FIRST: it needs a working git, which masking then takes away. @@ -118,7 +119,8 @@ jobs: if: matrix.mode == 'mask' run: | set -a; . ./clean-machine.env; set +a - bash .github/scripts/clean-machine-assert.sh absent + UNSLOTH_CLEAN_ALLOW_WORKING='${{ matrix.allow_working }}' \ + bash .github/scripts/clean-machine-assert.sh absent - name: Install id: install @@ -189,9 +191,10 @@ jobs: run: | set -a; . ./clean-machine.env; set +a # The tauri leg cannot honour UNSLOTH_STUDIO_HOME (see the Install step), so - # it installed into the legacy root and that is where to look. + # it installed into the legacy root. llama.cpp sits at /llama.cpp and + # the venv at /studio, so this is ~/.unsloth, not ~/.unsloth/studio. if [ "${{ matrix.delivery }}" = "tauri" ]; then - HOME_DIR="$HOME/.unsloth/studio" + HOME_DIR="$HOME/.unsloth" else HOME_DIR="$UNSLOTH_STUDIO_HOME" fi From 0699e5c72b2a3c88fd7f6410c475a36d1c42489b Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 13:45:35 +0000 Subject: [PATCH 03/36] Make the clean-machine legs able to fail The toolchain strip never ran on the automatic triggers: inputs exists only for workflow_dispatch, and GitHub coerces '' and false alike to 0, so `inputs.strip_toolchain != false` was false. Confirmed on a pull_request run where the strip step reports skipped. Gate on the event instead. Also: scrub the Machine and User registry PATH, since install.ps1 rebuilds $env:Path from them mid-install and the toolchain came back; stop dropping WindowsApps unconditionally, which removed winget on the winget=visible leg too; fail rather than annotate when a bundle ships no installer or no CLI; run the bundled installer, which a headless launch never reaches; resolve the newest desktop-v* release instead of a pinned immutable tag; and give the two macOS matrix rows distinct artifact names. --- .github/scripts/clean-machine-assert.sh | 20 ++- .../workflows/clean-machine-install-ci.yml | 142 ++++++++++++++++-- .../desktop-app-clean-machine-ci.yml | 84 ++++++++++- 3 files changed, 221 insertions(+), 25 deletions(-) diff --git a/.github/scripts/clean-machine-assert.sh b/.github/scripts/clean-machine-assert.sh index 9a281043d2..686c067324 100755 --- a/.github/scripts/clean-machine-assert.sh +++ b/.github/scripts/clean-machine-assert.sh @@ -9,7 +9,10 @@ # the installer quietly installed Xcode CLT behind our back. # notools The trace recorded no compiler/git/brew invocation (trace mode). # nobuild The install log shows no source build (no sdist, no cmake, no -# "Building wheel"). This is the wheels-only contract. +# "Building wheel" from pip and no "Building ==" from uv). +# This is the wheels-only contract. It needs UNSLOTH_VERBOSE=1 on the +# installer, otherwise run_install_cmd (install.sh:193-243) throws the +# uv output away on success and there is nothing here to read. # # Usage: bash .github/scripts/clean-machine-assert.sh absent notools nobuild set -uo pipefail @@ -107,8 +110,19 @@ for check in "$@"; do if [ ! -f "$LOG" ]; then fail "nobuild requested but $LOG is missing" else - _built="$(grep -oiE "building wheel for [a-z0-9._-]+" "$LOG" 2>/dev/null \ - | sed -E 's/.* for //' | tr 'A-Z' 'a-z' | sort -u || true)" + # The installer runs `uv pip install`, and uv does NOT use pip's phrasing. + # It prints ` Building ==` and ` Built ==` + # to stderr, as plain lines once stderr is not a TTY (astral-sh/uv#11165), so + # the pip-only pattern left _built empty on every uv source build. Match both + # spellings. Requiring `==` or ` @ ` after the name keeps this off the + # installer's own lowercase "building frontend..." progress text. Strip ANSI + # first so a coloured run (FORCE_COLOR) still parses. + _esc=$(printf '\033') + _built="$(sed -E "s/${_esc}\[[0-9;]*[A-Za-z]//g" "$LOG" 2>/dev/null \ + | grep -oiE "building wheel for [a-z0-9._-]+|building [a-z0-9._-]+(==| @ )" \ + | tr 'A-Z' 'a-z' \ + | sed -E -e 's/^building wheel for //' -e 's/^building //' -e 's/(==| @ )$//' \ + | sort -u || true)" _bad="" for pkg in $_built; do case " $_allow " in *" $pkg "*) continue ;; esac diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index 104bcfed3b..4afc55c45e 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -54,6 +54,10 @@ env: UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home # No wildcard bind -> no ifconfig.me / check-host.net calls on the startup path. UNSLOTH_STUDIO_DISABLE_PUBLIC_CHECK: '1' + # Without this, run_install_cmd (install.sh:193-243) sends every `uv pip install` + # to a temp file and DELETES it on success, so logs/install.log holds no uv output + # and the `nobuild` assertion can only ever report "built: none". + UNSLOTH_VERBOSE: '1' jobs: # ── macOS: the reported failure ──────────────────────────────────────────── @@ -208,7 +212,9 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: clean-mac-${{ matrix.os }}-${{ matrix.mode }}-${{ matrix.delivery }} + # The rows at lines 74 and 82 differ only in `flags`, so the flags have to + # be in the name: artifacts are immutable per run and the second upload 409s. + name: clean-mac-${{ matrix.os }}-${{ matrix.mode }}-${{ matrix.delivery }}${{ matrix.flags && format('-{0}', matrix.flags) || '' }} path: | logs/ runner-baseline.txt @@ -299,10 +305,18 @@ jobs: if: matrix.label == 'ubuntu2404-nonroot' run: | useradd -m tester + # Switching user without a login shell keeps the caller's environment, so the + # workflow-wide UNSLOTH_STUDIO_HOME follows tester in -- and install.sh both + # resolves AND validates that override in _resolve_studio_destinations + # (install.sh:503-559), which runs long before the elevation gate at + # install.sh:840-861. Without a writable target this leg dies on + # "cannot be created" instead of on "cannot elevate". + mkdir -p "$UNSLOTH_STUDIO_HOME" # No sudo installed and not root -> exercises the "cannot elevate" branch. - chown -R tester logs install.sh + chown -R tester logs install.sh "$UNSLOTH_STUDIO_HOME" - name: Install (root) + id: install_root if: matrix.label != 'ubuntu2404-nonroot' run: | set -o pipefail @@ -328,6 +342,23 @@ jobs: exit 1 fi + # The nonroot leg checks that its expected failure is the expected one. This leg + # is continue-on-error too, so without the same check a bootstrap outage or an + # unrelated early exit is tolerated exactly like the intentional diagnostic. + - name: Assert the Fedora failure is the unsupported-package-manager one + if: always() && matrix.label == 'fedora41' + run: | + if [ "${{ steps.install_root.outcome }}" = "success" ]; then + echo "::warning::fedora install succeeded -- non-apt support may now exist; retire this leg" + exit 0 + fi + [ -f logs/install.log ] || { echo "::error::fedora leg produced no install log"; exit 1; } + tail -40 logs/install.log + if ! grep -qiE "Automatic system package installation is supported on apt-based|Fedora/RHEL: sudo dnf install" logs/install.log; then + echo "::error::fedora leg failed for a reason other than the unsupported package manager" + exit 1 + fi + - name: Assert no source build if: always() run: | @@ -413,8 +444,16 @@ jobs: # The platform line proves the wsl branch was taken rather than plain linux. Select-String -Path logs/wsl-install.log -Pattern 'platform|\[TAURI:DIAG\]|wsl' -ErrorAction SilentlyContinue | Select-Object -First 10 - wsl -d unsloth-ci -u root -- sh -c 'test -x "$HOME/.unsloth/studio/unsloth_studio/bin/unsloth" && "$HOME/.unsloth/studio/unsloth_studio/bin/unsloth" --version || echo "no CLI installed"' 2>&1 | - Tee-Object -FilePath logs/wsl-verify.log + # No `|| echo`: substituting a message for the missing CLI made the inner + # shell -- and so this step, and so the job -- succeed even when the install + # produced nothing usable, which is the half of the question this step asks. + $verify = wsl -d unsloth-ci -u root -- sh -c 'set -e; test -x "$HOME/.unsloth/studio/unsloth_studio/bin/unsloth"; "$HOME/.unsloth/studio/unsloth_studio/bin/unsloth" --version' 2>&1 + $verifyRc = $LASTEXITCODE + $verify | Tee-Object -FilePath logs/wsl-verify.log + if ($verifyRc -ne 0) { + Write-Host '::error::WSL install left no usable unsloth CLI' + exit 1 + } - name: Tear the distro down if: always() @@ -472,14 +511,57 @@ jobs: $drop = @('hostedtoolcache\windows\Python', 'WindowsApps', '\Git\', 'CMake', 'Microsoft Visual Studio', 'BuildTools', 'LLVM', 'MSYS', 'mingw', 'Strawberry') - $kept = ($env:PATH -split ';') | Where-Object { - $p = $_; $p -and -not ($drop | Where-Object { $p -like "*$_*" }) + # winget ships as an app-execution alias inside ...\Local\Microsoft\WindowsApps, + # which the blanket drop above removes on EVERY leg -- so winget=visible was + # silently running the same no-winget fallback as winget=masked. Resolve it + # before the scrub and hand it back through a shim, so the visible leg gets + # winget without also getting the Store's python.exe alias back. + # windows-11-arm has no winget at all on the hosted image + # (actions/runner-images#14083), so only windows-latest can carry it. + $wantWinget = ('${{ matrix.winget }}' -ne 'masked') -and ('${{ matrix.os }}' -eq 'windows-latest') + $wingetCmd = Get-Command winget -ErrorAction SilentlyContinue + $scrub = { + param($entries) + $out = $entries | Where-Object { + $p = $_; $p -and -not ($drop | Where-Object { $p -like "*$_*" }) + } + if ('${{ matrix.winget }}' -eq 'masked') { + $out = $out | Where-Object { $_ -notlike '*WinGet*' -and $_ -notlike '*Microsoft\WindowsApps*' } + } + ,@($out) } - if ('${{ matrix.winget }}' -eq 'masked') { - $kept = $kept | Where-Object { $_ -notlike '*WinGet*' -and $_ -notlike '*Microsoft\WindowsApps*' } + $kept = & $scrub ($env:PATH -split ';') + if ($wantWinget) { + if (-not $wingetCmd) { + Write-Host '::error::winget was not on PATH before scrubbing; this leg cannot test the winget path' + exit 1 + } + $shim = Join-Path $env:RUNNER_TEMP 'winget-shim' + New-Item -ItemType Directory -Force -Path $shim | Out-Null + Set-Content -LiteralPath (Join-Path $shim 'winget.cmd') -Encoding ascii ` + -Value "@`"$($wingetCmd.Source)`" %*" + $kept = @($shim) + $kept } $newPath = ($kept -join ';') "PATH=$newPath" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + # install.ps1 calls Refresh-SessionPath (defined install.ps1:318-337, called at + # 1246/1278/1295/1360/1369/2797), which rebuilds $env:Path from the Machine and + # User registry values. Scrubbing only the process PATH therefore lasts until + # the first bootstrap refresh, after which Git/CMake/VS/LLVM are back and the + # rest of the install is no longer running on a simulated clean machine. + # The runner is ephemeral, so rewrite the registry copies too. Expand first: + # SetEnvironmentVariable rewrites REG_EXPAND_SZ as REG_SZ (dotnet/runtime#1442). + foreach ($scope in 'Machine','User') { + $raw = [System.Environment]::GetEnvironmentVariable('Path', $scope) + if ([string]::IsNullOrWhiteSpace($raw)) { continue } + $expanded = [System.Environment]::ExpandEnvironmentVariables($raw) -split ';' + try { + [System.Environment]::SetEnvironmentVariable('Path', ((& $scrub $expanded) -join ';'), $scope) + } catch { + Write-Host "::error::could not scrub the $scope PATH ($($_.Exception.Message)); the simulation would not survive Refresh-SessionPath" + exit 1 + } + } foreach ($v in 'VSINSTALLDIR','VCINSTALLDIR','WindowsSdkDir','INCLUDE','LIB','LIBPATH') { "$v=" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 } @@ -492,34 +574,64 @@ jobs: $f = Get-Command $t -ErrorAction SilentlyContinue Write-Host ("{0,-8} {1}" -f $t, $(if ($f) { $f.Source } else { 'ABSENT' })) } - if ('${{ matrix.winget }}' -eq 'masked' -and (Get-Command winget -ErrorAction SilentlyContinue)) { - Write-Host '::error::winget still resolvable; masking failed' + $winget = Get-Command winget -ErrorAction SilentlyContinue + Write-Host ("winget {0}" -f $(if ($winget) { $winget.Source } else { 'ABSENT' })) + if ('${{ matrix.winget }}' -eq 'masked') { + if ($winget) { + Write-Host '::error::winget still resolvable; masking failed' + exit 1 + } + } elseif ('${{ matrix.os }}' -eq 'windows-latest' -and -not $winget) { + # Without this the visible leg quietly degrades into a second masked leg and + # the normal winget bootstrap is never exercised by any job in this workflow. + Write-Host '::error::winget is not resolvable on the visible leg; the winget bootstrap is not under test' exit 1 } + foreach ($scope in 'Machine','User') { + Write-Host ("{0} PATH after scrub: {1}" -f $scope, [System.Environment]::GetEnvironmentVariable('Path', $scope)) + } - name: Install id: install shell: pwsh run: | $ErrorActionPreference = 'Continue' - & ./install.ps1 -SkipTorch *>&1 | Tee-Object -FilePath logs/install.log + # No -SkipTorch: install.ps1 has no param block and its parser matches + # `--no-torch` only (install.ps1:112-142), so the token was silently dropped + # and every Windows leg installed torch anyway. Torch is exactly what the + # assert below needs, so ask for it explicitly rather than by accident. + & ./install.ps1 *>&1 | Tee-Object -FilePath logs/install.log $rc = $LASTEXITCODE Write-Host "installer exit code: $rc" exit $rc - - name: Assert torch loads (the VCRedist contract) + - name: Assert torch loads, and record what that does and does not prove if: steps.install.outcome == 'success' shell: pwsh run: | - # The prebuilt llama-server and PyTorch both link the VC++ runtime. - # Ensure-VCRedist only runs when winget exists, so on the masked leg this - # is the assertion that catches a silently broken install. + # HONESTY NOTE: the hosted image ships the VC++ 2015-2022 runtime in System32 + # and it cannot be removed without breaking the runner, so a successful + # `import torch` here does NOT prove that a genuinely clean no-winget machine + # would have the runtime -- Test-VCRedistInstalled (studio/setup.ps1:875) + # finds the preinstalled DLL and Ensure-VCRedist (setup.ps1:891) short-circuits + # before it ever needs winget. Record that, then assert what CAN fail here: + # torch imports, and the masked leg really did take the no-winget path. + $sys32 = Join-Path $env:WINDIR 'System32\vcruntime140_1.dll' + Write-Host "preinstalled System32 vcruntime140_1.dll: $(Test-Path $sys32)" $py = Join-Path $env:UNSLOTH_STUDIO_HOME 'unsloth_studio\Scripts\python.exe' if (-not (Test-Path $py)) { $py = (Get-Command python -ErrorAction SilentlyContinue).Source } if (-not $py) { Write-Host '::error::no python from the install'; exit 1 } & $py -c "import ctypes.util, sys; print('VCRUNTIME140:', ctypes.util.find_library('vcruntime140'))" & $py -c "import torch; print('torch', torch.__version__)" if ($LASTEXITCODE -ne 0) { Write-Host '::error::torch failed to import (VC++ runtime missing?)'; exit 1 } + if ('${{ matrix.winget }}' -eq 'masked') { + # install.ps1:1098, the no-winget branch of the winget check. + $noWinget = 'will require Python + uv to be already installed' + if (-not (Select-String -Path logs/install.log -Pattern $noWinget -SimpleMatch -Quiet)) { + Write-Host '::error::masked leg never reported winget as unavailable; it did not take the no-winget path' + exit 1 + } + } - name: Upload logs if: always() diff --git a/.github/workflows/desktop-app-clean-machine-ci.yml b/.github/workflows/desktop-app-clean-machine-ci.yml index 246e423dcc..190dd3bdf9 100644 --- a/.github/workflows/desktop-app-clean-machine-ci.yml +++ b/.github/workflows/desktop-app-clean-machine-ci.yml @@ -59,7 +59,10 @@ permissions: env: REL_REPO: ${{ inputs.release_repo || 'unsloth-test/unsloth-test' }} - REL_TAG: ${{ inputs.release_tag || 'desktop-v0.1.50-beta' }} + # Empty unless dispatched. A pinned tag is an immutable fixture, so a nightly + # against it can never catch a newly published broken bundle; each download step + # resolves the newest desktop-v* release when this is empty. + REL_TAG: ${{ inputs.release_tag || '' }} UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home UNSLOTH_STUDIO_DISABLE_PUBLIC_CHECK: '1' @@ -88,12 +91,28 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | mkdir -p dl logs + # Desktop releases are prereleases (release-desktop.yml keeps them off + # repo-wide "latest"), so resolve the newest desktop-v* tag explicitly. + if [ -z "$REL_TAG" ]; then + REL_TAG="$(gh release list --repo "$REL_REPO" --limit 100 --exclude-drafts \ + --json tagName,createdAt \ + --jq '[.[] | select(.tagName | startswith("desktop-v"))] + | sort_by(.createdAt) | reverse | .[0].tagName // empty')" + [ -n "$REL_TAG" ] || { echo "::error::no desktop-v* release in $REL_REPO"; exit 1; } + echo "resolved release tag: $REL_TAG" + echo "REL_TAG=$REL_TAG" >> "$GITHUB_ENV" + fi gh release download "$REL_TAG" --repo "$REL_REPO" \ --pattern '*aarch64.dmg' --dir dl ls -la dl - name: Strip the developer toolchain - if: ${{ inputs.strip_toolchain != false }} + # `inputs` exists only for workflow_dispatch, so on pull_request and + # schedule `inputs.strip_toolchain` is the empty string -- and loose + # equality coerces both '' and false to 0, making `!= false` FALSE. The + # automatic runs would keep the hosted toolchain, which is the one thing + # this workflow exists to remove. Gate on the event instead. + if: ${{ github.event_name != 'workflow_dispatch' || inputs.strip_toolchain }} run: | bash .github/scripts/clean-machine-env.sh mask --remove set -a; . ./clean-machine.env; set +a @@ -124,10 +143,34 @@ jobs: codesign -dv --verbose=2 "$APP" 2>&1 | head -20 || true spctl -a -vvv -t install "$APP" 2>&1 | head -5 || \ echo "::warning::Gatekeeper assessment failed -- users see 'cannot be opened' unless notarised" - # The bundled installer is what actually failed for users. - test -f "$APP/Contents/Resources/install.sh" \ - && echo "bundled install.sh present" \ - || echo "::error::no bundled install.sh in the app" + # The bundled installer is what actually failed for users. `::error::` is + # only an annotation and `echo` exits 0, so the old `|| echo` form let a + # bundle with no installer pass this step. + if [ -f "$APP/Contents/Resources/install.sh" ]; then + echo "bundled install.sh present" + else + echo "::error::no bundled install.sh in the app" + exit 1 + fi + + - name: Run the bundled installer, the path first launch takes + run: | + set -a; [ -f ./clean-machine.env ] && . ./clean-machine.env; set +a + set -o pipefail + APP="$(ls -d /Applications/*Unsloth*.app | head -1)" + # A headless runner never clicks Install: preflight sets `not_installed` + # and returns (studio/frontend/src/hooks/use-tauri-backend.ts:252-254) and + # startup-screen.tsx:388-389 waits for the button. Launching alone would + # therefore sit on that screen for 90s and pass without ever running the + # bundled installer. Invoke it the way studio/src-tauri/src/install.rs + # does: --tauri, stdin closed, no tty. --tauri rejects a custom studio + # home (install.sh:102-114), so drop the workspace-scoped override. + env -u UNSLOTH_STUDIO_HOME \ + bash "$APP/Contents/Resources/install.sh" --tauri --no-torch \ + < /dev/null 2>&1 | tee logs/bundled-install.log + PY="$HOME/.unsloth/studio/unsloth_studio/bin/python" + [ -x "$PY" ] || { echo "::error::bundled installer left no venv at $PY"; exit 1; } + "$PY" -V - name: Launch and prove it stays up run: | @@ -199,6 +242,17 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | mkdir -p dl logs + # Desktop releases are prereleases (release-desktop.yml keeps them off + # repo-wide "latest"), so resolve the newest desktop-v* tag explicitly. + if [ -z "$REL_TAG" ]; then + REL_TAG="$(gh release list --repo "$REL_REPO" --limit 100 --exclude-drafts \ + --json tagName,createdAt \ + --jq '[.[] | select(.tagName | startswith("desktop-v"))] + | sort_by(.createdAt) | reverse | .[0].tagName // empty')" + [ -n "$REL_TAG" ] || { echo "::error::no desktop-v* release in $REL_REPO"; exit 1; } + echo "resolved release tag: $REL_TAG" + echo "REL_TAG=$REL_TAG" >> "$GITHUB_ENV" + fi pat='*.deb'; [ "${{ matrix.kind }}" = "appimage" ] && pat='*.AppImage' gh release download "$REL_TAG" --repo "$REL_REPO" --pattern "$pat" --dir dl ls -la dl @@ -284,11 +338,27 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | mkdir -p dl logs + # Desktop releases are prereleases (release-desktop.yml keeps them off + # repo-wide "latest"), so resolve the newest desktop-v* tag explicitly. + if [ -z "$REL_TAG" ]; then + REL_TAG="$(gh release list --repo "$REL_REPO" --limit 100 --exclude-drafts \ + --json tagName,createdAt \ + --jq '[.[] | select(.tagName | startswith("desktop-v"))] + | sort_by(.createdAt) | reverse | .[0].tagName // empty')" + [ -n "$REL_TAG" ] || { echo "::error::no desktop-v* release in $REL_REPO"; exit 1; } + echo "resolved release tag: $REL_TAG" + echo "REL_TAG=$REL_TAG" >> "$GITHUB_ENV" + fi gh release download "$REL_TAG" --repo "$REL_REPO" --pattern '*setup.exe' --dir dl ls -la dl - name: Strip developer tooling from PATH - if: ${{ inputs.strip_toolchain != false }} + # `inputs` exists only for workflow_dispatch, so on pull_request and + # schedule `inputs.strip_toolchain` is the empty string -- and loose + # equality coerces both '' and false to 0, making `!= false` FALSE. The + # automatic runs would keep the hosted toolchain, which is the one thing + # this workflow exists to remove. Gate on the event instead. + if: ${{ github.event_name != 'workflow_dispatch' || inputs.strip_toolchain }} shell: pwsh run: | $drop = @('hostedtoolcache\windows\Python', 'WindowsApps', '\Git\', 'CMake', From eea0433f0e6e8551d3d75f2a016223f790340d6d Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 15:13:48 +0000 Subject: [PATCH 04/36] Make the Windows and Linux clean-machine legs honest The Windows scrub only touched PATH, so the legs were green while not clean: run 30365014702 logged "python ABSENT" and then "Python 3.13 already installed" with uv resolving C:\hostedtoolcache\windows\Python\3.13.14\arm64\python.exe. py.exe lives in C:\Windows and uv discovers interpreters itself, so take the toolcache off disk and fail when tooling survives, instead of only printing it. The Linux desktop legs never stripped anything, and the tauri.log step was all || true so it could not fail. Run the bundled installer the way install.rs does, with --tauri alone, and assert torch: passing --no-torch skipped the slowest half of first launch and let the venv check pass over it. Pin the WSL rootfs to a dated build; current/ is a rolling alias and the digest next to it is fixed. --- .../workflows/clean-machine-install-ci.yml | 26 +++++++++++++++++-- .../desktop-app-clean-machine-ci.yml | 12 ++++++++- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index 4afc55c45e..4012276f42 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -404,7 +404,7 @@ jobs: run: | # WSL2 is present on windows-2022+ runner images; only a distro is missing. wsl --set-default-version 2 - $url = 'https://cloud-images.ubuntu.com/wsl/releases/24.04/current/ubuntu-noble-wsl-amd64-24.04lts.rootfs.tar.gz' + $url = 'https://cloud-images.ubuntu.com/wsl/releases/24.04/20240423/ubuntu-noble-wsl-amd64-24.04lts.rootfs.tar.gz' $expected = '2a790896740b14d637dbdc583cce1ba081ac53b9e9cdb46dc09a2f73abbd9934' New-Item -ItemType Directory -Force -Path wsl-dist, logs | Out-Null Invoke-WebRequest -Uri $url -OutFile wsl-dist/rootfs.tar.gz -UseBasicParsing -TimeoutSec 900 @@ -542,6 +542,17 @@ jobs: -Value "@`"$($wingetCmd.Source)`" %*" $kept = @($shim) + $kept } + # Take the toolcache Python off disk, not just off PATH. py.exe lives in + # C:\Windows (which must stay) and uv does its own interpreter discovery, so + # both reach the toolcache no matter what PATH says -- which is how a leg + # printing `python ABSENT` still installed with the runner's 3.13.14. + foreach ($tc in @("$env:AGENT_TOOLSDIRECTORY\Python", 'C:\hostedtoolcache\windows\Python')) { + if ($tc -and (Test-Path $tc)) { + try { Rename-Item -LiteralPath $tc -NewName 'Python.masked' -ErrorAction Stop + Write-Host "masked toolcache python: $tc" } + catch { Write-Host "::error::could not mask $tc ($($_.Exception.Message)); the leg would not be clean"; exit 1 } + } + } $newPath = ($kept -join ';') "PATH=$newPath" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 # install.ps1 calls Refresh-SessionPath (defined install.ps1:318-337, called at @@ -570,9 +581,20 @@ jobs: - name: Verify the simulation took effect shell: pwsh run: | - foreach ($t in 'python','git','cmake','cl') { + $leaked = @() + # `py` too: the launcher lives in C:\Windows, which the scrub keeps, and it + # finds the toolcache Python the scrub just removed from PATH. + foreach ($t in 'python','py','git','cmake','cl') { $f = Get-Command $t -ErrorAction SilentlyContinue Write-Host ("{0,-8} {1}" -f $t, $(if ($f) { $f.Source } else { 'ABSENT' })) + if ($f -and $t -ne 'py') { $leaked += "$t -> $($f.Source)" } + } + # Printing alone could not fail, and the leg was green while not clean: + # run 30365014702 logged `python ABSENT` and then `Python 3.13 already + # installed` / `Using CPython ... C:\hostedtoolcache\windows\Python\...`. + if ($leaked) { + Write-Host "::error::developer tooling survived the scrub: $($leaked -join '; ')" + exit 1 } $winget = Get-Command winget -ErrorAction SilentlyContinue Write-Host ("winget {0}" -f $(if ($winget) { $winget.Source } else { 'ABSENT' })) diff --git a/.github/workflows/desktop-app-clean-machine-ci.yml b/.github/workflows/desktop-app-clean-machine-ci.yml index 190dd3bdf9..aaa9c3a19a 100644 --- a/.github/workflows/desktop-app-clean-machine-ci.yml +++ b/.github/workflows/desktop-app-clean-machine-ci.yml @@ -166,11 +166,15 @@ jobs: # does: --tauri, stdin closed, no tty. --tauri rejects a custom studio # home (install.sh:102-114), so drop the workspace-scoped override. env -u UNSLOTH_STUDIO_HOME \ - bash "$APP/Contents/Resources/install.sh" --tauri --no-torch \ + bash "$APP/Contents/Resources/install.sh" --tauri \ < /dev/null 2>&1 | tee logs/bundled-install.log PY="$HOME/.unsloth/studio/unsloth_studio/bin/python" [ -x "$PY" ] || { echo "::error::bundled installer left no venv at $PY"; exit 1; } "$PY" -V + # install.rs passes only --tauri, so torch is part of first launch. Dropping + # --no-torch here and asserting torch keeps the venv check from passing over + # a bundle whose only failure is the torch install. + "$PY" -c "import torch; print('torch', torch.__version__)" - name: Launch and prove it stays up run: | @@ -204,7 +208,13 @@ jobs: tail -60 "$f" # The two fields the bug report turned on. grep -E "disposition=|can_auto_repair=|Xcode Command Line|ModuleNotFoundError" "$f" || true + found=1 done + # Everything above is `|| true`, so on its own this step could not fail while + # the header sells the tauri.log disposition as an acceptance criterion. + # setup_logging (src-tauri/src/main.rs:50-67) opens tauri.log unconditionally + # at process start, so no log at all means the binary never got that far. + [ "${found:-0}" = "1" ] || { echo "::error::the app wrote no tauri.log; it never reached setup_logging"; exit 1; } - name: Restore the runner if: always() From 90ec9462a096bb51fd92419fc715e88aa9402ae5 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 19:19:48 +0000 Subject: [PATCH 05/36] Give the Linux and WSL legs an assertion that can fail The Linux rows' only post-install gate was nobuild, a log grep, so an installer exiting 0 having produced nothing kept a required leg green. The WSL job and the Windows job both already check the install runs; the Linux job now does too. The WSL detection half only printed its Select-String, and the alternation also matches "platform linux", so a regression that skipped every WSL-specific branch would still pass as a plain-Linux install. Assert the exact marker, stripping ANSI first since step writes the label in reverse video. Probed against three fixtures: real wsl log passes, platform linux fails, missing log fails. --- .../workflows/clean-machine-install-ci.yml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index 4012276f42..0d568431db 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -359,6 +359,17 @@ jobs: exit 1 fi + # nobuild only reads the log, so an installer that exits 0 having done nothing + # satisfies it. These are the required Linux rows and, unlike the WSL and + # Windows jobs, they had no check that the install produced anything runnable. + - name: Assert the install is actually usable + if: steps.install_root.outcome == 'success' + run: | + VENV="$UNSLOTH_STUDIO_HOME/unsloth_studio" + [ -x "$VENV/bin/python" ] || { echo "::error::installer exited 0 but left no managed Python at $VENV/bin/python"; ls -la "$UNSLOTH_STUDIO_HOME" || true; exit 1; } + "$VENV/bin/python" -V + [ -x "$VENV/bin/unsloth" ] || { echo "::error::installer exited 0 but left no unsloth CLI at $VENV/bin/unsloth"; exit 1; } + - name: Assert no source build if: always() run: | @@ -444,6 +455,22 @@ jobs: # The platform line proves the wsl branch was taken rather than plain linux. Select-String -Path logs/wsl-install.log -Pattern 'platform|\[TAURI:DIAG\]|wsl' -ErrorAction SilentlyContinue | Select-Object -First 10 + # Printing could not fail, and that alternation also matches `platform linux`. + # If detection regresses, every WSL-specific branch is skipped and this job + # still passes as a plain-Linux install, which is the one thing no other job + # covers. `step` writes the label in reverse video, so strip ANSI first or an + # anchored match can never hit. + $esc = [char]27 + $platformLines = @( + Get-Content logs/wsl-install.log -ErrorAction SilentlyContinue | + ForEach-Object { $_ -replace "$esc\[[0-9;]*[A-Za-z]", '' } | + Where-Object { $_ -match '^\s*platform\s+\S' } + ) + $platformLines | ForEach-Object { Write-Host "platform line: $_" } + if (-not ($platformLines | Where-Object { $_ -match '^\s*platform\s+wsl\s*$' })) { + Write-Host '::error::installer never reported ''platform wsl''; the WSL branch was not exercised' + exit 1 + } # No `|| echo`: substituting a message for the missing CLI made the inner # shell -- and so this step, and so the job -- succeed even when the install # produced nothing usable, which is the half of the question this step asks. From 231fcc3cf0c063db4b2064ea99feafed4aabee39 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 19:35:50 +0000 Subject: [PATCH 06/36] Tighten the clean-machine comments Compress the comment blocks across the clean-machine workflows and scripts. The explanations of why each check is written the way it is stay; the padding, restatement and duplication go. No code or workflow logic changes. --- .github/scripts/clean-machine-assert.sh | 68 +++-- .github/scripts/clean-machine-env.sh | 68 +++-- .../workflows/clean-machine-install-ci.yml | 232 +++++++++--------- .../desktop-app-clean-machine-ci.yml | 109 ++++---- 4 files changed, 224 insertions(+), 253 deletions(-) diff --git a/.github/scripts/clean-machine-assert.sh b/.github/scripts/clean-machine-assert.sh index 686c067324..9a81ba2186 100755 --- a/.github/scripts/clean-machine-assert.sh +++ b/.github/scripts/clean-machine-assert.sh @@ -4,15 +4,14 @@ # # Assert the clean-machine contract after an install attempt. # -# absent The toolchain really was absent for the whole run. Guards against a -# leg that "passed" only because masking silently failed, or because -# the installer quietly installed Xcode CLT behind our back. +# absent The toolchain really was absent for the whole run. Guards against a leg +# that "passed" only because masking silently failed, or because the +# installer quietly installed Xcode CLT behind our back. # notools The trace recorded no compiler/git/brew invocation (trace mode). -# nobuild The install log shows no source build (no sdist, no cmake, no -# "Building wheel" from pip and no "Building ==" from uv). -# This is the wheels-only contract. It needs UNSLOTH_VERBOSE=1 on the -# installer, otherwise run_install_cmd (install.sh:193-243) throws the -# uv output away on success and there is nothing here to read. +# nobuild The wheels-only contract: no "Building wheel" from pip, no +# "Building ==" from uv. Needs UNSLOTH_VERBOSE=1, else +# run_install_cmd (install.sh:193-243) discards the uv output on success +# and there is nothing here to read. # # Usage: bash .github/scripts/clean-machine-assert.sh absent notools nobuild set -uo pipefail @@ -28,11 +27,9 @@ for check in "$@"; do case "$check" in absent) - # Deliberately NOT a `command -v` check. On a real virgin Mac /usr/bin/git and - # /usr/bin/cc EXIST as Xcode CLT stubs, so `command -v git` SUCCEEDS -- running - # it is what fails ("xcrun: error: invalid active developer path"). Asserting on - # `command -v` would therefore be unfaithful and would fail on a correctly masked - # runner. The honest invariant is: the tool must not WORK. + # Deliberately NOT `command -v`: on a virgin Mac /usr/bin/{git,cc} EXIST as CLT + # stubs, so `command -v` succeeds and only RUNNING them fails ("xcrun: error: + # invalid active developer path"). The honest invariant is: must not WORK. if xcode-select -p >/dev/null 2>&1; then fail "xcode-select -p still resolves to $(xcode-select -p 2>/dev/null); not a clean Mac" else @@ -41,10 +38,10 @@ for check in "$@"; do for tool in git cc clang cmake; do command -v "$tool" >/dev/null 2>&1 || { ok "$tool not on PATH"; continue; } if "$tool" --version >/dev/null 2>&1; then - # On Intel runners /usr/bin/git keeps working once the CLT are gone, so it - # is not CLT-provided there and no masking can remove it. cc and clang do - # become stubs, and the consumer path needs no git on macOS, so report it - # rather than calling the simulation broken. + # On Intel runners /usr/bin/git is not CLT-provided and keeps working once + # the CLT are gone, so no masking can remove it. cc and clang do become + # stubs and the macOS consumer path needs no git, so report rather than + # call the simulation broken. case " ${UNSLOTH_CLEAN_ALLOW_WORKING:-} " in *" $tool "*) echo "[assert] NOTE $tool still works ($(command -v "$tool")); allowed on this runner" @@ -68,19 +65,18 @@ for check in "$@"; do if [ -z "$TRACE" ] || [ ! -f "$TRACE" ]; then fail "notools requested but no trace file (\$UNSLOTH_TOOL_TRACE=$TRACE)" else - # git is legitimate under --local (it installs unsloth-zoo from a git URL); - # UNSLOTH_ALLOW_TOOLS lets that leg allow-list it explicitly. + # git is legitimate under --local (unsloth-zoo comes from a git URL), so that + # leg allow-lists it via UNSLOTH_ALLOW_TOOLS. allow="${UNSLOTH_ALLOW_TOOLS:-}" hits="" while IFS=$'\t' read -r tool rest; do [ -n "$tool" ] || continue case " $allow " in *" $tool "*) continue ;; esac - # `xcode-select -p` ASKS whether a toolchain is selected; it cannot build - # anything. The installer has to ask in order to tell the user whether a - # source build is available, and the whole point of the fix is that it then - # carries on without one. Treating the question as toolchain USE would fail - # the very leg that proves the toolchain was never used. `--install`, which - # pops the CLT installer, stays a hit. + # `xcode-select -p` only ASKS whether a toolchain is selected; the installer + # has to ask, and the point of the fix is that it carries on without one. + # Counting the question as toolchain USE would fail the very leg that proves + # the toolchain was never used. `--install`, which pops the CLT installer, + # stays a hit. if [ "$tool" = "xcode-select" ]; then case "$rest" in -p|--print-path|-v|--version|"") continue ;; @@ -98,25 +94,21 @@ for check in "$@"; do ;; nobuild) - # "Built an sdist" is NOT the same as "needed a compiler". Four packages on the - # macOS path are sdist-only PURE PYTHON projects that build fine with no - # toolchain (verified by resolving each against cp313/macos-arm64): + # "Built an sdist" is NOT "needed a compiler". Four packages on the macOS path + # are sdist-only PURE PYTHON (verified against cp313/macos-arm64): # openai-whisper, argbind, randomname -- no version ever ships a wheel # antlr4-python3-runtime==4.9.3 -- pinned below the 4.13.2 wheel - # Failing on those would be a false alarm, so the contract asserted here is - # "nothing that needs a COMPILER was built", with that allowlist subtracted. - # UNSLOTH_ALLOW_SDIST can extend it. + # Failing on those is a false alarm, so the contract is "nothing needing a + # COMPILER was built". UNSLOTH_ALLOW_SDIST extends the allowlist. _allow="openai-whisper argbind randomname antlr4-python3-runtime ${UNSLOTH_ALLOW_SDIST:-}" if [ ! -f "$LOG" ]; then fail "nobuild requested but $LOG is missing" else - # The installer runs `uv pip install`, and uv does NOT use pip's phrasing. - # It prints ` Building ==` and ` Built ==` - # to stderr, as plain lines once stderr is not a TTY (astral-sh/uv#11165), so - # the pip-only pattern left _built empty on every uv source build. Match both - # spellings. Requiring `==` or ` @ ` after the name keeps this off the - # installer's own lowercase "building frontend..." progress text. Strip ANSI - # first so a coloured run (FORCE_COLOR) still parses. + # uv does NOT use pip's phrasing: it prints `Building ==` to + # stderr (astral-sh/uv#11165), so the pip-only pattern left _built empty on + # every uv source build. Match both spellings. Requiring `==` or ` @ ` after + # the name keeps this off the installer's own lowercase "building frontend..." + # progress text. Strip ANSI first so a coloured run (FORCE_COLOR) parses. _esc=$(printf '\033') _built="$(sed -E "s/${_esc}\[[0-9;]*[A-Za-z]//g" "$LOG" 2>/dev/null \ | grep -oiE "building wheel for [a-z0-9._-]+|building [a-z0-9._-]+(==| @ )" \ diff --git a/.github/scripts/clean-machine-env.sh b/.github/scripts/clean-machine-env.sh index c1e838d35a..1ed51bbac9 100755 --- a/.github/scripts/clean-machine-env.sh +++ b/.github/scripts/clean-machine-env.sh @@ -2,25 +2,20 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. # -# Simulate a virgin developer machine on a GitHub-hosted runner, so the installer -# is exercised the way a real user's brand-new Mac / PC exercises it. +# Simulate a virgin developer machine on a GitHub-hosted runner. Two modes, because +# "the tool is absent" and "the installer never called the tool" cannot be simulated +# by the same mechanism: # -# Two modes, because "the tool is absent" and "the installer never called the tool" -# CANNOT be simulated by the same mechanism: +# mask Make the toolchain genuinely ABSENT: scrub PATH to OS defaults and (with +# --remove) move the real toolchain aside, so `command -v git` correctly +# FAILS, as on a clean Mac. A failing "poison shim" would do the opposite -- +# `command -v` finds it and reports the tool as present -- so no shims here. +# trace Leave the toolchain working but route it through logging wrappers that log +# the call then exec the real binary, proving whether the installer ever +# REACHES for a compiler/git without changing behaviour. # -# mask Make the toolchain genuinely ABSENT. Scrubs PATH down to the OS -# defaults and (with --remove) moves the real toolchain aside. After -# this, `command -v git` correctly FAILS, which is what a clean Mac does. -# A failing "poison shim" on PATH would do the opposite -- `command -v` -# finds it and reports the tool as present -- so shims are NOT used here. -# -# trace Leave the toolchain working, but route it through logging wrappers that -# record the invocation and then exec the real binary. Proves whether the -# installer ever REACHES for a compiler/git, without changing behaviour. -# -# Writes shell exports to $CLEAN_ENV_FILE (default ./clean-machine.env) for the -# caller to `source`. Nothing is exported globally, so other workflow steps -# (checkout, upload-artifact) keep a normal environment. +# Writes shell exports to $CLEAN_ENV_FILE (default ./clean-machine.env) to `source`; +# nothing is exported globally, so other steps keep a normal environment. # # Usage: # bash .github/scripts/clean-machine-env.sh mask [--remove] @@ -55,8 +50,8 @@ TOOLS="xcode-select xcrun clang clang++ cc c++ gcc g++ git cmake make brew ninja note() { echo "[clean-machine] $*"; } # ── PATH scrub ──────────────────────────────────────────────────────────────── -# Keep only OS-default system dirs. Drops Homebrew, the hosted Python toolcache, -# setup-* shims, pipx, cargo, and every other preinstalled developer dir. +# Keep only OS-default system dirs: drops Homebrew, the hosted Python toolcache, +# setup-* shims, pipx, cargo and every other preinstalled developer dir. scrub_path() { local keep out="" if [ "$OS" = "Darwin" ]; then @@ -76,11 +71,9 @@ if [ "$MODE" = "mask" ]; then NEWPATH="$(scrub_path)" { echo "export PATH='$NEWPATH'" - # DEVELOPER_DIR must be UNSET, not pointed at a fake path: `xcode-select -p` - # honours DEVELOPER_DIR and prints it verbatim with exit 0, so setting it to a - # nonexistent dir makes the probe SUCCEED -- the exact opposite of a clean Mac, - # where DEVELOPER_DIR is unset and the missing /var/db/xcode_select_link is what - # makes `xcode-select -p` fail. + # UNSET, not a fake path: `xcode-select -p` honours DEVELOPER_DIR and prints it + # verbatim with exit 0, so a nonexistent dir makes the probe SUCCEED. On a clean + # Mac it is unset and the missing xcode_select_link is what makes the probe fail. echo "unset DEVELOPER_DIR || true" echo "unset SDKROOT CC CXX CFLAGS CXXFLAGS LDFLAGS CMAKE_GENERATOR CMAKE_PREFIX_PATH || true" echo "export HOMEBREW_NO_AUTO_UPDATE=1" @@ -88,11 +81,10 @@ if [ "$MODE" = "mask" ]; then } >> "$ENV_FILE" if [ "$REMOVE" = "1" ] && [ "$OS" = "Darwin" ]; then - # Best-effort real removal. Each step is independent and recorded in - # restore.sh so an `if: always()` step can put the runner back. - # /var/db/xcode_select_link is exactly what `xcode-select -p` reads, so - # removing it reproduces a virgin Mac's gate precisely. `xcode-select --reset` - # is NOT enough: it can reselect a full Xcode.app. + # Best-effort real removal; each step is independent and recorded in restore.sh + # so an `if: always()` step can put the runner back. xcode_select_link is exactly + # what `xcode-select -p` reads, so removing it reproduces a virgin Mac's gate. + # `xcode-select --reset` is NOT enough: it can reselect a full Xcode.app. if [ -e /var/db/xcode_select_link ]; then if sudo rm -f /var/db/xcode_select_link 2>/dev/null; then note "removed /var/db/xcode_select_link" @@ -101,8 +93,8 @@ if [ "$MODE" = "mask" ]; then note "WARN could not remove /var/db/xcode_select_link" fi fi - # Moving the CLT dir aside turns /usr/bin/{cc,clang,git} into dead shims, so - # the run also proves the install needs no compiler at all. + # Moving the CLT dir aside turns /usr/bin/{cc,clang,git} into dead shims, so the + # run also proves the install needs no compiler at all. if [ -d /Library/Developer/CommandLineTools ]; then if sudo mv /Library/Developer/CommandLineTools /Library/Developer/CommandLineTools.masked 2>/dev/null; then note "moved CommandLineTools aside" @@ -111,11 +103,11 @@ if [ "$MODE" = "mask" ]; then note "WARN could not move CommandLineTools" fi fi - # Xcode.app must go too. With the select link removed AND CommandLineTools moved, - # `xcode-select -p` does not fail -- it falls through to whatever Xcode bundle the - # runner image ships (observed: /Applications/Xcode_16.4.app/Contents/Developer), - # which re-arms /usr/bin/git and /usr/bin/cc and silently un-cleans the machine. - # A rename is instant regardless of bundle size: same filesystem, no copy. + # Xcode.app must go too: with the link removed AND CommandLineTools moved, + # `xcode-select -p` still does not fail, it falls through to the image's Xcode + # bundle (observed: /Applications/Xcode_16.4.app/Contents/Developer), which + # re-arms /usr/bin/{git,cc} and silently un-cleans the machine. A rename is + # instant regardless of bundle size: same filesystem, no copy. for app in /Applications/Xcode*.app; do [ -d "$app" ] || continue if sudo mv "$app" "${app}.masked" 2>/dev/null; then @@ -143,8 +135,8 @@ if [ "$MODE" = "trace" ]; then for tool in $TOOLS; do real="$(command -v "$tool" 2>/dev/null || true)" [ -n "$real" ] || continue - # Wrapper logs the call then execs the REAL binary, so behaviour is unchanged - # and the trace answers "did the installer reach for this?" honestly. + # Logs the call then execs the REAL binary: behaviour unchanged, so the trace + # answers "did the installer reach for this?" honestly. cat > "$BIN/$tool" <> "$TRACE" diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index 0d568431db..bc78ae040d 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -4,14 +4,12 @@ # Proves Unsloth installs on a machine that has never seen a developer toolchain. # # Why this exists: studio-mac-install-matrix.yml runs `install.sh --local --no-torch` -# on runners that already have Xcode CLT selected AND actions/setup-python -# preinstalled, so the macOS dependency gate in install.sh never fires there -- and -# `--local` is precisely the mode that legitimately needs git. A brand-new Mac -# therefore hits a hard `exit 1` that no CI job covered. +# on runners with Xcode CLT selected AND actions/setup-python preinstalled, so the +# macOS dependency gate never fires there -- and `--local` is precisely the mode that +# legitimately needs git. A brand-new Mac hits a hard `exit 1` no CI job covered. # -# Hosted runners are developer machines, so each job simulates absence rather than -# being virgin. Two modes, because they answer different questions and cannot be -# done by the same mechanism (see .github/scripts/clean-machine-env.sh): +# Hosted runners are developer machines, so each job simulates absence. Two modes, +# answering different questions (see .github/scripts/clean-machine-env.sh): # mask -> the toolchain is genuinely unusable; does the install still work? # trace -> the toolchain works but is logged; does the installer ever call it? # Linux is the exception: containers are genuinely clean. @@ -55,8 +53,8 @@ env: # No wildcard bind -> no ifconfig.me / check-host.net calls on the startup path. UNSLOTH_STUDIO_DISABLE_PUBLIC_CHECK: '1' # Without this, run_install_cmd (install.sh:193-243) sends every `uv pip install` - # to a temp file and DELETES it on success, so logs/install.log holds no uv output - # and the `nobuild` assertion can only ever report "built: none". + # to a temp file and DELETES it on success, so the `nobuild` assertion can only + # ever report "built: none". UNSLOTH_VERBOSE: '1' jobs: @@ -66,14 +64,14 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 40 continue-on-error: ${{ matrix.experimental }} - # Explicit legs rather than a full cross-product: the interesting dimensions are + # Explicit legs, not a full cross-product: the interesting dimensions are # (does the toolchain exist) x (how the script is delivered), not every pairing. strategy: fail-fast: false matrix: include: - # The reported failure, in the shape users run it. Default install (torch - # included) because that is what a consumer actually gets. + # The reported failure, in the shape users run it. Default install (with + # torch) because that is what a consumer actually gets. - {os: macos-14, mode: mask, delivery: pipe, flags: '', experimental: false} - {os: macos-14, mode: mask, delivery: file, flags: '', experimental: false} # What the desktop app runs: no tty, stdin closed, TAURI markers on. @@ -82,9 +80,8 @@ jobs: - {os: macos-14, mode: trace, delivery: file, flags: '', experimental: false} # --no-torch is the one macOS path that can still want a compiler # (sentencepiece has no guaranteed cp313 arm64 wheel), so probe it apart - # from the default path instead of letting it mask the gate under test. + # from the default path instead of letting it hide the gate under test. - {os: macos-14, mode: mask, delivery: file, flags: '--no-torch', experimental: true} - # OS-version dimension. - {os: macos-15, mode: mask, delivery: pipe, flags: '', experimental: false} - {os: macos-26, mode: mask, delivery: file, flags: '', experimental: true} # Intel pins python 3.12 and its /usr/bin/git is not CLT-provided, so it @@ -97,8 +94,8 @@ jobs: with: persist-credentials: false - # Deliberately no actions/setup-python: install.sh must bring its own - # uv-managed CPython, exactly as it must on a user's machine. + # No actions/setup-python on purpose: install.sh must bring its own uv-managed + # CPython, exactly as it must on a user's machine. - name: Record the pre-masking toolchain run: | @@ -139,18 +136,17 @@ jobs: FLAGS="${{ matrix.flags }}" case "${{ matrix.delivery }}" in file) - # Plain file execution: isolates "installer logic broken" from + # Plain file execution isolates "installer logic broken" from # "curl-pipe delivery broken". bash install.sh $FLAGS 2>&1 | tee logs/install.log || rc=$? ;; pipe) # The shape users actually run. install.sh is ~150KB of top-level - # statements, so an early `exit` leaves the writer with a closed - # pipe -> `curl: (56)`. Piping a local file reproduces that - # faithfully without depending on unsloth.ai being current. - # On pull_request/push this input is empty, which correctly falls through to - # the checked-out ref -- only an explicit dispatch tests unsloth.ai. - if [ "${{ inputs.installer_source }}" = "published" ]; then + # statements, so an early `exit` leaves the writer with a closed pipe + # -> `curl: (56)`. Piping a local file reproduces that faithfully + # without depending on unsloth.ai being current. This input is empty on + # pull_request/push, so only an explicit dispatch tests unsloth.ai. + if [ "${{ inputs.installer_source }}" = "published" ]; then curl -fsSL https://unsloth.ai/install.sh | sh -s -- $FLAGS 2>&1 | tee logs/install.log || rc=$? else # `sh -s --` with no further args would pass an empty positional, @@ -164,10 +160,10 @@ jobs: ;; tauri) # Exactly how the desktop app invokes it: no tty, stdin closed. - # --tauri rejects a custom UNSLOTH_STUDIO_HOME outright (the desktop app - # still uses the legacy ~/.unsloth/studio root), so the workspace-scoped - # value every other leg relies on has to go here or the installer exits - # before it does any work. The runner is ephemeral, so the real home is + # --tauri rejects a custom UNSLOTH_STUDIO_HOME outright (it still uses + # the legacy ~/.unsloth/studio root), so the workspace-scoped value + # every other leg relies on must be dropped or the installer exits + # before doing any work. The runner is ephemeral, so the real home is # as disposable as the override. env -u UNSLOTH_STUDIO_HOME \ bash install.sh --tauri $FLAGS < /dev/null 2>&1 | tee logs/install.log || rc=$? @@ -175,8 +171,8 @@ jobs: esac echo "install_rc=$rc" >> "$GITHUB_OUTPUT" echo "installer exit code: $rc" - # The pipe legs are the ones that expose curl:(56); surface it explicitly - # rather than leaving it buried in a 4000-line log. + # The pipe legs expose curl:(56); surface it rather than leaving it buried + # in a 4000-line log. if grep -qE "curl: \(5[36]\)|Failure writing output to destination" logs/install.log; then echo "::warning::curl reported a broken pipe -- an early exit killed the reader" fi @@ -194,9 +190,9 @@ jobs: if: steps.install.outcome == 'success' run: | set -a; . ./clean-machine.env; set +a - # The tauri leg cannot honour UNSLOTH_STUDIO_HOME (see the Install step), so - # it installed into the legacy root. llama.cpp sits at /llama.cpp and - # the venv at /studio, so this is ~/.unsloth, not ~/.unsloth/studio. + # The tauri leg cannot honour UNSLOTH_STUDIO_HOME (see Install), so it went + # to the legacy root: llama.cpp sits at /llama.cpp and the venv at + # /studio, so this is ~/.unsloth, not ~/.unsloth/studio. if [ "${{ matrix.delivery }}" = "tauri" ]; then HOME_DIR="$HOME/.unsloth" else @@ -212,8 +208,8 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - # The rows at lines 74 and 82 differ only in `flags`, so the flags have to - # be in the name: artifacts are immutable per run and the second upload 409s. + # Two matrix rows differ only in `flags`, so flags must be in the name: + # artifacts are immutable per run and the second upload 409s. name: clean-mac-${{ matrix.os }}-${{ matrix.mode }}-${{ matrix.delivery }}${{ matrix.flags && format('-{0}', matrix.flags) || '' }} path: | logs/ @@ -230,8 +226,8 @@ jobs: container: ${{ matrix.image }} timeout-minutes: 40 continue-on-error: ${{ matrix.experimental }} - # Container jobs default to `sh -e`, which is dash: `set -o pipefail` is an - # "Illegal option" there and kills the step before the installer even starts. + # Container jobs default to `sh -e` (dash), where `set -o pipefail` is an + # "Illegal option" that kills the step before the installer even starts. defaults: run: shell: bash @@ -250,7 +246,7 @@ jobs: runner: ubuntu-24.04-arm experimental: false # No elevation: today this hard-fails at install.sh:856-861. Expected - # failure -- the point is to pin the message and prove it is actionable + # failure; the point is to pin the message and prove it is actionable # rather than a bare `curl: (56)`. - label: ubuntu2404-nonroot image: ubuntu:24.04 @@ -270,9 +266,9 @@ jobs: printf '%-8s %s\n' "$t" "$(command -v $t 2>/dev/null || echo ABSENT)" done | tee /tmp/container-baseline.txt - # The advertised `curl | sh` cannot even start on an image without curl, so - # the bootstrap transport is provisioned separately from the installer's own - # dependencies. Everything else stays absent. + # The advertised `curl | sh` cannot even start on an image without curl, so the + # transport is provisioned apart from the installer's own dependencies. + # Everything else stays absent. - name: Provision only the bootstrap transport run: | if command -v apt-get >/dev/null 2>&1; then @@ -281,17 +277,16 @@ jobs: dnf install -y -q ca-certificates curl fi - # No actions/checkout here on purpose: it requires git, and a container with git - # preinstalled is not the clean machine under test. Fetch the two files we need - # over the transport provisioned above -- and fetch the INSTALLER from the same - # ref, so these legs can validate a fix instead of only the published script. + # No actions/checkout on purpose: it needs git, and a container with git + # preinstalled is not the clean machine under test. Fetch over the transport + # above, and fetch the INSTALLER from the same ref so these legs can validate a + # fix instead of only the published script. - name: Fetch installer + assert script for this ref run: | mkdir -p logs .github/scripts raw="https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/${GITHUB_SHA}" curl -fsSL "$raw/.github/scripts/clean-machine-assert.sh" -o .github/scripts/clean-machine-assert.sh - # On pull_request/push this input is empty, which correctly falls through to - # the checked-out ref -- only an explicit dispatch tests unsloth.ai. + # Empty on pull_request/push, so only an explicit dispatch tests unsloth.ai. if [ "${{ inputs.installer_source }}" = "published" ]; then curl -fsSL https://unsloth.ai/install.sh -o install.sh echo "installer: published (unsloth.ai)" @@ -305,12 +300,11 @@ jobs: if: matrix.label == 'ubuntu2404-nonroot' run: | useradd -m tester - # Switching user without a login shell keeps the caller's environment, so the - # workflow-wide UNSLOTH_STUDIO_HOME follows tester in -- and install.sh both - # resolves AND validates that override in _resolve_studio_destinations - # (install.sh:503-559), which runs long before the elevation gate at - # install.sh:840-861. Without a writable target this leg dies on - # "cannot be created" instead of on "cannot elevate". + # Switching user without a login shell keeps the caller's environment, so + # the workflow-wide UNSLOTH_STUDIO_HOME follows tester in, and install.sh + # validates that override in _resolve_studio_destinations (503-559), long + # before the elevation gate (840-861). Without a writable target this leg + # dies on "cannot be created" instead of on "cannot elevate". mkdir -p "$UNSLOTH_STUDIO_HOME" # No sudo installed and not root -> exercises the "cannot elevate" branch. chown -R tester logs install.sh "$UNSLOTH_STUDIO_HOME" @@ -321,8 +315,8 @@ jobs: run: | set -o pipefail rc=0 - # Piped, because that is the advertised command and the shape that turns an - # early exit into curl:(56). + # Piped: the advertised command, and the shape that turns an early exit + # into curl:(56). cat install.sh | sh 2>&1 | tee logs/install.log || rc=$? echo "installer exit code: $rc" exit "$rc" @@ -335,16 +329,16 @@ jobs: su tester -c 'cat install.sh | sh' > logs/install.log 2>&1 || rc=$? echo "installer exit code: $rc" tail -40 logs/install.log - # It may legitimately fail; what must NOT happen is an unexplained exit or a - # bare broken-pipe error standing in for a real diagnosis. + # It may legitimately fail; what must NOT happen is an unexplained exit or + # a bare broken-pipe error standing in for a real diagnosis. if [ "$rc" != "0" ] && ! grep -qiE "sudo is not available|apt-get install|missing:|permission" logs/install.log; then echo "::error::unprivileged install failed with no actionable message" exit 1 fi - # The nonroot leg checks that its expected failure is the expected one. This leg - # is continue-on-error too, so without the same check a bootstrap outage or an - # unrelated early exit is tolerated exactly like the intentional diagnostic. + # This leg is continue-on-error like the nonroot one, so without the same check + # a bootstrap outage or an unrelated early exit would be tolerated exactly like + # the intentional diagnostic. - name: Assert the Fedora failure is the unsupported-package-manager one if: always() && matrix.label == 'fedora41' run: | @@ -360,8 +354,8 @@ jobs: fi # nobuild only reads the log, so an installer that exits 0 having done nothing - # satisfies it. These are the required Linux rows and, unlike the WSL and - # Windows jobs, they had no check that the install produced anything runnable. + # satisfies it. These required Linux rows had no check that the install + # produced anything runnable, unlike the WSL and Windows jobs. - name: Assert the install is actually usable if: steps.install_root.outcome == 'success' run: | @@ -392,13 +386,13 @@ jobs: # ── WSL ─────────────────────────────────────────────────────────────────── # install.sh carries ~126 lines of WSL-specific logic (the `linux|wsl` dependency - # branch, UNSLOTH_WSL_REROUTED, the Strix Halo reroute to 24.04), and none of it - # had ever run in CI -- tests/sh/test_strixhalo_wsl_reroute.sh extracts functions - # and mocks the environment, which cannot catch anything about a real WSL. + # branch, UNSLOTH_WSL_REROUTED, the Strix Halo reroute to 24.04) that had never run + # in CI: tests/sh/test_strixhalo_wsl_reroute.sh extracts functions and mocks the + # environment, which cannot catch anything about a real WSL. # - # No third-party action: the official Ubuntu WSL rootfs plus `wsl --import` is - # deterministic, checksum-verifiable, and avoids adding a supply-chain dependency - # to a repo that audits its lockfiles. + # No third-party action: the official Ubuntu rootfs plus `wsl --import` is + # deterministic and checksum-verifiable, and adds no supply-chain dependency to a + # repo that audits its lockfiles. wsl: name: wsl ubuntu-24.04 runs-on: windows-latest @@ -426,25 +420,23 @@ jobs: } wsl --import unsloth-ci "$PWD/wsl-dist/instance" "$PWD/wsl-dist/rootfs.tar.gz" --version 2 wsl -d unsloth-ci -- uname -a - # A freshly imported rootfs is genuinely bare: no curl, no git, no compiler. + # A freshly imported rootfs is genuinely bare: no curl, git or compiler. # That is the clean machine, not a simulation of one. wsl -d unsloth-ci -- sh -c 'for t in curl wget git gcc cmake python3 sudo; do printf "%-8s %s\n" "$t" "$(command -v $t || echo ABSENT)"; done' - name: Install inside WSL, piped exactly as documented shell: pwsh run: | - # Only ca-certificates + curl, because the advertised one-liner cannot even - # start without a transport. Everything else must come from the installer. + # Only ca-certificates + curl: the advertised one-liner cannot start without + # a transport. Everything else must come from the installer. wsl -d unsloth-ci -u root -- sh -c 'apt-get update -qq && apt-get install -y -qq --no-install-recommends ca-certificates curl' 2>&1 | Tee-Object -FilePath logs/wsl-bootstrap.log # Copy the script in rather than reaching across /mnt/c: a DrvFs path brings - # Windows file permissions and CRLF risk with it, and neither is what a real - # WSL user's install looks like. + # Windows permissions and CRLF risk, neither of which a real WSL user has. $wslPath = (wsl -d unsloth-ci -- wslpath -a "$($env:GITHUB_WORKSPACE -replace '\\','/')/install.sh").Trim() Write-Host "installer source in WSL: $wslPath" wsl -d unsloth-ci -u root -- cp "$wslPath" /root/install.sh - # Feed it through a pipe: same shape as `curl ... | sh`, so an early exit - # still exposes the broken-pipe problem, but the script under test is this - # ref rather than whatever production currently serves. + # Piped, same shape as `curl ... | sh`, so an early exit still exposes the + # broken pipe, but the script under test is this ref not production's. wsl -d unsloth-ci -u root -- sh -c 'cd /root && cat install.sh | sh' 2>&1 | Tee-Object -FilePath logs/wsl-install.log Write-Host "installer exit: $LASTEXITCODE" @@ -455,11 +447,11 @@ jobs: # The platform line proves the wsl branch was taken rather than plain linux. Select-String -Path logs/wsl-install.log -Pattern 'platform|\[TAURI:DIAG\]|wsl' -ErrorAction SilentlyContinue | Select-Object -First 10 - # Printing could not fail, and that alternation also matches `platform linux`. - # If detection regresses, every WSL-specific branch is skipped and this job - # still passes as a plain-Linux install, which is the one thing no other job - # covers. `step` writes the label in reverse video, so strip ANSI first or an - # anchored match can never hit. + # Printing could not fail, and that alternation also matches + # `platform linux`: if detection regresses, every WSL branch is skipped and + # this job still passes as a plain-Linux install, the one thing no other job + # covers. `step` writes the label in reverse video, so strip ANSI first or + # an anchored match can never hit. $esc = [char]27 $platformLines = @( Get-Content logs/wsl-install.log -ErrorAction SilentlyContinue | @@ -472,8 +464,8 @@ jobs: exit 1 } # No `|| echo`: substituting a message for the missing CLI made the inner - # shell -- and so this step, and so the job -- succeed even when the install - # produced nothing usable, which is the half of the question this step asks. + # shell, this step and the job all succeed even when the install produced + # nothing usable, which is half of what this step asks. $verify = wsl -d unsloth-ci -u root -- sh -c 'set -e; test -x "$HOME/.unsloth/studio/unsloth_studio/bin/unsloth"; "$HOME/.unsloth/studio/unsloth_studio/bin/unsloth" --version' 2>&1 $verifyRc = $LASTEXITCODE $verify | Tee-Object -FilePath logs/wsl-verify.log @@ -509,10 +501,10 @@ jobs: - os: windows-latest winget: 'visible' experimental: false - # The no-winget path (LTSC / Server / managed corporate machines) falls - # back to python.org + astral.sh and is completely untested today. It is - # also the path where Ensure-VCRedist silently does not run, which leaves - # torch unable to load -- hence the explicit `import torch` assert below. + # The no-winget path (LTSC / Server / managed corporate machines) falls back + # to python.org + astral.sh and is untested today. It is also where + # Ensure-VCRedist silently does not run, leaving torch unable to load -- + # hence the explicit `import torch` assert below. - os: windows-latest winget: 'masked' experimental: false @@ -531,20 +523,20 @@ jobs: shell: pwsh run: | New-Item -ItemType Directory -Force -Path logs | Out-Null - # Drop preinstalled Python, git, CMake, VS/LLVM and the WindowsApps - # aliases from PATH. A full Visual Studio uninstall is not realistic in - # CI (registry + vswhere discovery, slow, may need a reboot), so PATH and - # env scrubbing is the honest approximation -- recorded as such. + # Drop preinstalled Python, git, CMake, VS/LLVM and the WindowsApps aliases + # from PATH. A full Visual Studio uninstall is not realistic in CI (registry + # + vswhere discovery, slow, may need a reboot), so PATH and env scrubbing + # is the honest approximation, recorded as such. $drop = @('hostedtoolcache\windows\Python', 'WindowsApps', '\Git\', 'CMake', 'Microsoft Visual Studio', 'BuildTools', 'LLVM', 'MSYS', 'mingw', 'Strawberry') - # winget ships as an app-execution alias inside ...\Local\Microsoft\WindowsApps, - # which the blanket drop above removes on EVERY leg -- so winget=visible was + # winget is an app-execution alias in ...\Local\Microsoft\WindowsApps, which + # the blanket drop above removes on EVERY leg -- so winget=visible was # silently running the same no-winget fallback as winget=masked. Resolve it # before the scrub and hand it back through a shim, so the visible leg gets - # winget without also getting the Store's python.exe alias back. - # windows-11-arm has no winget at all on the hosted image - # (actions/runner-images#14083), so only windows-latest can carry it. + # winget without the Store's python.exe alias back. windows-11-arm has no + # winget on the hosted image (actions/runner-images#14083), so only + # windows-latest can carry it. $wantWinget = ('${{ matrix.winget }}' -ne 'masked') -and ('${{ matrix.os }}' -eq 'windows-latest') $wingetCmd = Get-Command winget -ErrorAction SilentlyContinue $scrub = { @@ -569,10 +561,10 @@ jobs: -Value "@`"$($wingetCmd.Source)`" %*" $kept = @($shim) + $kept } - # Take the toolcache Python off disk, not just off PATH. py.exe lives in + # Take the toolcache Python off disk, not just off PATH: py.exe lives in # C:\Windows (which must stay) and uv does its own interpreter discovery, so - # both reach the toolcache no matter what PATH says -- which is how a leg - # printing `python ABSENT` still installed with the runner's 3.13.14. + # both reach the toolcache whatever PATH says -- which is how a leg printing + # `python ABSENT` still installed with the runner's 3.13.14. foreach ($tc in @("$env:AGENT_TOOLSDIRECTORY\Python", 'C:\hostedtoolcache\windows\Python')) { if ($tc -and (Test-Path $tc)) { try { Rename-Item -LiteralPath $tc -NewName 'Python.masked' -ErrorAction Stop @@ -582,13 +574,13 @@ jobs: } $newPath = ($kept -join ';') "PATH=$newPath" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - # install.ps1 calls Refresh-SessionPath (defined install.ps1:318-337, called at - # 1246/1278/1295/1360/1369/2797), which rebuilds $env:Path from the Machine and - # User registry values. Scrubbing only the process PATH therefore lasts until - # the first bootstrap refresh, after which Git/CMake/VS/LLVM are back and the - # rest of the install is no longer running on a simulated clean machine. - # The runner is ephemeral, so rewrite the registry copies too. Expand first: - # SetEnvironmentVariable rewrites REG_EXPAND_SZ as REG_SZ (dotnet/runtime#1442). + # install.ps1's Refresh-SessionPath (318-337, called at 1246/1278/1295/1360/ + # 1369/2797) rebuilds $env:Path from the Machine and User registry values, so + # scrubbing only the process PATH lasts until the first bootstrap refresh, + # after which Git/CMake/VS/LLVM are back and the rest of the install is no + # longer clean. The runner is ephemeral, so rewrite the registry copies too. + # Expand first: SetEnvironmentVariable rewrites REG_EXPAND_SZ as REG_SZ + # (dotnet/runtime#1442). foreach ($scope in 'Machine','User') { $raw = [System.Environment]::GetEnvironmentVariable('Path', $scope) if ([string]::IsNullOrWhiteSpace($raw)) { continue } @@ -610,15 +602,15 @@ jobs: run: | $leaked = @() # `py` too: the launcher lives in C:\Windows, which the scrub keeps, and it - # finds the toolcache Python the scrub just removed from PATH. + # finds the toolcache Python that the scrub only removed from PATH. foreach ($t in 'python','py','git','cmake','cl') { $f = Get-Command $t -ErrorAction SilentlyContinue Write-Host ("{0,-8} {1}" -f $t, $(if ($f) { $f.Source } else { 'ABSENT' })) if ($f -and $t -ne 'py') { $leaked += "$t -> $($f.Source)" } } - # Printing alone could not fail, and the leg was green while not clean: - # run 30365014702 logged `python ABSENT` and then `Python 3.13 already - # installed` / `Using CPython ... C:\hostedtoolcache\windows\Python\...`. + # Printing alone could not fail, and the leg was green while not clean: run + # 30365014702 logged `python ABSENT` then `Python 3.13 already installed` / + # `Using CPython ... C:\hostedtoolcache\windows\Python\...`. if ($leaked) { Write-Host "::error::developer tooling survived the scrub: $($leaked -join '; ')" exit 1 @@ -631,8 +623,8 @@ jobs: exit 1 } } elseif ('${{ matrix.os }}' -eq 'windows-latest' -and -not $winget) { - # Without this the visible leg quietly degrades into a second masked leg and - # the normal winget bootstrap is never exercised by any job in this workflow. + # Without this the visible leg quietly degrades into a second masked leg + # and no job in this workflow exercises the normal winget bootstrap. Write-Host '::error::winget is not resolvable on the visible leg; the winget bootstrap is not under test' exit 1 } @@ -646,9 +638,9 @@ jobs: run: | $ErrorActionPreference = 'Continue' # No -SkipTorch: install.ps1 has no param block and its parser matches - # `--no-torch` only (install.ps1:112-142), so the token was silently dropped - # and every Windows leg installed torch anyway. Torch is exactly what the - # assert below needs, so ask for it explicitly rather than by accident. + # `--no-torch` only (112-142), so the token was silently dropped and every + # Windows leg installed torch anyway. The assert below needs torch, so ask + # for it explicitly rather than by accident. & ./install.ps1 *>&1 | Tee-Object -FilePath logs/install.log $rc = $LASTEXITCODE Write-Host "installer exit code: $rc" @@ -659,12 +651,12 @@ jobs: shell: pwsh run: | # HONESTY NOTE: the hosted image ships the VC++ 2015-2022 runtime in System32 - # and it cannot be removed without breaking the runner, so a successful - # `import torch` here does NOT prove that a genuinely clean no-winget machine - # would have the runtime -- Test-VCRedistInstalled (studio/setup.ps1:875) - # finds the preinstalled DLL and Ensure-VCRedist (setup.ps1:891) short-circuits - # before it ever needs winget. Record that, then assert what CAN fail here: - # torch imports, and the masked leg really did take the no-winget path. + # and it cannot be removed without breaking the runner, so `import torch` + # succeeding here does NOT prove a genuinely clean no-winget machine has the + # runtime: Test-VCRedistInstalled (studio/setup.ps1:875) finds the + # preinstalled DLL and Ensure-VCRedist (891) short-circuits before it needs + # winget. Record that, then assert what CAN fail: torch imports, and the + # masked leg really did take the no-winget path. $sys32 = Join-Path $env:WINDIR 'System32\vcruntime140_1.dll' Write-Host "preinstalled System32 vcruntime140_1.dll: $(Test-Path $sys32)" $py = Join-Path $env:UNSLOTH_STUDIO_HOME 'unsloth_studio\Scripts\python.exe' diff --git a/.github/workflows/desktop-app-clean-machine-ci.yml b/.github/workflows/desktop-app-clean-machine-ci.yml index aaa9c3a19a..17d1189782 100644 --- a/.github/workflows/desktop-app-clean-machine-ci.yml +++ b/.github/workflows/desktop-app-clean-machine-ci.yml @@ -4,29 +4,26 @@ # Installs and launches the SHIPPED desktop app on a machine stripped of developer # tooling, on all three platforms. # -# studio-tauri-smoke.yml compiles the Tauri crate; release-desktop.yml produces the -# bundles. Neither takes a published artifact, puts it on a clean machine, and checks -# that it starts -- which is exactly the gap the reported failures fell through: both -# came from the packaged app running its bundled -# Contents/Resources/install.sh, a path no CI job exercised. +# studio-tauri-smoke.yml compiles the Tauri crate and release-desktop.yml produces the +# bundles, but neither puts a published artifact on a clean machine and checks that it +# starts -- the gap the reported failures fell through: both came from the packaged app +# running its bundled Contents/Resources/install.sh, which no CI job exercised. # -# What "runs" means here, given hosted runners have no interactive desktop session: -# - the bundle installs / mounts / extracts -# - the binary is present, of the right architecture, and passes the OS gatekeeper -# checks a user would hit (macOS quarantine + codesign, Windows installer exit) -# - the process starts and STAYS UP past its preflight (it does not exit or crash), -# which is where an unhappy app dies -# - it writes tauri.log, and that log shows the preflight disposition -- the same -# field that read `ManagedReady` over an unbootable venv in the bug report -# Linux gets the strongest check: a real webview under Xvfb. +# Hosted runners have no interactive desktop session, so "runs" means: the bundle +# installs / mounts / extracts, the binary is present, of the right architecture, and +# passes the gatekeeper checks a user would hit (macOS quarantine + codesign, Windows +# installer exit), the process STAYS UP past its preflight (where an unhappy app dies), +# and it writes tauri.log showing the preflight disposition -- the field that read +# `ManagedReady` over an unbootable venv in the bug report. Linux gets the strongest +# check: a real webview under Xvfb. name: Desktop app clean machine on: - # Also on PRs that touch this job or the machine-stripping scripts. workflow_dispatch - # alone is not enough to validate a change to the job itself: dispatch resolves the + # Also on PRs touching this job or the machine-stripping scripts: workflow_dispatch + # alone cannot validate a change to the job itself, because dispatch resolves the # workflow from the DEFAULT branch, so a new or edited file on a feature branch can - # never be dispatched, and the job would first run only after merging blind. + # never be dispatched and would first run only after merging blind. pull_request: paths: - '.github/workflows/desktop-app-clean-machine-ci.yml' @@ -59,8 +56,8 @@ permissions: env: REL_REPO: ${{ inputs.release_repo || 'unsloth-test/unsloth-test' }} - # Empty unless dispatched. A pinned tag is an immutable fixture, so a nightly - # against it can never catch a newly published broken bundle; each download step + # Empty unless dispatched: a pinned tag is an immutable fixture, so a nightly + # against it could never catch a newly published broken bundle. Each download step # resolves the newest desktop-v* release when this is empty. REL_TAG: ${{ inputs.release_tag || '' }} UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home @@ -91,8 +88,8 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | mkdir -p dl logs - # Desktop releases are prereleases (release-desktop.yml keeps them off - # repo-wide "latest"), so resolve the newest desktop-v* tag explicitly. + # Desktop releases are prereleases (never repo-wide "latest"), so the + # newest desktop-v* tag has to be resolved explicitly. if [ -z "$REL_TAG" ]; then REL_TAG="$(gh release list --repo "$REL_REPO" --limit 100 --exclude-drafts \ --json tagName,createdAt \ @@ -107,11 +104,10 @@ jobs: ls -la dl - name: Strip the developer toolchain - # `inputs` exists only for workflow_dispatch, so on pull_request and - # schedule `inputs.strip_toolchain` is the empty string -- and loose - # equality coerces both '' and false to 0, making `!= false` FALSE. The - # automatic runs would keep the hosted toolchain, which is the one thing - # this workflow exists to remove. Gate on the event instead. + # `inputs` exists only for workflow_dispatch, so on pull_request and schedule + # `inputs.strip_toolchain` is '' -- and loose equality coerces both '' and + # false to 0, making `!= false` FALSE, so automatic runs would keep the hosted + # toolchain this workflow exists to remove. Gate on the event instead. if: ${{ github.event_name != 'workflow_dispatch' || inputs.strip_toolchain }} run: | bash .github/scripts/clean-machine-env.sh mask --remove @@ -121,8 +117,8 @@ jobs: - name: Mount and install run: | DMG="$(ls dl/*.dmg | head -1)" - # A real download is quarantined; Gatekeeper treats it differently from a - # locally built bundle, and that difference is a genuine failure mode. + # A real download is quarantined, and Gatekeeper treats that differently + # from a locally built bundle: a genuine failure mode. xattr -w com.apple.quarantine \ "0081;$(printf %x $(date +%s));Safari;" "$DMG" 2>/dev/null || true hdiutil attach "$DMG" -nobrowse -quiet -mountpoint /Volumes/UnslothCI @@ -145,7 +141,7 @@ jobs: echo "::warning::Gatekeeper assessment failed -- users see 'cannot be opened' unless notarised" # The bundled installer is what actually failed for users. `::error::` is # only an annotation and `echo` exits 0, so the old `|| echo` form let a - # bundle with no installer pass this step. + # bundle with no installer pass. if [ -f "$APP/Contents/Resources/install.sh" ]; then echo "bundled install.sh present" else @@ -158,22 +154,22 @@ jobs: set -a; [ -f ./clean-machine.env ] && . ./clean-machine.env; set +a set -o pipefail APP="$(ls -d /Applications/*Unsloth*.app | head -1)" - # A headless runner never clicks Install: preflight sets `not_installed` - # and returns (studio/frontend/src/hooks/use-tauri-backend.ts:252-254) and - # startup-screen.tsx:388-389 waits for the button. Launching alone would - # therefore sit on that screen for 90s and pass without ever running the - # bundled installer. Invoke it the way studio/src-tauri/src/install.rs - # does: --tauri, stdin closed, no tty. --tauri rejects a custom studio - # home (install.sh:102-114), so drop the workspace-scoped override. + # A headless runner never clicks Install: preflight sets `not_installed` and + # returns (studio/frontend/src/hooks/use-tauri-backend.ts:252-254) while + # startup-screen.tsx:388-389 waits for the button, so launching alone would + # sit on that screen for 90s and pass without ever running the bundled + # installer. Invoke it as studio/src-tauri/src/install.rs does: --tauri, + # stdin closed, no tty. --tauri rejects a custom studio home + # (install.sh:102-114), so drop the workspace-scoped override. env -u UNSLOTH_STUDIO_HOME \ bash "$APP/Contents/Resources/install.sh" --tauri \ < /dev/null 2>&1 | tee logs/bundled-install.log PY="$HOME/.unsloth/studio/unsloth_studio/bin/python" [ -x "$PY" ] || { echo "::error::bundled installer left no venv at $PY"; exit 1; } "$PY" -V - # install.rs passes only --tauri, so torch is part of first launch. Dropping - # --no-torch here and asserting torch keeps the venv check from passing over - # a bundle whose only failure is the torch install. + # install.rs passes only --tauri, so torch is part of first launch: + # asserting it stops the venv check passing a bundle whose only failure is + # the torch install. "$PY" -c "import torch; print('torch', torch.__version__)" - name: Launch and prove it stays up @@ -210,8 +206,8 @@ jobs: grep -E "disposition=|can_auto_repair=|Xcode Command Line|ModuleNotFoundError" "$f" || true found=1 done - # Everything above is `|| true`, so on its own this step could not fail while - # the header sells the tauri.log disposition as an acceptance criterion. + # Everything above is `|| true`, so this step could not fail while the + # header sells the tauri.log disposition as an acceptance criterion. # setup_logging (src-tauri/src/main.rs:50-67) opens tauri.log unconditionally # at process start, so no log at all means the binary never got that far. [ "${found:-0}" = "1" ] || { echo "::error::the app wrote no tauri.log; it never reached setup_logging"; exit 1; } @@ -252,8 +248,8 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | mkdir -p dl logs - # Desktop releases are prereleases (release-desktop.yml keeps them off - # repo-wide "latest"), so resolve the newest desktop-v* tag explicitly. + # Desktop releases are prereleases (never repo-wide "latest"), so the + # newest desktop-v* tag has to be resolved explicitly. if [ -z "$REL_TAG" ]; then REL_TAG="$(gh release list --repo "$REL_REPO" --limit 100 --exclude-drafts \ --json tagName,createdAt \ @@ -270,9 +266,9 @@ jobs: - name: Install with NO dev tooling, only runtime libs run: | # Deliberately not build-essential/cmake/git: a user installing a .deb has - # none of that. WebKit + Xvfb are runtime requirements of the app itself, - # and apt pulls the .deb's declared deps -- if that list is wrong, this - # step is what catches it. + # none of that. WebKit + Xvfb are runtime requirements of the app, and apt + # pulls the .deb's declared deps -- if that list is wrong, this step catches + # it. sudo apt-get update -qq sudo apt-get install -y -qq --no-install-recommends xvfb if [ "${{ matrix.kind }}" = "deb" ]; then @@ -293,8 +289,8 @@ jobs: - name: Launch under Xvfb and prove it stays up run: | # Linux is the one platform where a hosted runner can give the app a real - # display, so this is the strongest "does the UI actually come up" check - # available without self-hosted hardware. + # display, so this is the strongest "does the UI come up" check available + # without self-hosted hardware. xvfb-run -a --server-args="-screen 0 1440x900x24" \ "$BIN" > logs/app-stdout.log 2>&1 & APP_PID=$! @@ -348,8 +344,8 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | mkdir -p dl logs - # Desktop releases are prereleases (release-desktop.yml keeps them off - # repo-wide "latest"), so resolve the newest desktop-v* tag explicitly. + # Desktop releases are prereleases (never repo-wide "latest"), so the + # newest desktop-v* tag has to be resolved explicitly. if [ -z "$REL_TAG" ]; then REL_TAG="$(gh release list --repo "$REL_REPO" --limit 100 --exclude-drafts \ --json tagName,createdAt \ @@ -363,11 +359,10 @@ jobs: ls -la dl - name: Strip developer tooling from PATH - # `inputs` exists only for workflow_dispatch, so on pull_request and - # schedule `inputs.strip_toolchain` is the empty string -- and loose - # equality coerces both '' and false to 0, making `!= false` FALSE. The - # automatic runs would keep the hosted toolchain, which is the one thing - # this workflow exists to remove. Gate on the event instead. + # `inputs` exists only for workflow_dispatch, so on pull_request and schedule + # `inputs.strip_toolchain` is '' -- and loose equality coerces both '' and + # false to 0, making `!= false` FALSE, so automatic runs would keep the hosted + # toolchain this workflow exists to remove. Gate on the event instead. if: ${{ github.event_name != 'workflow_dispatch' || inputs.strip_toolchain }} shell: pwsh run: | @@ -382,8 +377,8 @@ jobs: shell: pwsh run: | $exe = (Get-ChildItem dl/*setup.exe | Select-Object -First 1).FullName - # /S is the NSIS silent switch. A user double-clicks, but an installer that - # cannot run unattended also cannot be scripted or MDM-deployed. + # /S is the NSIS silent switch: a user double-clicks, but an installer that + # cannot run unattended cannot be scripted or MDM-deployed either. $p = Start-Process -FilePath $exe -ArgumentList '/S' -Wait -PassThru Write-Host "installer exit: $($p.ExitCode)" if ($p.ExitCode -ne 0) { Write-Host "::error::silent install failed"; exit 1 } From 6937234f2d6f33b30872c989b2d08182a7c218d0 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 19:57:35 +0000 Subject: [PATCH 07/36] Point the nightly at the repo that publishes, and let its checks fail REL_REPO defaulted to unsloth-test/unsloth-test, which holds one release frozen at 2026-07-27, while release-desktop.yml publishes into github.repository. The schedule was re-testing the same fixture forever and could never see a broken production bundle. The windows job carried a blanket continue-on-error, so its NSIS assertions could not gate. lipo -archs prints and exits 0 for a thin binary and `|| true` swallowed even that, so the architecture was never checked; fall back to file, which survives the CLT mask. And require the preflight disposition line rather than the mere existence of tauri.log, which setup_logging creates at process start regardless. --- .../desktop-app-clean-machine-ci.yml | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/.github/workflows/desktop-app-clean-machine-ci.yml b/.github/workflows/desktop-app-clean-machine-ci.yml index 17d1189782..621b3d1c68 100644 --- a/.github/workflows/desktop-app-clean-machine-ci.yml +++ b/.github/workflows/desktop-app-clean-machine-ci.yml @@ -38,7 +38,7 @@ on: release_repo: description: 'owner/name hosting the desktop release' type: string - default: 'unsloth-test/unsloth-test' + default: '' strip_toolchain: description: 'Strip developer tooling before installing' type: boolean @@ -55,7 +55,11 @@ permissions: contents: read env: - REL_REPO: ${{ inputs.release_repo || 'unsloth-test/unsloth-test' }} + # release-desktop.yml publishes into github.repository, so a nightly aimed + # anywhere else goes green over a broken production bundle. + # unsloth-test/unsloth-test holds one frozen release, so the schedule was + # re-testing the same fixture forever. + REL_REPO: ${{ inputs.release_repo || github.repository }} # Empty unless dispatched: a pinned tag is an immutable fixture, so a nightly # against it could never catch a newly published broken bundle. Each download step # resolves the newest desktop-v* release when this is empty. @@ -133,7 +137,17 @@ jobs: APP="$(ls -d /Applications/*Unsloth*.app | head -1)" BIN="$APP/Contents/MacOS/$(/usr/libexec/PlistBuddy -c 'Print :CFBundleExecutable' "$APP/Contents/Info.plist")" file "$BIN" - lipo -archs "$BIN" || true + # `lipo -archs` prints and exits 0 for a thin x86_64 binary, and `|| true` + # swallowed even that, so "the right architecture" was never asserted. lipo + # is an xcrun shim too, absent once the strip step moved CommandLineTools + # aside; /usr/bin/file is base system. + ARCHS="$(lipo -archs "$BIN" 2>/dev/null || true)" + [ -n "$ARCHS" ] || ARCHS="$(file -b "$BIN")" + echo "architectures: $ARCHS" + case "$ARCHS" in + *arm64*|*aarch64*) ;; + *) echo "::error::the aarch64 .dmg carries no arm64 binary ($ARCHS)"; exit 1 ;; + esac # Report rather than gate: an unnotarised beta is expected to fail # assessment, but a user WILL hit this, so it must be visible. codesign -dv --verbose=2 "$APP" 2>&1 | head -20 || true @@ -205,12 +219,17 @@ jobs: # The two fields the bug report turned on. grep -E "disposition=|can_auto_repair=|Xcode Command Line|ModuleNotFoundError" "$f" || true found=1 + if grep -qE "desktop_preflight completed disposition=" "$f"; then disposition=1; fi done # Everything above is `|| true`, so this step could not fail while the # header sells the tauri.log disposition as an acceptance criterion. # setup_logging (src-tauri/src/main.rs:50-67) opens tauri.log unconditionally # at process start, so no log at all means the binary never got that far. [ "${found:-0}" = "1" ] || { echo "::error::the app wrote no tauri.log; it never reached setup_logging"; exit 1; } + # setup_logging opens tauri.log at process start, so its existence is implied + # by the launch step. The disposition line is the field the bug report turned + # on, so a process that hangs before preflight must not pass. + [ "${disposition:-0}" = "1" ] || { echo "::error::tauri.log records no desktop_preflight disposition; the app never completed preflight"; exit 1; } - name: Restore the runner if: always() @@ -331,7 +350,6 @@ jobs: name: desktop windows runs-on: windows-latest timeout-minutes: 45 - continue-on-error: true steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 From 85305a2163fe428c5cb1aa9555a2f8e476ac8c67 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 28 Jul 2026 20:09:03 +0000 Subject: [PATCH 08/36] Stop four clean-machine checks from passing over a real failure Re-run `absent` after the install on the masked macOS legs. It only ran before, so an installer that quietly selected the Xcode CLT or installed a compiler left the leg green while every later source build could succeed, which is the one thing clean-machine-assert.sh says `absent` guards the whole run against. Fail the Windows simulation when py.exe can still start an interpreter. The launcher binary itself may stay, but Find-CompatiblePython probes `py` first (install.ps1:1130-1153), so an interpreter registered outside the two renamed toolcache directories gets reused and Python bootstrap is never exercised. Exempting `py` without ever running it left that unchecked. Propagate the WSL installer exit code. It was printed and discarded, and the CLI check does not compensate: install.sh links the `unsloth` shim (4174-4182) before it reports a failing studio/setup.sh (4219-4230), so a late setup failure leaves a shim whose --version succeeds. Run the bundled installer in the Linux desktop jobs. The launch step only proves the process stayed alive, and on a fresh home preflight reports not_installed and the app waits on the install screen, so both required rows passed after 90 seconds without ever touching the shipped install.sh. Locate the resource in the deb payload or the extracted AppImage, run it the way install.rs does, and require a managed venv that can import torch. --- .../workflows/clean-machine-install-ci.yml | 31 +++++++++++++++++-- .../desktop-app-clean-machine-ci.yml | 28 +++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index bc78ae040d..bd37f9b1ce 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -183,8 +183,14 @@ jobs: run: | set -a; . ./clean-machine.env; set +a checks="nobuild" + # `absent` ran only BEFORE the install, so an installer that quietly + # selected the CLT or installed a compiler left the leg green while every + # later source build could succeed -- the exact behaviour the assert script + # says `absent` guards the whole run against. Re-run it after the install. + [ "${{ matrix.mode }}" = "mask" ] && checks="$checks absent" [ "${{ matrix.mode }}" = "trace" ] && checks="$checks notools" - bash .github/scripts/clean-machine-assert.sh $checks + UNSLOTH_CLEAN_ALLOW_WORKING='${{ matrix.allow_working }}' \ + bash .github/scripts/clean-machine-assert.sh $checks - name: Assert llama.cpp loads if: steps.install.outcome == 'success' @@ -438,7 +444,16 @@ jobs: # Piped, same shape as `curl ... | sh`, so an early exit still exposes the # broken pipe, but the script under test is this ref not production's. wsl -d unsloth-ci -u root -- sh -c 'cd /root && cat install.sh | sh' 2>&1 | Tee-Object -FilePath logs/wsl-install.log - Write-Host "installer exit: $LASTEXITCODE" + $installRc = $LASTEXITCODE + Write-Host "installer exit: $installRc" + # Printing the code discarded it. The CLI check in the next step does not + # compensate: install.sh links the `unsloth` shim (4174-4182) BEFORE it + # reports a failing studio/setup.sh (4219-4230), so a late setup failure + # leaves a shim whose --version succeeds and the whole job looked green. + if ($installRc -ne 0) { + Write-Host "::error::WSL installer exited $installRc" + exit $installRc + } - name: Did it detect WSL, and did it end up usable? if: always() @@ -608,6 +623,18 @@ jobs: Write-Host ("{0,-8} {1}" -f $t, $(if ($f) { $f.Source } else { 'ABSENT' })) if ($f -and $t -ne 'py') { $leaked += "$t -> $($f.Source)" } } + # The launcher binary may stay, but an interpreter it can still START is a + # leak: Find-CompatiblePython (install.ps1:1130-1153) probes `py` first, so + # any version registered outside the two renamed toolcache directories gets + # reused and Python bootstrap never runs. Exempting `py` without running it + # left that unchecked. + if (Get-Command py -ErrorAction SilentlyContinue) { + Write-Host "py -0p:"; & py -0p 2>&1 | ForEach-Object { Write-Host " $_" } + foreach ($v in '-3.11', '-3.12', '-3.13') { + $out = & py $v -c "import sys; print(sys.executable)" 2>&1 + if ($LASTEXITCODE -eq 0) { $leaked += "py $v -> $out" } + } + } # Printing alone could not fail, and the leg was green while not clean: run # 30365014702 logged `python ABSENT` then `Python 3.13 already installed` / # `Using CPython ... C:\hostedtoolcache\windows\Python\...`. diff --git a/.github/workflows/desktop-app-clean-machine-ci.yml b/.github/workflows/desktop-app-clean-machine-ci.yml index 621b3d1c68..b8ca0b1738 100644 --- a/.github/workflows/desktop-app-clean-machine-ci.yml +++ b/.github/workflows/desktop-app-clean-machine-ci.yml @@ -305,6 +305,34 @@ jobs: echo "BIN=$BIN" >> "$GITHUB_ENV" echo "binary: $BIN" + - name: Run the bundled installer, the path first launch takes + run: | + set -o pipefail + # The launch step below only proves the process stayed alive: on a fresh + # home preflight reports not_installed and the app sits on the install + # screen waiting for a click (use-tauri-backend.ts:252-254, + # startup-screen.tsx:388-389), so a bundle whose embedded install.sh is + # missing or broken passed both Linux rows. tauri.conf.json:56-59 ships + # install.sh as a bundle resource, so find it where the bundle put it and + # run it as install.rs does. + if [ "${{ matrix.kind }}" = "deb" ]; then + SH="$(dpkg -L "$(dpkg-deb -f dl/*.deb Package)" | grep -E '/install\.sh$' | head -1)" + else + (cd dl && "$(ls *.AppImage | head -1)" --appimage-extract >/dev/null) + SH="$(find dl/squashfs-root -name install.sh -type f | head -1)" + fi + [ -n "$SH" ] && [ -f "$SH" ] || { echo "::error::the bundle ships no install.sh resource"; exit 1; } + echo "bundled installer: $SH" + # --tauri rejects a custom studio home (install.sh:102-114), so drop the + # workspace-scoped override, and close stdin as install.rs does. + env -u UNSLOTH_STUDIO_HOME \ + bash "$SH" --tauri < /dev/null 2>&1 | tee logs/bundled-install.log + PY="$HOME/.unsloth/studio/unsloth_studio/bin/python" + [ -x "$PY" ] || { echo "::error::bundled installer left no venv at $PY"; exit 1; } + "$PY" -V + # install.rs passes only --tauri, so torch is part of first launch. + "$PY" -c "import torch; print('torch', torch.__version__)" + - name: Launch under Xvfb and prove it stays up run: | # Linux is the one platform where a hosted runner can give the app a real From b573f067d12716db0c4e59d4748f272f8732c1e1 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 28 Jul 2026 20:18:12 +0000 Subject: [PATCH 09/36] Prove the trace wrapper records before trusting an empty trace The `notools` check reads an absence: it passes when the trace file contains no compiler, git or brew invocation. A shim directory that never reached PATH produces exactly the same empty file as an installer that touched nothing, so the single leg carrying that assertion would stay green no matter what the installer did. "Verify the simulation actually took effect" only ran for mask mode, which left the trace leg with nothing checking its own instrumentation. Call git explicitly after sourcing the environment and require it to appear in the trace, then truncate the file so the self-test entry does not count against the install. The call has to be explicit because macOS reaches _has_working_git only under STUDIO_LOCAL_INSTALL (install.sh:2026), so no consumer leg on that platform probes git on its own. --- .../workflows/clean-machine-install-ci.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index bd37f9b1ce..6ed02a7144 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -123,6 +123,25 @@ jobs: UNSLOTH_CLEAN_ALLOW_WORKING='${{ matrix.allow_working }}' \ bash .github/scripts/clean-machine-assert.sh absent + - name: Verify the trace actually records + if: matrix.mode == 'trace' + run: | + # `notools` reads an absence, so a shim dir that never reached PATH is + # indistinguishable from an installer that touched nothing, and the one leg + # carrying that assertion would pass no matter what the installer did. + # Prove the wrapper records before trusting an empty file. macOS never + # probes git off the --local path, so this must be an explicit call. + set -a; . ./clean-machine.env; set +a + [ -n "$UNSLOTH_TOOL_TRACE" ] || { echo "::error::trace mode set no UNSLOTH_TOOL_TRACE"; exit 1; } + git --version >/dev/null 2>&1 || true + if ! grep -q "^git[[:space:]]" "$UNSLOTH_TOOL_TRACE"; then + echo "::error::the trace wrapper did not record a git call, so notools proves nothing" + echo "PATH=$PATH"; command -v git; cat "$UNSLOTH_TOOL_TRACE" || true + exit 1 + fi + echo "trace wrapper records; clearing the self-test entry" + : > "$UNSLOTH_TOOL_TRACE" + - name: Install id: install env: From 50afa4d21c9040b4bf8c5aeeff378d968b34fd41 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 21:56:16 +0000 Subject: [PATCH 10/36] Stop the Windows clean-machine check failing on its own probe exit code All three Windows legs failed "Verify the simulation took effect" with no ::error:: printed at all. The check itself was right: the mask step logged "masked toolcache python: C:\hostedtoolcache\windows\Python", python/git/cmake/cl were ABSENT, no `py -3.x` probe started an interpreter, and the winget assertions were satisfied. The step still exited 1. The cause is $LASTEXITCODE leaking out of the step. The last external command is the `py -3.13` probe, which is SUPPOSED to fail; Get-Command and Write-Host are cmdlets and never reset $LASTEXITCODE, and the runner appends `if ((Test-Path -LiteralPath variable:\LASTEXITCODE)) { exit $LASTEXITCODE }` to every pwsh step (actions/runner#351). So a clean machine reported failure, and because this step runs before Install, no Windows leg has ever reached the installer. Clear $LASTEXITCODE after the probe loop and end with an explicit exit 0. The leak detection is untouched: a surviving python/git/cmake/cl, or a `py -3.x` that actually starts, still exits 1. Also print each probe's exit code and output, so the next failure here explains itself instead of being silent, and label `py -0p` as what it is. The launcher reads the registry, which the on-disk toolcache rename cannot rewrite, so -0p keeps naming paths that no longer exist. Unlabelled it reads like a leak. Accept the Fedora leg's real outcome instead of a message that can be absent The fedora assertion only accepted the unsupported-package-manager hard exit. That is still what this ref's install.sh does, but the pending installer change replaces it with a warning that lets the install continue, at which point the old grep matches nothing and the step fails for the wrong reason. Handle both, strictly. If the log shows the newer "using prebuilt llama.cpp (missing:" warning, the Linux gate demonstrably did not hard-stop, and the only tolerated failure past that point is release lag: install.sh comes from this ref while unsloth comes from PyPI, and the released studio/install_python_stack.py has no "skip triton kernels when git is missing" guard, so it still fetches the git+https triton_kernels requirement on a machine with no git. Anything else after that warning fails the step. Otherwise the old hard-exit message is still required. A missing log, a bootstrap outage or any unrecognised failure all remain errors, and the step retires to a plain success assertion once a release ships the no-git skip. --- .../workflows/clean-machine-install-ci.yml | 50 +++++++++++++++++-- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index 6ed02a7144..9e95e7eba6 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -364,7 +364,7 @@ jobs: # This leg is continue-on-error like the nonroot one, so without the same check # a bootstrap outage or an unrelated early exit would be tolerated exactly like # the intentional diagnostic. - - name: Assert the Fedora failure is the unsupported-package-manager one + - name: Assert the Fedora outcome is a known one if: always() && matrix.label == 'fedora41' run: | if [ "${{ steps.install_root.outcome }}" = "success" ]; then @@ -373,8 +373,30 @@ jobs: fi [ -f logs/install.log ] || { echo "::error::fedora leg produced no install log"; exit 1; } tail -40 logs/install.log + # install.sh comes from this ref, so which of the two accepted outcomes + # applies depends on which dependency gate this ref carries. + if grep -q "using prebuilt llama.cpp (missing:" logs/install.log; then + # The gate no longer hard-stops on a non-apt distro: it warns that the + # optional build tools are absent and carries on. Reaching this warning is + # what proves the Linux gate did not stop the install. + # Past that point the only accepted failure is release lag: install.sh is + # taken from this ref but unsloth is installed from PyPI, and the released + # studio/install_python_stack.py has no "skip the triton kernels when git + # is missing" guard, so it still fetches the git+https triton_kernels + # requirement on a machine that has no git. Once a release carries that + # guard this whole step retires to a plain success assertion. + if ! grep -q "Installing triton kernels (pip) failed" logs/install.log; then + echo "::error::fedora got past the dependency warning then failed for a new reason, not the known triton/git release lag" + exit 1 + fi + echo "::warning::fedora fails only on triton_kernels (git+https) from the released unsloth; drop this step once a release ships the no-git skip" + exit 0 + fi + # This ref still hard-exits on a non-apt package manager. Pin that message so + # a bootstrap outage or an unrelated early exit is not tolerated as if it + # were the intentional diagnostic. if ! grep -qiE "Automatic system package installation is supported on apt-based|Fedora/RHEL: sudo dnf install" logs/install.log; then - echo "::error::fedora leg failed for a reason other than the unsupported package manager" + echo "::error::fedora leg failed neither at the unsupported-package-manager gate nor at the known triton/git release lag" exit 1 fi @@ -648,11 +670,27 @@ jobs: # reused and Python bootstrap never runs. Exempting `py` without running it # left that unchecked. if (Get-Command py -ErrorAction SilentlyContinue) { - Write-Host "py -0p:"; & py -0p 2>&1 | ForEach-Object { Write-Host " $_" } + # -0p prints the launcher's REGISTRY view. The mask step renames the + # toolcache directory on disk but cannot rewrite those registry entries, + # so -0p keeps naming paths that no longer exist. It is context for a + # failure, never evidence of one -- only a probe that starts counts. + Write-Host "py -0p (stale registry entries; masked paths no longer exist on disk):" + & py -0p 2>&1 | ForEach-Object { Write-Host " $_" } foreach ($v in '-3.11', '-3.12', '-3.13') { $out = & py $v -c "import sys; print(sys.executable)" 2>&1 - if ($LASTEXITCODE -eq 0) { $leaked += "py $v -> $out" } + $rc = $LASTEXITCODE + # Print every probe: when this check next fails it must say why. + Write-Host ("py {0} -> exit {1}: {2}" -f $v, $rc, (($out | Out-String).Trim() -replace '\r?\n', ' / ')) + if ($rc -eq 0) { $leaked += "py $v -> $out" } } + # A probe that FAILS is the outcome we want, but it leaves $LASTEXITCODE + # non-zero, and Get-Command/Write-Host are cmdlets that never reset it. + # The runner appends + # if ((Test-Path -LiteralPath variable:\LASTEXITCODE)) { exit $LASTEXITCODE } + # to every pwsh step (actions/runner#351), so all three Windows legs + # exited 1 with no ::error:: printed, on machines that were in fact clean + # -- and never reached the Install step at all. + $global:LASTEXITCODE = 0 } # Printing alone could not fail, and the leg was green while not clean: run # 30365014702 logged `python ABSENT` then `Python 3.13 already installed` / @@ -677,6 +715,10 @@ jobs: foreach ($scope in 'Machine','User') { Write-Host ("{0} PATH after scrub: {1}" -f $scope, [System.Environment]::GetEnvironmentVariable('Path', $scope)) } + # Every failure above exits 1 explicitly, so reaching here means the machine + # is clean. Be explicit rather than leaving the runner's appended + # `exit $LASTEXITCODE` to decide. + exit 0 - name: Install id: install From 9a8a749d07158aa216bec4c0e4ee80f6390d44f0 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 28 Jul 2026 22:02:47 +0000 Subject: [PATCH 11/36] Make the AppImage Linux row actually extract, and hold Linux to the macOS preflight bar The appimage row invoked the extractor by bare filename, and a command word with no slash is resolved through PATH rather than the working directory, so the extraction exited 127 and the bundled-installer assertion below it never ran. Prefix it with ./ so the row exercises what it claims to. The Linux log step also asserted nothing: it skipped a missing log with continue and discarded the grep with || true. The launch step only proves the process stayed alive for 90 seconds, and the bundled-installer checks do not exercise the Rust preflight path, so an app that hung before preflight completed passed both required Linux rows. Require the same desktop_preflight completed disposition= record the macOS rows already do. --- .../workflows/desktop-app-clean-machine-ci.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/desktop-app-clean-machine-ci.yml b/.github/workflows/desktop-app-clean-machine-ci.yml index b8ca0b1738..8694cd4e1b 100644 --- a/.github/workflows/desktop-app-clean-machine-ci.yml +++ b/.github/workflows/desktop-app-clean-machine-ci.yml @@ -318,7 +318,9 @@ jobs: if [ "${{ matrix.kind }}" = "deb" ]; then SH="$(dpkg -L "$(dpkg-deb -f dl/*.deb Package)" | grep -E '/install\.sh$' | head -1)" else - (cd dl && "$(ls *.AppImage | head -1)" --appimage-extract >/dev/null) + # ls returns a bare filename here, and a command word with no slash is + # resolved through PATH, not the cwd, so this needs the ./ prefix. + (cd dl && "./$(ls *.AppImage | head -1)" --appimage-extract >/dev/null) SH="$(find dl/squashfs-root -name install.sh -type f | head -1)" fi [ -n "$SH" ] && [ -f "$SH" ] || { echo "::error::the bundle ships no install.sh resource"; exit 1; } @@ -362,7 +364,18 @@ jobs: [ -f "$f" ] || continue echo "=== $f ==="; cp "$f" logs/ 2>/dev/null || true; tail -60 "$f" grep -E "disposition=|can_auto_repair=|ModuleNotFoundError" "$f" || true + found=1 + if grep -qE "desktop_preflight completed disposition=" "$f"; then disposition=1; fi done + # Same acceptance criterion the macOS rows already enforce. Everything + # above is `|| true` and the loop skips a missing log outright, so + # without these two lines the step could not fail. setup_logging + # (src-tauri/src/main.rs:50-67) opens tauri.log unconditionally at + # process start, so no log at all means the binary never got that far, + # and the launch step only proves the process stayed alive: an app that + # hangs before preflight completes would otherwise pass both Linux rows. + [ "${found:-0}" = "1" ] || { echo "::error::the app wrote no tauri.log; it never reached setup_logging"; exit 1; } + [ "${disposition:-0}" = "1" ] || { echo "::error::tauri.log records no desktop_preflight disposition; the app never completed preflight"; exit 1; } - name: Upload logs if: always() From 3be21e87cf4adf7e5a4b262f63d520e5d7faca48 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 22:25:13 +0000 Subject: [PATCH 12/36] Put the branch's own Python under test on the clean-machine legs install.sh and install.ps1 come from the ref under test, but they install unsloth from PyPI, which is the consumer path and has to stay that way. That left everything Python-side coming out of the released wheel: studio/setup.sh, studio/setup.ps1, studio/install_python_stack.py, and every requirements and constraints file those resolve through Path(__file__). A branch that changes constraints.txt or setup.ps1 therefore got a green run that proved nothing about the change, and some legs proved less than they looked. The Fedora assertion was already carrying a hand-written workaround for exactly this, tolerating a triton/git failure on the grounds that the released package lags the ref. Legs marked overlay: true now re-point the venv at the ref just before studio setup runs, through UNSLOTH_CI_SOURCE_OVERLAY: a --no-deps editable install of the checkout. That makes import studio resolve to the working tree, so the existing setup-script lookup finds the ref's setup.sh / setup.ps1 and install_python_stack reads the ref's constraints, with no other change to either installer. Not --local: --local additionally installs unsloth-zoo from a git+https URL, which genuinely needs git, and git absence is the whole point of the masked legs. The overlay resolves no dependencies and clones nothing, so it holds up with git, cmake and the compilers all gone. It is not a consumer knob either: no flag, no usage entry, ignored unless the variable names a directory with a pyproject.toml in it. Four legs stay on the released package deliberately, each for its own reason, recorded in the header: the mac pipe legs keep an end-to-end signal on what a user actually runs; the trace leg would otherwise answer its own question, since the editable build calls git through setuptools-scm's file finder; the non-root Linux leg dies before a venv exists; and WSL only ever receives install.sh, not a source tree. Two supporting fixes the overlay depends on or exposes: install_python_stack.py discarded uv's output whenever a step succeeded, so the nobuild assertion, which reads the install log, could not see a source build in the dependency phase at all. That is the phase that installs studio.txt, where an sdist-only dependency actually turns up, and it reported "built: none" regardless. It now echoes successful output under UNSLOTH_VERBOSE, matching what install.sh's run_install_cmd already does. nobuild now ignores "Building @ file://" lines. A local-path build is something the caller pointed at, never a dependency resolution chose, and index dependencies always print ==, so a real sdist from PyPI is still caught, including one named unsloth. Each overlaid leg also asserts it really was overlaid, so an unset variable cannot quietly put the whole matrix back on the released wheel. --- .github/scripts/clean-machine-assert.sh | 9 + .../workflows/clean-machine-install-ci.yml | 171 +++++++++++++++--- install.ps1 | 38 ++++ install.sh | 32 ++++ studio/install_python_stack.py | 16 ++ 5 files changed, 245 insertions(+), 21 deletions(-) diff --git a/.github/scripts/clean-machine-assert.sh b/.github/scripts/clean-machine-assert.sh index 9a81ba2186..c2c65708cf 100755 --- a/.github/scripts/clean-machine-assert.sh +++ b/.github/scripts/clean-machine-assert.sh @@ -109,8 +109,17 @@ for check in "$@"; do # every uv source build. Match both spellings. Requiring `==` or ` @ ` after # the name keeps this off the installer's own lowercase "building frontend..." # progress text. Strip ANSI first so a coloured run (FORCE_COLOR) parses. + # + # `Building @ file://...` is dropped before the names are read: a + # local-path build is something the caller pointed at (install.sh --local, + # or the UNSLOTH_CI_SOURCE_OVERLAY editable overlay the CI legs use to put + # the branch's Python code under test), never a dependency that resolution + # chose. Dependencies from an index always print `==`, so + # this drops no real signal -- a genuine sdist pulled from PyPI is still + # caught, including one named unsloth. _esc=$(printf '\033') _built="$(sed -E "s/${_esc}\[[0-9;]*[A-Za-z]//g" "$LOG" 2>/dev/null \ + | grep -viE "building [a-z0-9._-]+ @ file://" \ | grep -oiE "building wheel for [a-z0-9._-]+|building [a-z0-9._-]+(==| @ )" \ | tr 'A-Z' 'a-z' \ | sed -E -e 's/^building wheel for //' -e 's/^building //' -e 's/(==| @ )$//' \ diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index 9e95e7eba6..cd630f7b3b 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -13,6 +13,39 @@ # mask -> the toolchain is genuinely unusable; does the install still work? # trace -> the toolchain works but is logged; does the installer ever call it? # Linux is the exception: containers are genuinely clean. +# +# ── What each leg actually puts under test ──────────────────────────────────── +# install.sh / install.ps1 come from this ref, but they install unsloth FROM PyPI, +# because that is the consumer path and it has to stay that way. Everything +# Python-side is therefore read out of the RELEASED wheel: studio/setup.sh, +# studio/setup.ps1, studio/install_python_stack.py, and every requirements and +# constraints file those resolve through Path(__file__). Left alone, this workflow +# can only ever validate the two shell installers, and a branch that changes +# constraints.txt or setup.ps1 gets a green run that proves nothing about the +# change. The `Assert the Fedora outcome is a known one` step below was already +# carrying a hand-written workaround for exactly this. +# +# So legs with `overlay: true` re-point the venv at this ref before studio setup +# runs, via UNSLOTH_CI_SOURCE_OVERLAY (install.sh / install.ps1, just above their +# "Run studio setup" section): a `--no-deps` editable install of the checkout. +# That makes `import studio` resolve to the working tree, so the existing +# setup-script lookup finds this ref's setup.sh / setup.ps1 and install_python_stack +# reads this ref's constraints. It is deliberately NOT `install.sh --local`: +# --local also installs `unsloth-zoo @ git+https://...`, which genuinely needs git, +# and git absence is the whole point of the masked legs. The overlay resolves no +# dependencies and clones nothing, so it still works with git, cmake and the +# compilers all gone. +# +# Legs left on `overlay: false`, and why: +# mac */mask/pipe the `curl | sh` shape a user runs. Kept end-to-end on the +# released package so a broken PyPI release still shows up. +# mac macos-14/trace `notools` asserts the installer never reaches for git, and +# the editable build itself calls `git rev-parse` / +# `git archive` through setuptools-scm's file finder. An +# overlay here would answer the leg's own question for it. +# linux ubuntu2404-nonroot dies at the elevation gate before a venv exists. +# wsl only install.sh is copied into the distro; there is no +# source tree inside WSL to overlay from. name: Clean machine install @@ -70,23 +103,28 @@ jobs: fail-fast: false matrix: include: + # `overlay` decides whether this ref's Python code is put under test at all; + # see the header. The pipe legs stay on the released package on purpose. + # # The reported failure, in the shape users run it. Default install (with # torch) because that is what a consumer actually gets. - - {os: macos-14, mode: mask, delivery: pipe, flags: '', experimental: false} - - {os: macos-14, mode: mask, delivery: file, flags: '', experimental: false} + - {os: macos-14, mode: mask, delivery: pipe, flags: '', experimental: false, overlay: false} + - {os: macos-14, mode: mask, delivery: file, flags: '', experimental: false, overlay: true} # What the desktop app runs: no tty, stdin closed, TAURI markers on. - - {os: macos-14, mode: mask, delivery: tauri, flags: '', experimental: false} + - {os: macos-14, mode: mask, delivery: tauri, flags: '', experimental: false, overlay: true} # Toolchain present but logged: does the installer ever reach for it? - - {os: macos-14, mode: trace, delivery: file, flags: '', experimental: false} + # No overlay: the editable build calls git itself (setuptools-scm), which + # would plant the very evidence `notools` exists to look for. + - {os: macos-14, mode: trace, delivery: file, flags: '', experimental: false, overlay: false} # --no-torch is the one macOS path that can still want a compiler # (sentencepiece has no guaranteed cp313 arm64 wheel), so probe it apart # from the default path instead of letting it hide the gate under test. - - {os: macos-14, mode: mask, delivery: file, flags: '--no-torch', experimental: true} - - {os: macos-15, mode: mask, delivery: pipe, flags: '', experimental: false} - - {os: macos-26, mode: mask, delivery: file, flags: '', experimental: true} + - {os: macos-14, mode: mask, delivery: file, flags: '--no-torch', experimental: true, overlay: true} + - {os: macos-15, mode: mask, delivery: pipe, flags: '', experimental: false, overlay: false} + - {os: macos-26, mode: mask, delivery: file, flags: '', experimental: true, overlay: true} # Intel pins python 3.12 and its /usr/bin/git is not CLT-provided, so it # survives masking. Informational only. - - {os: macos-15-intel, mode: mask, delivery: file, flags: '', experimental: true, allow_working: 'git'} + - {os: macos-15-intel, mode: mask, delivery: file, flags: '', experimental: true, overlay: true, allow_working: 'git'} steps: # checkout FIRST: it needs a working git, which masking then takes away. @@ -148,6 +186,11 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Withheld on PR: this step runs checked-out PR code. HF_TOKEN: ${{ github.event_name != 'pull_request' && secrets.HF_TOKEN || '' }} + # Empty, and therefore ignored by install.sh, on the non-overlay legs. + # Also empty for `installer_source: published`, where the script under test + # is production's and has no such hook: overlaying this ref's Python onto it + # would report on neither one honestly. + UNSLOTH_CI_SOURCE_OVERLAY: ${{ matrix.overlay && inputs.installer_source != 'published' && github.workspace || '' }} run: | set -a; . ./clean-machine.env; set +a set -o pipefail @@ -197,6 +240,18 @@ jobs: fi exit "$rc" + # Without this the gap comes back silently: install.sh ignores an unset + # UNSLOTH_CI_SOURCE_OVERLAY, so a typo in the matrix or the expression would + # put every leg back on the released wheel and nothing in the run would say so. + - name: Assert this ref's Python was really put under test + if: matrix.overlay && inputs.installer_source != 'published' && steps.install.outcome == 'success' + run: | + grep -q "CI: overlaying source checkout" logs/install.log || { + echo "::error::this leg is marked overlay: true but the installer never overlaid the checkout, so it only tested the released package" + exit 1 + } + echo "overlay applied; this leg exercised this ref's Python" + - name: Assert no source build and no toolchain use if: always() && steps.install.outcome == 'success' run: | @@ -266,23 +321,28 @@ jobs: image: ubuntu:24.04 runner: ubuntu-latest experimental: false + overlay: true - label: ubuntu2404-arm-root image: ubuntu:24.04 runner: ubuntu-24.04-arm experimental: false + overlay: true # No elevation: today this hard-fails at install.sh:856-861. Expected # failure; the point is to pin the message and prove it is actionable - # rather than a bare `curl: (56)`. + # rather than a bare `curl: (56)`. No overlay: it never gets as far as a + # venv, so there would be nothing to overlay into. - label: ubuntu2404-nonroot image: ubuntu:24.04 runner: ubuntu-latest experimental: true + overlay: false # Non-apt: today this hard-fails at install.sh:2034. Expected failure; # forces the decision on whether dnf/pacman/zypper get supported. - label: fedora41 image: fedora:41 runner: ubuntu-latest experimental: true + overlay: true steps: - name: Describe the container's starting state @@ -296,10 +356,17 @@ jobs: # Everything else stays absent. - name: Provision only the bootstrap transport run: | + # tar and gzip ride along on the overlay legs: with no actions/checkout here + # (it needs git) the only way to get this ref's source into the container is + # to fetch and unpack an archive over the same transport. Neither is a + # compiler, git or cmake, so the clean-machine premise is untouched. Both + # are usually already in the base image; naming them just makes it certain. + pkgs="ca-certificates curl" + if [ "${{ matrix.overlay }}" = "true" ]; then pkgs="$pkgs tar gzip"; fi if command -v apt-get >/dev/null 2>&1; then - apt-get update -qq && apt-get install -y -qq --no-install-recommends ca-certificates curl + apt-get update -qq && apt-get install -y -qq --no-install-recommends $pkgs elif command -v dnf >/dev/null 2>&1; then - dnf install -y -q ca-certificates curl + dnf install -y -q $pkgs fi # No actions/checkout on purpose: it needs git, and a container with git @@ -321,6 +388,19 @@ jobs: fi wc -l install.sh + # The overlay needs a source tree, and these legs deliberately have no + # actions/checkout. codeload serves the same commit as a tarball over plain + # HTTPS, so this ref's Python code gets in without a git client. + - name: Fetch this ref's source tree for the overlay + if: matrix.overlay && inputs.installer_source != 'published' + run: | + set -e + mkdir -p ci-source + curl -fsSL "https://codeload.github.com/${GITHUB_REPOSITORY}/tar.gz/${GITHUB_SHA}" \ + | tar -xz -C ci-source --strip-components=1 + [ -f ci-source/pyproject.toml ] || { echo "::error::source tarball for ${GITHUB_SHA} unpacked without a pyproject.toml"; ls -la ci-source; exit 1; } + echo "overlay source: $(pwd)/ci-source" + - name: Create an unprivileged user if: matrix.label == 'ubuntu2404-nonroot' run: | @@ -339,6 +419,13 @@ jobs: if: matrix.label != 'ubuntu2404-nonroot' run: | set -o pipefail + # Resolved here rather than in `env:` so it tracks the step's real working + # directory: a container job remaps the workspace and github.workspace is + # not something this needs to depend on. + if [ -d ci-source ]; then + export UNSLOTH_CI_SOURCE_OVERLAY="$PWD/ci-source" + echo "overlaying this ref's source from $UNSLOTH_CI_SOURCE_OVERLAY" + fi rc=0 # Piped: the advertised command, and the shape that turns an early exit # into curl:(56). @@ -379,27 +466,42 @@ jobs: # The gate no longer hard-stops on a non-apt distro: it warns that the # optional build tools are absent and carries on. Reaching this warning is # what proves the Linux gate did not stop the install. - # Past that point the only accepted failure is release lag: install.sh is - # taken from this ref but unsloth is installed from PyPI, and the released + # Past that point the accepted failure used to be release lag: install.sh + # came from this ref but unsloth from PyPI, and the released # studio/install_python_stack.py has no "skip the triton kernels when git - # is missing" guard, so it still fetches the git+https triton_kernels - # requirement on a machine that has no git. Once a release carries that - # guard this whole step retires to a plain success assertion. - if ! grep -q "Installing triton kernels (pip) failed" logs/install.log; then - echo "::error::fedora got past the dependency warning then failed for a new reason, not the known triton/git release lag" + # is missing" guard, so it fetched the git+https triton_kernels + # requirement on a machine with no git. This leg is now overlaid with this + # ref's Python (see the header), so that guard is this ref's own code and + # the triton failure must NOT come back. Accepting it here would be + # accepting a regression in the guard as if it were release lag. + if grep -q "Installing triton kernels (pip) failed" logs/install.log; then + echo "::error::triton kernels still failed with this ref's install_python_stack.py overlaid, so its no-git skip did not hold" exit 1 fi - echo "::warning::fedora fails only on triton_kernels (git+https) from the released unsloth; drop this step once a release ships the no-git skip" - exit 0 + # Nothing past the dependency warning is acceptable any more: the one + # tolerated failure was the released package lagging this ref, and the + # overlay removes that difference. A failure here is this ref's own. + echo "::error::fedora got past the dependency warning and still failed, with this ref's Python overlaid; there is no known-good outcome left to accept" + exit 1 fi # This ref still hard-exits on a non-apt package manager. Pin that message so # a bootstrap outage or an unrelated early exit is not tolerated as if it # were the intentional diagnostic. if ! grep -qiE "Automatic system package installation is supported on apt-based|Fedora/RHEL: sudo dnf install" logs/install.log; then - echo "::error::fedora leg failed neither at the unsupported-package-manager gate nor at the known triton/git release lag" + echo "::error::fedora leg failed neither at the unsupported-package-manager gate nor past the dependency warning" exit 1 fi + # See the macOS job: proves the leg is testing what its matrix row claims. + - name: Assert this ref's Python was really put under test + if: matrix.overlay && inputs.installer_source != 'published' && steps.install_root.outcome == 'success' + run: | + grep -q "CI: overlaying source checkout" logs/install.log || { + echo "::error::this leg is marked overlay: true but the installer never overlaid the checkout, so it only tested the released package" + exit 1 + } + echo "overlay applied; this leg exercised this ref's Python" + # nobuild only reads the log, so an installer that exits 0 having done nothing # satisfies it. These required Linux rows had no check that the install # produced anything runnable, unlike the WSL and Windows jobs. @@ -557,16 +659,29 @@ jobs: - os: windows-latest winget: 'visible' experimental: false + overlay: true # The no-winget path (LTSC / Server / managed corporate machines) falls back # to python.org + astral.sh and is untested today. It is also where # Ensure-VCRedist silently does not run, leaving torch unable to load -- # hence the explicit `import torch` assert below. + # + # This leg fails at studio/setup.ps1:1652-1670, the unconditional + # "Git is required but could not be installed automatically" gate: no winget + # means no way to fetch git, and setup.ps1 refuses to continue without it. + # Before the overlay that failure came out of the RELEASED setup.ps1 and said + # nothing about this ref. It now comes out of this ref's own copy, which + # still carries the same gate, so the leg is red for a real and currently + # unfixed reason rather than for an untestable one. Relaxing that gate to + # `--local` and llama.cpp source builds is what turns it green, and this + # overlay is what will let this workflow see it happen. - os: windows-latest winget: 'masked' experimental: false + overlay: true - os: windows-11-arm winget: 'visible' experimental: true + overlay: true steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -723,6 +838,9 @@ jobs: - name: Install id: install shell: pwsh + env: + # Empty, and therefore ignored by install.ps1, on the non-overlay legs. + UNSLOTH_CI_SOURCE_OVERLAY: ${{ matrix.overlay && inputs.installer_source != 'published' && github.workspace || '' }} run: | $ErrorActionPreference = 'Continue' # No -SkipTorch: install.ps1 has no param block and its parser matches @@ -734,6 +852,17 @@ jobs: Write-Host "installer exit code: $rc" exit $rc + # See the macOS job: proves the leg is testing what its matrix row claims. + - name: Assert this ref's Python was really put under test + if: matrix.overlay && inputs.installer_source != 'published' && steps.install.outcome == 'success' + shell: pwsh + run: | + if (-not (Select-String -Path logs/install.log -Pattern 'CI: overlaying source checkout' -SimpleMatch -Quiet)) { + Write-Host '::error::this leg is marked overlay: true but the installer never overlaid the checkout, so it only tested the released package' + exit 1 + } + Write-Host "overlay applied; this leg exercised this ref's Python" + - name: Assert torch loads, and record what that does and does not prove if: steps.install.outcome == 'success' shell: pwsh diff --git a/install.ps1 b/install.ps1 index 0b06cb3ea1..efc3d59da6 100644 --- a/install.ps1 +++ b/install.ps1 @@ -2623,6 +2623,44 @@ exit 0 } } + # ── CI only: overlay a source checkout over the package just installed ── + # Mirrors the same block in install.sh. Not a consumer knob: no command-line + # switch, absent from the usage text, and ignored unless + # UNSLOTH_CI_SOURCE_OVERLAY names a directory holding a pyproject.toml. + # + # Why it exists: the clean-machine legs run THIS script from a branch, but + # the script installs unsloth from PyPI, which is the consumer path and must + # stay that way. Everything Python-side is then read out of the released + # wheel -- studio/setup.ps1, studio/install_python_stack.py, and every + # requirements/constraints file resolved through Path(__file__) -- so a + # branch could not be validated by the very workflow that exists to validate + # it. The `& $UnslothExe studio setup` call below goes through the CLI, and + # an editable overlay makes _PACKAGE_ROOT in unsloth_cli/commands/studio.py + # resolve to the working tree by PEP 660 __file__, exactly as the --local + # note on the Tauri overlay above describes, so setup.ps1 comes from the + # branch with no further change here. + # + # --local is deliberately NOT used for this: it also installs + # `unsloth-zoo @ git+https://github.com/unslothai/unsloth-zoo`, which + # genuinely requires git, and git absence is exactly what the masked leg + # proves. This overlay is editable + --no-deps only. It resolves no + # dependencies, clones nothing, and builds only unsloth's own pure-Python + # metadata, so it still works with git, cmake and MSVC all missing. + if ($env:UNSLOTH_CI_SOURCE_OVERLAY) { + $CiOverlayRoot = $env:UNSLOTH_CI_SOURCE_OVERLAY + if (-not (Test-Path -LiteralPath (Join-Path $CiOverlayRoot "pyproject.toml"))) { + Write-Host "[ERROR] UNSLOTH_CI_SOURCE_OVERLAY is set to '$CiOverlayRoot' but there is no pyproject.toml there." -ForegroundColor Red + return (Exit-InstallFailure "UNSLOTH_CI_SOURCE_OVERLAY has no pyproject.toml: $CiOverlayRoot") + } + substep "CI: overlaying source checkout (editable, no deps): $CiOverlayRoot" + # Retry: the editable build downloads its pinned build backend from PyPI, + # so it carries the same transient-network risk as every other step. + $CiOverlayExit = Invoke-InstallCommandRetry -Label "overlay CI source checkout" -Command { uv pip install --python $VenvPython --no-deps -e $CiOverlayRoot } + if ($CiOverlayExit -ne 0) { + return (Exit-InstallFailure "Failed to overlay the CI source checkout (exit code $CiOverlayExit)" $CiOverlayExit) + } + } + # ── Run studio setup ── # setup.ps1 will handle installing Git, CMake, Visual Studio Build Tools, # CUDA Toolkit, and other dependencies automatically via winget. Node.js is diff --git a/install.sh b/install.sh index 376daa8fab..a2f233fbad 100755 --- a/install.sh +++ b/install.sh @@ -4068,6 +4068,38 @@ if [ "$SKIP_TORCH" = false ] && [ -n "${TORCH_INDEX_URL:-}" ]; then fi fi +# ── CI only: overlay a source checkout over the package just installed ── +# Not a consumer knob: no command-line flag, absent from --help, and ignored +# unless UNSLOTH_CI_SOURCE_OVERLAY names a directory holding a pyproject.toml. +# +# Why it exists: the clean-machine legs run THIS script from a branch, but the +# script installs unsloth from PyPI, which is the consumer path and must stay +# that way. Everything Python-side is then read out of the released wheel -- +# studio/setup.sh, studio/setup.ps1, studio/install_python_stack.py, and every +# requirements/constraints file it resolves through Path(__file__) -- so a +# branch could not be validated by the very workflow that exists to validate +# it. Overlaying the checkout as an editable install re-points import studio at +# the working tree, and the existing importlib.resources lookup below then +# finds the branch's setup.sh with no further change. +# +# --local is deliberately NOT used for this: it also installs +# `unsloth-zoo @ git+https://github.com/unslothai/unsloth-zoo`, which genuinely +# requires git, and git absence is exactly what these legs prove. This overlay +# is editable + --no-deps only. It resolves no dependencies, clones nothing, +# and builds only unsloth's own pure-Python metadata, so it still works with +# git, cmake and the C/C++ compilers all missing. +if [ -n "${UNSLOTH_CI_SOURCE_OVERLAY:-}" ]; then + if [ ! -f "$UNSLOTH_CI_SOURCE_OVERLAY/pyproject.toml" ]; then + echo "[ERROR] UNSLOTH_CI_SOURCE_OVERLAY is set to '$UNSLOTH_CI_SOURCE_OVERLAY' but there is no pyproject.toml there." >&2 + exit 1 + fi + substep "CI: overlaying source checkout (editable, no deps): $UNSLOTH_CI_SOURCE_OVERLAY" + # Retry: the editable build downloads its pinned build backend from PyPI, so + # it carries the same transient-network risk as every other install step. + run_install_cmd_retry "overlay CI source checkout" uv pip install --python "$_VENV_PY" \ + --no-deps -e "$UNSLOTH_CI_SOURCE_OVERLAY" +fi + # ── Run studio setup ── tauri_log "STEP" "Running Unsloth setup" # When --local, use the repo's own setup.sh directly. diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py index 4004a3b048..dbf93f638d 100644 --- a/studio/install_python_stack.py +++ b/studio/install_python_stack.py @@ -2736,6 +2736,11 @@ def pip_install_try( env = _install_env_for_cmd(cmd), ) if result.returncode == 0: + # Same reasoning as pip_install: a successful install that built from + # source is exactly what the clean-machine `nobuild` assert exists to + # catch, and it can only see what reaches the log. + if VERBOSE and result.stdout: + print(_redact_install_output(result.stdout)) return True if VERBOSE and result.stdout: # pip/uv echo index URLs (credentials included) in failure output. @@ -2791,6 +2796,17 @@ def pip_install( **_windows_hidden_subprocess_kwargs(), ) if result.returncode == 0: + # Echo the successful output under UNSLOTH_VERBOSE, the same way + # install.sh's run_install_cmd does. Dropping it made the whole + # dependency phase invisible to anything reading the install log: + # .github/scripts/clean-machine-assert.sh's `nobuild` check greps + # for uv's "Building ==", so a source build here -- and + # this is the step that installs studio.txt, where an sdist-only + # dependency actually shows up -- left it reporting "built: none" + # and the leg green. Redacted, because uv echoes index URLs with + # credentials in them. + if VERBOSE and result.stdout: + print(_redact_install_output(result.stdout)) return print(_red(f" uv failed, falling back to pip...")) if result.stdout: From afbdaa09b7d9bffc3bcf315065300ad8d4fde9b2 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 22:47:33 +0000 Subject: [PATCH 13/36] Allowlist the triton-kernels pure-Python sdist, and record why Windows on ARM is red The two ubuntu2404 root legs went red at "Assert no source build" reporting triton-kernels. That is not a regression in what the installer does. Those builds have always happened; they only became visible now that pip_install stopped discarding uv's output on success, which is what finally let the nobuild check read the dependency phase at all. So the question was whether each build actually needs a compiler. Checked against the real artifacts rather than assumed: openai-whisper 20250625, randomname 0.2.1, argbind 0.3.9 -- no version of any of the three has ever published a wheel; antlr4-python3-runtime is pinned at 4.9.3, below the first release that ships one. All four sdists use setuptools.build_meta, declare no ext_modules, and contain no .c/.cpp/.pyx/.rs file. Already allowlisted, correctly. triton-kernels is the same category and was the only name failing. It is pinned to the triton repo's python/triton_kernels subdirectory; that tree is 75 files of Python, a four-line pyproject.toml, no setup.py and no native source at all. The kernels are Triton DSL compiled at runtime, not at install time. It is also a direct URL the installer names itself rather than something resolution picked, and only Linux reaches it. It belongs in the allowlist, so add it with that reasoning written down. The allowlist match is now lowercased and underscore-folded on both sides. The requirement spells the package triton_kernels while uv prints triton-kernels, and an allowlist that matched only one spelling would pass by luck rather than by intent. A plain pyarrow sdist is still caught. The two data-designer @ file:// plugin builds needed nothing: they are in-tree local paths, already dropped by the same rule that exempts the source overlay's own build. Separately, the windows-11-arm leg fails for a real reason and should keep failing. The ARM handling itself works, the log shows torchaudio being skipped and torch plus torchvision installing from wheels. What stops it is that pyarrow and hf-transfer publish no win_arm64 wheel at all, so uv falls back to their sdists and they fail on CMake configure and on openssl-sys wanting perl. That is a product gap on the platform, not a gap in the simulation, so the leg stays experimental and keeps reporting it. Record that above the matrix entry so the next reader does not re-diagnose it. --- .github/scripts/clean-machine-assert.sh | 22 ++++++++++++++++--- .../workflows/clean-machine-install-ci.yml | 11 ++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.github/scripts/clean-machine-assert.sh b/.github/scripts/clean-machine-assert.sh index c2c65708cf..22b29e090a 100755 --- a/.github/scripts/clean-machine-assert.sh +++ b/.github/scripts/clean-machine-assert.sh @@ -94,13 +94,28 @@ for check in "$@"; do ;; nobuild) - # "Built an sdist" is NOT "needed a compiler". Four packages on the macOS path - # are sdist-only PURE PYTHON (verified against cp313/macos-arm64): + # "Built an sdist" is NOT "needed a compiler". Every name below was checked + # against its actual sdist: setuptools.build_meta backend, no ext_modules, + # and not one .c/.cpp/.pyx/.rs file in the archive, so the PEP 517 build is + # a pure-Python metadata-and-copy step that completes with no compiler. # openai-whisper, argbind, randomname -- no version ever ships a wheel # antlr4-python3-runtime==4.9.3 -- pinned below the 4.13.2 wheel + # triton-kernels -- studio/backend/requirements/ + # triton-kernels.txt pins it to a git URL under the triton repo's + # python/triton_kernels subdirectory. That tree is 75 files of Python + # with a four-line pyproject.toml and no setup.py; the kernels are + # Triton DSL compiled at runtime, never at install time. It is also a + # direct URL the installer names itself, not something resolution + # chose, and only the Linux legs reach it (install_python_stack.py + # skips the step on Windows and macOS). # Failing on those is a false alarm, so the contract is "nothing needing a # COMPILER was built". UNSLOTH_ALLOW_SDIST extends the allowlist. - _allow="openai-whisper argbind randomname antlr4-python3-runtime ${UNSLOTH_ALLOW_SDIST:-}" + # + # Lowercased and underscore-folded on both sides, because a project's + # distribution name and the name uv prints can disagree on the separator: + # the requirement says triton_kernels, the build line says triton-kernels, + # and an allowlist that matched only one spelling would silently miss. + _allow="$(printf '%s' "openai-whisper argbind randomname antlr4-python3-runtime triton-kernels ${UNSLOTH_ALLOW_SDIST:-}" | tr 'A-Z_' 'a-z-')" if [ ! -f "$LOG" ]; then fail "nobuild requested but $LOG is missing" else @@ -123,6 +138,7 @@ for check in "$@"; do | grep -oiE "building wheel for [a-z0-9._-]+|building [a-z0-9._-]+(==| @ )" \ | tr 'A-Z' 'a-z' \ | sed -E -e 's/^building wheel for //' -e 's/^building //' -e 's/(==| @ )$//' \ + | tr '_' '-' \ | sort -u || true)" _bad="" for pkg in $_built; do diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index cd630f7b3b..44b29e472b 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -678,6 +678,17 @@ jobs: winget: 'masked' experimental: false overlay: true + # Windows on ARM gets as far as the dependency install and then stops on + # two packages that publish no win_arm64 wheel at all: + # pyarrow==25.0.0 (pulled in by datasets) -- PyPI has win_amd64 only, + # so uv falls back to the sdist and its CMake configure fails + # hf-transfer==0.1.9 -- a maturin/Rust sdist whose openssl-sys build + # script wants perl, which the image does not have + # The ARM-specific handling added for this platform is working: the log + # shows "windows on arm: skipping torchaudio", and torch 2.10.0+cpu and + # torchvision both install from wheels. The redness that remains is a + # real product gap on this platform, not a gap in the simulation, so the + # leg stays experimental and keeps reporting it rather than hiding it. - os: windows-11-arm winget: 'visible' experimental: true From 06d2725e090c0bc83dbc9012ef816ca42534a39f Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 23:18:06 +0000 Subject: [PATCH 14/36] Exercise the bundled Windows installer, and stop mislabelling installer sources Four things that let a leg go green while proving nothing. The desktop Windows job installed the bundle and launched it, and that was all. On a fresh profile preflight reports not_installed and the app sits on the install screen waiting for a click, so the process happily stays alive for 90 seconds without the bundled install.ps1 ever running. A bundle that shipped no install.ps1 resource, or a broken one, passed this job -- which is the packaged app failure the workflow exists to catch. macOS and Linux already invoke their bundled script directly; Windows now does the same, via the resource NSIS laid down next to the exe, invoked the way install.rs invokes it, then asserts the managed venv exists and can import torch. Its timeout goes to 60 minutes because a full torch install on a Windows runner is the slowest of the three. A manual run that selects installer_source: published only redirected the macOS and Linux jobs. WSL kept copying the checked-out install.sh and Windows kept running the checked-out install.ps1, so a run asking whether the script on unsloth.ai works reported on this ref under the published label. Both now honor the selection; install.ps1 advertises its own unsloth.ai URL, so published has a meaning on Windows too. Both branches stay empty on pull_request and push, so automatic runs are unchanged. The push-to-main filter listed only install.sh, install.ps1 and this workflow, while the PR filter also covers setup.sh, setup.ps1, install_python_stack.py and the clean-machine helpers. A direct push touching those skipped the workflow entirely, so the post-merge backstop never ran for the files the source overlay was added to cover. The two lists now match. Neither filter covered studio/backend/requirements, even though the overlay exists precisely so a constraints change is resolved on a machine with no compiler and no cached wheels. The update-smoke workflows cannot stand in: they start from a preinstalled Python and full developer tooling. --- .../workflows/clean-machine-install-ci.yml | 54 ++++++++++++++++--- .../desktop-app-clean-machine-ci.yml | 49 ++++++++++++++++- 2 files changed, 96 insertions(+), 7 deletions(-) diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index 44b29e472b..a2ddcdce62 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -57,13 +57,28 @@ on: - 'studio/setup.sh' - 'studio/setup.ps1' - 'studio/install_python_stack.py' + # The overlay exists so a constraints or requirements change is actually + # exercised here (see the header). Without these paths the one workflow that + # resolves them on a machine with no compiler and no wheels cached never runs + # for the PR that changes them, and the update-smoke jobs cannot stand in: + # they start from a preinstalled Python and full developer tooling. + - 'studio/backend/requirements/**' - '.github/scripts/clean-machine-*.sh' - '.github/workflows/clean-machine-install-ci.yml' push: branches: [main] + # Same list as the PR filter. A direct push to main that changed setup.sh, + # setup.ps1, install_python_stack.py, a requirements file or a clean-machine + # helper skipped this workflow entirely, so the post-merge run that is supposed + # to be the backstop for exactly those files never happened. paths: - 'install.sh' - 'install.ps1' + - 'studio/setup.sh' + - 'studio/setup.ps1' + - 'studio/install_python_stack.py' + - 'studio/backend/requirements/**' + - '.github/scripts/clean-machine-*.sh' - '.github/workflows/clean-machine-install-ci.yml' workflow_dispatch: inputs: @@ -579,11 +594,25 @@ jobs: # Only ca-certificates + curl: the advertised one-liner cannot start without # a transport. Everything else must come from the installer. wsl -d unsloth-ci -u root -- sh -c 'apt-get update -qq && apt-get install -y -qq --no-install-recommends ca-certificates curl' 2>&1 | Tee-Object -FilePath logs/wsl-bootstrap.log - # Copy the script in rather than reaching across /mnt/c: a DrvFs path brings - # Windows permissions and CRLF risk, neither of which a real WSL user has. - $wslPath = (wsl -d unsloth-ci -- wslpath -a "$($env:GITHUB_WORKSPACE -replace '\\','/')/install.sh").Trim() - Write-Host "installer source in WSL: $wslPath" - wsl -d unsloth-ci -u root -- cp "$wslPath" /root/install.sh + # A dispatch that selects `published` is asking whether the script on + # unsloth.ai works; running the checked-out one here and reporting the leg + # green answered a different question under the same name. Empty on + # pull_request/push, so automatic runs stay on this ref. + if ('${{ inputs.installer_source }}' -eq 'published') { + Write-Host 'installer: published (unsloth.ai)' + wsl -d unsloth-ci -u root -- sh -c 'curl -fsSL https://unsloth.ai/install.sh -o /root/install.sh' + if ($LASTEXITCODE -ne 0) { + Write-Host '::error::could not fetch the published installer inside WSL' + exit 1 + } + } else { + # Copy the script in rather than reaching across /mnt/c: a DrvFs path + # brings Windows permissions and CRLF risk, neither of which a real WSL + # user has. + $wslPath = (wsl -d unsloth-ci -- wslpath -a "$($env:GITHUB_WORKSPACE -replace '\\','/')/install.sh").Trim() + Write-Host "installer source in WSL: $wslPath" + wsl -d unsloth-ci -u root -- cp "$wslPath" /root/install.sh + } # Piped, same shape as `curl ... | sh`, so an early exit still exposes the # broken pipe, but the script under test is this ref not production's. wsl -d unsloth-ci -u root -- sh -c 'cd /root && cat install.sh | sh' 2>&1 | Tee-Object -FilePath logs/wsl-install.log @@ -854,11 +883,24 @@ jobs: UNSLOTH_CI_SOURCE_OVERLAY: ${{ matrix.overlay && inputs.installer_source != 'published' && github.workspace || '' }} run: | $ErrorActionPreference = 'Continue' + # Windows ships its own published script (install.ps1:3), so `published` + # has a meaning here too. Running the checked-out one regardless made a + # dispatch that asked about unsloth.ai report on this ref instead. Empty on + # pull_request/push, so automatic runs stay on this ref. + $script = './install.ps1' + if ('${{ inputs.installer_source }}' -eq 'published') { + Invoke-WebRequest -Uri https://unsloth.ai/install.ps1 ` + -OutFile published-install.ps1 -UseBasicParsing -TimeoutSec 300 + $script = './published-install.ps1' + Write-Host 'installer: published (unsloth.ai)' + } else { + Write-Host "installer: this ref ($env:GITHUB_SHA)" + } # No -SkipTorch: install.ps1 has no param block and its parser matches # `--no-torch` only (112-142), so the token was silently dropped and every # Windows leg installed torch anyway. The assert below needs torch, so ask # for it explicitly rather than by accident. - & ./install.ps1 *>&1 | Tee-Object -FilePath logs/install.log + & $script *>&1 | Tee-Object -FilePath logs/install.log $rc = $LASTEXITCODE Write-Host "installer exit code: $rc" exit $rc diff --git a/.github/workflows/desktop-app-clean-machine-ci.yml b/.github/workflows/desktop-app-clean-machine-ci.yml index 8694cd4e1b..2079857d80 100644 --- a/.github/workflows/desktop-app-clean-machine-ci.yml +++ b/.github/workflows/desktop-app-clean-machine-ci.yml @@ -390,7 +390,9 @@ jobs: windows: name: desktop windows runs-on: windows-latest - timeout-minutes: 45 + # 60, not 45: this job now runs the bundled installer, and a full torch install + # on a Windows runner is the slowest of the three platforms. + timeout-minutes: 60 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -448,6 +450,51 @@ jobs: Write-Host "installed: $($found.FullName)" "APP_EXE=$($found.FullName)" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + - name: Run the bundled installer, the path first launch takes + shell: pwsh + run: | + # The launch step below only proves the process stayed alive. On a fresh + # profile preflight reports not_installed and the app waits for the user to + # click Install (use-tauri-backend.ts:252-254, startup-screen.tsx:388-389), + # so this job passed on a bundle whose embedded install.ps1 was missing or + # broken -- the packaged-app failure the workflow exists to catch, and the + # one thing the macOS and Linux rows now check and Windows did not. + # tauri.conf.json:56-59 ships install.ps1 as a bundle resource, so find it + # where NSIS put it and invoke it as install.rs:326-341 does. + $root = Split-Path -Parent $env:APP_EXE + $ps1 = Get-ChildItem -Path $root -Recurse -Filter 'install.ps1' -ErrorAction SilentlyContinue | + Select-Object -First 1 + if (-not $ps1) { + Write-Host '::error::the bundle ships no install.ps1 resource' + exit 1 + } + Write-Host "bundled installer: $($ps1.FullName)" + # --tauri rejects a custom studio home (install.ps1:189-215), so drop the + # workspace-scoped override the same way install.rs scrubs it (354-357). + Remove-Item Env:UNSLOTH_STUDIO_HOME -ErrorAction SilentlyContinue + & powershell.exe -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass ` + -File $ps1.FullName --tauri *>&1 | Tee-Object -FilePath logs/bundled-install.log + $rc = $LASTEXITCODE + Write-Host "bundled installer exit: $rc" + if ($rc -ne 0) { + Write-Host "::error::bundled installer exited $rc" + exit $rc + } + # The exit code alone is not enough: it is the venv the app then boots from. + $py = Join-Path $env:USERPROFILE '.unsloth\studio\unsloth_studio\Scripts\python.exe' + if (-not (Test-Path $py)) { + Write-Host "::error::bundled installer left no venv at $py" + exit 1 + } + & $py -V + # install.rs passes only --tauri, so torch is part of first launch, and a + # venv that cannot import it is the unbootable environment from the report. + & $py -c "import torch; print('torch', torch.__version__)" + if ($LASTEXITCODE -ne 0) { + Write-Host '::error::the bundled install produced a venv with no working torch' + exit 1 + } + - name: Launch and prove it stays up shell: pwsh run: | From b905784f77eea359e81350820cf6abce58f15f04 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 23:54:57 +0000 Subject: [PATCH 15/36] Make the Linux and Windows desktop legs clean, and honour published on every macOS delivery The desktop workflow claims all three platforms are stripped, but only macOS and Windows had a strip step and the Windows one scrubbed the process PATH only. Both gaps let a bundle that needs a developer toolchain pass the one workflow whose premise is that it must not. Linux: the job ignored strip_toolchain entirely and ran the bundled install.sh with the runner's git, gcc, cmake and make in /usr/bin. clean-machine-env.sh now has a Linux --remove branch that moves the resolved tool binaries aside, recorded in restore.sh, and the job calls it plus `assert absent` after the apt step (the .deb install needs dpkg) and before the bundled installer, with a restore step to match macOS. The loop repeats per tool so a name present in both /usr/bin and /usr/local/bin is fully masked rather than half masked. Windows: rewriting $env:PATH does not survive the bundled install.ps1, which calls Refresh-SessionPath (318-337) and rebuilds $env:Path from the Machine and User registry values, and py.exe in C:\Windows reaches the toolcache whatever PATH says. Ported the on-disk toolcache rename, the Machine/User registry scrub and the py -3.11/-3.12/-3.13 start probe from clean-machine-install-ci.yml, so the strip is proven rather than assumed. Windows preflight: the log step was Test-Path, Get-Content and Select-String, none of which can fail, so an app that hangs before preflight passed on the 90 second liveness check alone. It now asserts a tauri.log exists and carries a `desktop_preflight completed disposition=` line, the same unconstrained check macOS and Linux already make. The disposition VALUE is deliberately not constrained: ManagedReady over an unbootable venv is the reported bug. installer_source on macOS: only the pipe delivery branched on it, so a `published` dispatch ran the checked-out script on six of the eight macOS rows while the run was labelled published. The script is now resolved once at the top of the Install step and used by the file and tauri deliveries; pipe still re-fetches through the live transport, because that is half of what it tests. Linux, WSL and Windows already honoured the input. Also shortened the comments across the changed files, keeping the reasoning that says why each check exists. --- .github/scripts/clean-machine-assert.sh | 78 ++-- .github/scripts/clean-machine-env.sh | 49 ++- .../workflows/clean-machine-install-ci.yml | 407 +++++++++--------- .../desktop-app-clean-machine-ci.yml | 217 +++++++--- install.ps1 | 36 +- install.sh | 29 +- studio/install_python_stack.py | 22 +- 7 files changed, 455 insertions(+), 383 deletions(-) diff --git a/.github/scripts/clean-machine-assert.sh b/.github/scripts/clean-machine-assert.sh index 22b29e090a..d08ba5764b 100755 --- a/.github/scripts/clean-machine-assert.sh +++ b/.github/scripts/clean-machine-assert.sh @@ -4,14 +4,14 @@ # # Assert the clean-machine contract after an install attempt. # -# absent The toolchain really was absent for the whole run. Guards against a leg -# that "passed" only because masking silently failed, or because the -# installer quietly installed Xcode CLT behind our back. +# absent The toolchain really was absent for the whole run. Catches a leg that +# "passed" only because masking silently failed, or because the installer +# quietly installed Xcode CLT behind our back. # notools The trace recorded no compiler/git/brew invocation (trace mode). # nobuild The wheels-only contract: no "Building wheel" from pip, no -# "Building ==" from uv. Needs UNSLOTH_VERBOSE=1, else -# run_install_cmd (install.sh:193-243) discards the uv output on success -# and there is nothing here to read. +# "Building ==" from uv. Needs UNSLOTH_VERBOSE=1, or +# run_install_cmd (install.sh:193-243) discards uv's output on success +# and there is nothing to read. # # Usage: bash .github/scripts/clean-machine-assert.sh absent notools nobuild set -uo pipefail @@ -28,8 +28,8 @@ for check in "$@"; do absent) # Deliberately NOT `command -v`: on a virgin Mac /usr/bin/{git,cc} EXIST as CLT - # stubs, so `command -v` succeeds and only RUNNING them fails ("xcrun: error: - # invalid active developer path"). The honest invariant is: must not WORK. + # stubs, so `command -v` succeeds and only RUNNING them fails ("invalid active + # developer path"). The honest invariant is: must not WORK. if xcode-select -p >/dev/null 2>&1; then fail "xcode-select -p still resolves to $(xcode-select -p 2>/dev/null); not a clean Mac" else @@ -38,10 +38,9 @@ for check in "$@"; do for tool in git cc clang cmake; do command -v "$tool" >/dev/null 2>&1 || { ok "$tool not on PATH"; continue; } if "$tool" --version >/dev/null 2>&1; then - # On Intel runners /usr/bin/git is not CLT-provided and keeps working once - # the CLT are gone, so no masking can remove it. cc and clang do become - # stubs and the macOS consumer path needs no git, so report rather than - # call the simulation broken. + # On Intel runners /usr/bin/git is not CLT-provided and survives their + # removal, so no masking can take it away. cc and clang do become stubs and + # the macOS consumer path needs no git, so report rather than fail. case " ${UNSLOTH_CLEAN_ALLOW_WORKING:-} " in *" $tool "*) echo "[assert] NOTE $tool still works ($(command -v "$tool")); allowed on this runner" @@ -73,10 +72,9 @@ for check in "$@"; do [ -n "$tool" ] || continue case " $allow " in *" $tool "*) continue ;; esac # `xcode-select -p` only ASKS whether a toolchain is selected; the installer - # has to ask, and the point of the fix is that it carries on without one. - # Counting the question as toolchain USE would fail the very leg that proves - # the toolchain was never used. `--install`, which pops the CLT installer, - # stays a hit. + # has to ask, and the fix is that it carries on without one. Counting the + # question as USE would fail the very leg proving the toolchain went + # untouched. `--install`, which pops the CLT installer, stays a hit. if [ "$tool" = "xcode-select" ]; then case "$rest" in -p|--print-path|-v|--version|"") continue ;; @@ -94,44 +92,38 @@ for check in "$@"; do ;; nobuild) - # "Built an sdist" is NOT "needed a compiler". Every name below was checked - # against its actual sdist: setuptools.build_meta backend, no ext_modules, - # and not one .c/.cpp/.pyx/.rs file in the archive, so the PEP 517 build is - # a pure-Python metadata-and-copy step that completes with no compiler. + # "Built an sdist" is NOT "needed a compiler", so the contract is "nothing + # needing a COMPILER was built". Every name below was checked against its + # actual sdist: setuptools.build_meta backend, no ext_modules, not one + # .c/.cpp/.pyx/.rs file, so its PEP 517 build is a pure-Python copy step. # openai-whisper, argbind, randomname -- no version ever ships a wheel # antlr4-python3-runtime==4.9.3 -- pinned below the 4.13.2 wheel - # triton-kernels -- studio/backend/requirements/ - # triton-kernels.txt pins it to a git URL under the triton repo's - # python/triton_kernels subdirectory. That tree is 75 files of Python - # with a four-line pyproject.toml and no setup.py; the kernels are - # Triton DSL compiled at runtime, never at install time. It is also a - # direct URL the installer names itself, not something resolution - # chose, and only the Linux legs reach it (install_python_stack.py - # skips the step on Windows and macOS). - # Failing on those is a false alarm, so the contract is "nothing needing a - # COMPILER was built". UNSLOTH_ALLOW_SDIST extends the allowlist. + # triton-kernels -- requirements/triton-kernels.txt pins a git URL under + # the triton repo's python/triton_kernels subdirectory: 75 Python files, + # a four-line pyproject.toml, no setup.py, kernels compiled at runtime. + # A direct URL the installer names itself, not something resolution + # chose, and only the Linux legs reach it (install_python_stack.py skips + # the step on Windows and macOS). + # UNSLOTH_ALLOW_SDIST extends the allowlist. # - # Lowercased and underscore-folded on both sides, because a project's - # distribution name and the name uv prints can disagree on the separator: - # the requirement says triton_kernels, the build line says triton-kernels, - # and an allowlist that matched only one spelling would silently miss. + # Lowercased and underscore-folded on both sides: a distribution name and the + # name uv prints can disagree on the separator (requirement triton_kernels vs + # build line triton-kernels), and a one-spelling allowlist silently misses. _allow="$(printf '%s' "openai-whisper argbind randomname antlr4-python3-runtime triton-kernels ${UNSLOTH_ALLOW_SDIST:-}" | tr 'A-Z_' 'a-z-')" if [ ! -f "$LOG" ]; then fail "nobuild requested but $LOG is missing" else # uv does NOT use pip's phrasing: it prints `Building ==` to # stderr (astral-sh/uv#11165), so the pip-only pattern left _built empty on - # every uv source build. Match both spellings. Requiring `==` or ` @ ` after - # the name keeps this off the installer's own lowercase "building frontend..." + # every uv source build. Match both. Requiring `==` or ` @ ` after the name + # keeps this off the installer's own lowercase "building frontend..." # progress text. Strip ANSI first so a coloured run (FORCE_COLOR) parses. # - # `Building @ file://...` is dropped before the names are read: a - # local-path build is something the caller pointed at (install.sh --local, - # or the UNSLOTH_CI_SOURCE_OVERLAY editable overlay the CI legs use to put - # the branch's Python code under test), never a dependency that resolution - # chose. Dependencies from an index always print `==`, so - # this drops no real signal -- a genuine sdist pulled from PyPI is still - # caught, including one named unsloth. + # `Building @ file://...` is dropped first: a local-path build is + # something the caller pointed at (install.sh --local, or the + # UNSLOTH_CI_SOURCE_OVERLAY editable overlay), never a dependency resolution + # chose. Index dependencies always print `==`, so no signal is + # lost: a genuine sdist from PyPI is still caught, including one named unsloth. _esc=$(printf '\033') _built="$(sed -E "s/${_esc}\[[0-9;]*[A-Za-z]//g" "$LOG" 2>/dev/null \ | grep -viE "building [a-z0-9._-]+ @ file://" \ diff --git a/.github/scripts/clean-machine-env.sh b/.github/scripts/clean-machine-env.sh index 1ed51bbac9..138ba922d0 100755 --- a/.github/scripts/clean-machine-env.sh +++ b/.github/scripts/clean-machine-env.sh @@ -3,16 +3,16 @@ # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. # # Simulate a virgin developer machine on a GitHub-hosted runner. Two modes, because -# "the tool is absent" and "the installer never called the tool" cannot be simulated -# by the same mechanism: +# "the tool is absent" and "the installer never called the tool" need different +# mechanisms: # # mask Make the toolchain genuinely ABSENT: scrub PATH to OS defaults and (with -# --remove) move the real toolchain aside, so `command -v git` correctly -# FAILS, as on a clean Mac. A failing "poison shim" would do the opposite -- -# `command -v` finds it and reports the tool as present -- so no shims here. -# trace Leave the toolchain working but route it through logging wrappers that log -# the call then exec the real binary, proving whether the installer ever -# REACHES for a compiler/git without changing behaviour. +# --remove) move the real toolchain aside so `command -v git` correctly +# FAILS. Deliberately no "poison shims": a failing shim is still FOUND by +# `command -v`, which reports the tool as present, the opposite of clean. +# trace Leave the toolchain working behind wrappers that log the call then exec +# the real binary, answering whether the installer ever REACHES for a +# compiler/git without changing behaviour. # # Writes shell exports to $CLEAN_ENV_FILE (default ./clean-machine.env) to `source`; # nothing is exported globally, so other steps keep a normal environment. @@ -81,10 +81,10 @@ if [ "$MODE" = "mask" ]; then } >> "$ENV_FILE" if [ "$REMOVE" = "1" ] && [ "$OS" = "Darwin" ]; then - # Best-effort real removal; each step is independent and recorded in restore.sh - # so an `if: always()` step can put the runner back. xcode_select_link is exactly - # what `xcode-select -p` reads, so removing it reproduces a virgin Mac's gate. - # `xcode-select --reset` is NOT enough: it can reselect a full Xcode.app. + # Best effort, each step independent and recorded in restore.sh so an + # `if: always()` step can put the runner back. xcode_select_link is what + # `xcode-select -p` reads, so removing it reproduces a virgin Mac's gate; + # `xcode-select --reset` is NOT enough, it can reselect a full Xcode.app. if [ -e /var/db/xcode_select_link ]; then if sudo rm -f /var/db/xcode_select_link 2>/dev/null; then note "removed /var/db/xcode_select_link" @@ -106,8 +106,7 @@ if [ "$MODE" = "mask" ]; then # Xcode.app must go too: with the link removed AND CommandLineTools moved, # `xcode-select -p` still does not fail, it falls through to the image's Xcode # bundle (observed: /Applications/Xcode_16.4.app/Contents/Developer), which - # re-arms /usr/bin/{git,cc} and silently un-cleans the machine. A rename is - # instant regardless of bundle size: same filesystem, no copy. + # re-arms /usr/bin/{git,cc}. A rename is instant whatever the bundle size. for app in /Applications/Xcode*.app; do [ -d "$app" ] || continue if sudo mv "$app" "${app}.masked" 2>/dev/null; then @@ -128,6 +127,28 @@ if [ "$MODE" = "mask" ]; then fi done fi + + if [ "$REMOVE" = "1" ] && [ "$OS" = "Linux" ]; then + # A hosted Linux runner keeps git, gcc, cmake and make in /usr/bin, which the PATH + # scrub has to keep, so absence must be made real: move the resolved binaries + # aside (recorded in restore.sh). Versioned siblings like gcc-11 survive, but a + # consumer install invokes the unsuffixed names, which is what `absent` checks. + for tool in $TOOLS; do + # Repeat per tool: a runner can carry the same name in /usr/bin and + # /usr/local/bin, and moving only the first leaves the second on PATH. + for _ in 1 2 3 4; do + real="$(command -v "$tool" 2>/dev/null || true)" + [ -n "$real" ] && [ -e "$real" ] || break + if sudo mv "$real" "$real.masked" 2>/dev/null; then + note "moved $real aside" + echo "sudo mv '$real.masked' '$real' 2>/dev/null || true" >> "$RESTORE" + else + note "WARN could not move $real" + break + fi + done + done + fi fi # ── trace ───────────────────────────────────────────────────────────────────── diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index a2ddcdce62..b000b3380a 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -16,33 +16,31 @@ # # ── What each leg actually puts under test ──────────────────────────────────── # install.sh / install.ps1 come from this ref, but they install unsloth FROM PyPI, -# because that is the consumer path and it has to stay that way. Everything -# Python-side is therefore read out of the RELEASED wheel: studio/setup.sh, -# studio/setup.ps1, studio/install_python_stack.py, and every requirements and -# constraints file those resolve through Path(__file__). Left alone, this workflow -# can only ever validate the two shell installers, and a branch that changes -# constraints.txt or setup.ps1 gets a green run that proves nothing about the -# change. The `Assert the Fedora outcome is a known one` step below was already -# carrying a hand-written workaround for exactly this. +# the consumer path, which has to stay that way. Everything Python-side is therefore +# read out of the RELEASED wheel: studio/setup.sh, setup.ps1, +# install_python_stack.py, and every requirements and constraints file those reach +# through Path(__file__). Left alone this workflow validates only the two shell +# installers, and a branch changing constraints.txt or setup.ps1 gets a green run +# that proves nothing about the change; the `Assert the Fedora outcome is a known +# one` step below was already working around exactly that. # -# So legs with `overlay: true` re-point the venv at this ref before studio setup -# runs, via UNSLOTH_CI_SOURCE_OVERLAY (install.sh / install.ps1, just above their -# "Run studio setup" section): a `--no-deps` editable install of the checkout. -# That makes `import studio` resolve to the working tree, so the existing -# setup-script lookup finds this ref's setup.sh / setup.ps1 and install_python_stack -# reads this ref's constraints. It is deliberately NOT `install.sh --local`: -# --local also installs `unsloth-zoo @ git+https://...`, which genuinely needs git, -# and git absence is the whole point of the masked legs. The overlay resolves no -# dependencies and clones nothing, so it still works with git, cmake and the -# compilers all gone. +# So `overlay: true` legs re-point the venv at this ref before studio setup runs, via +# UNSLOTH_CI_SOURCE_OVERLAY (install.sh / install.ps1, just above their "Run studio +# setup" section): a `--no-deps` editable install of the checkout. `import studio` +# then resolves to the working tree, so the existing setup-script lookup finds this +# ref's setup.sh / setup.ps1 and install_python_stack reads this ref's constraints. +# Deliberately NOT `install.sh --local`: that also installs +# `unsloth-zoo @ git+https://...`, which genuinely needs git, and git absence is the +# whole point of the masked legs. The overlay resolves nothing and clones nothing, so +# it survives git, cmake and the compilers all being gone. # # Legs left on `overlay: false`, and why: # mac */mask/pipe the `curl | sh` shape a user runs. Kept end-to-end on the # released package so a broken PyPI release still shows up. # mac macos-14/trace `notools` asserts the installer never reaches for git, and # the editable build itself calls `git rev-parse` / -# `git archive` through setuptools-scm's file finder. An -# overlay here would answer the leg's own question for it. +# `git archive` through setuptools-scm's file finder, so an +# overlay would answer the leg's own question for it. # linux ubuntu2404-nonroot dies at the elevation gate before a venv exists. # wsl only install.sh is copied into the distro; there is no # source tree inside WSL to overlay from. @@ -59,18 +57,18 @@ on: - 'studio/install_python_stack.py' # The overlay exists so a constraints or requirements change is actually # exercised here (see the header). Without these paths the one workflow that - # resolves them on a machine with no compiler and no wheels cached never runs - # for the PR that changes them, and the update-smoke jobs cannot stand in: - # they start from a preinstalled Python and full developer tooling. + # resolves them with no compiler and no cached wheels never runs for the PR that + # changes them, and the update-smoke jobs cannot stand in: they start from a + # preinstalled Python and full developer tooling. - 'studio/backend/requirements/**' - '.github/scripts/clean-machine-*.sh' - '.github/workflows/clean-machine-install-ci.yml' push: branches: [main] - # Same list as the PR filter. A direct push to main that changed setup.sh, - # setup.ps1, install_python_stack.py, a requirements file or a clean-machine - # helper skipped this workflow entirely, so the post-merge run that is supposed - # to be the backstop for exactly those files never happened. + # Same list as the PR filter. A direct push to main touching setup.sh, setup.ps1, + # install_python_stack.py, a requirements file or a clean-machine helper skipped + # this workflow entirely, so the post-merge backstop for exactly those files never + # happened. paths: - 'install.sh' - 'install.ps1' @@ -100,9 +98,8 @@ env: UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home # No wildcard bind -> no ifconfig.me / check-host.net calls on the startup path. UNSLOTH_STUDIO_DISABLE_PUBLIC_CHECK: '1' - # Without this, run_install_cmd (install.sh:193-243) sends every `uv pip install` - # to a temp file and DELETES it on success, so the `nobuild` assertion can only - # ever report "built: none". + # Without this, run_install_cmd (install.sh:193-243) sends every `uv pip install` to + # a temp file and DELETES it on success, so `nobuild` can only report "built: none". UNSLOTH_VERBOSE: '1' jobs: @@ -112,28 +109,28 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 40 continue-on-error: ${{ matrix.experimental }} - # Explicit legs, not a full cross-product: the interesting dimensions are - # (does the toolchain exist) x (how the script is delivered), not every pairing. + # Explicit legs, not a full cross-product: the interesting dimensions are (does + # the toolchain exist) x (how the script is delivered), not every pairing. strategy: fail-fast: false matrix: include: - # `overlay` decides whether this ref's Python code is put under test at all; - # see the header. The pipe legs stay on the released package on purpose. + # `overlay` decides whether this ref's Python is put under test at all; see + # the header. The pipe legs stay on the released package on purpose. # - # The reported failure, in the shape users run it. Default install (with - # torch) because that is what a consumer actually gets. + # The reported failure, in the shape users run it, with torch because that + # is what a consumer gets. - {os: macos-14, mode: mask, delivery: pipe, flags: '', experimental: false, overlay: false} - {os: macos-14, mode: mask, delivery: file, flags: '', experimental: false, overlay: true} # What the desktop app runs: no tty, stdin closed, TAURI markers on. - {os: macos-14, mode: mask, delivery: tauri, flags: '', experimental: false, overlay: true} - # Toolchain present but logged: does the installer ever reach for it? - # No overlay: the editable build calls git itself (setuptools-scm), which - # would plant the very evidence `notools` exists to look for. + # Toolchain present but logged: does the installer ever reach for it? No + # overlay: the editable build calls git itself (setuptools-scm), planting the + # very evidence `notools` looks for. - {os: macos-14, mode: trace, delivery: file, flags: '', experimental: false, overlay: false} # --no-torch is the one macOS path that can still want a compiler - # (sentencepiece has no guaranteed cp313 arm64 wheel), so probe it apart - # from the default path instead of letting it hide the gate under test. + # (sentencepiece has no guaranteed cp313 arm64 wheel), so probe it apart from + # the default path rather than let it hide the gate under test. - {os: macos-14, mode: mask, delivery: file, flags: '--no-torch', experimental: true, overlay: true} - {os: macos-15, mode: mask, delivery: pipe, flags: '', experimental: false, overlay: false} - {os: macos-26, mode: mask, delivery: file, flags: '', experimental: true, overlay: true} @@ -179,11 +176,11 @@ jobs: - name: Verify the trace actually records if: matrix.mode == 'trace' run: | - # `notools` reads an absence, so a shim dir that never reached PATH is - # indistinguishable from an installer that touched nothing, and the one leg - # carrying that assertion would pass no matter what the installer did. - # Prove the wrapper records before trusting an empty file. macOS never - # probes git off the --local path, so this must be an explicit call. + # `notools` reads an absence, so a shim dir that never reached PATH looks + # exactly like an installer that touched nothing, and the one leg carrying + # that assertion would pass whatever the installer did. Prove the wrapper + # records before trusting an empty file. macOS never probes git off the + # --local path, so the call has to be explicit. set -a; . ./clean-machine.env; set +a [ -n "$UNSLOTH_TOOL_TRACE" ] || { echo "::error::trace mode set no UNSLOTH_TOOL_TRACE"; exit 1; } git --version >/dev/null 2>&1 || true @@ -201,28 +198,41 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Withheld on PR: this step runs checked-out PR code. HF_TOKEN: ${{ github.event_name != 'pull_request' && secrets.HF_TOKEN || '' }} - # Empty, and therefore ignored by install.sh, on the non-overlay legs. - # Also empty for `installer_source: published`, where the script under test - # is production's and has no such hook: overlaying this ref's Python onto it - # would report on neither one honestly. + # Empty, and so ignored by install.sh, on the non-overlay legs. Empty for + # `installer_source: published` too: the script under test is then + # production's and has no such hook, and overlaying this ref's Python onto it + # would report on neither honestly. UNSLOTH_CI_SOURCE_OVERLAY: ${{ matrix.overlay && inputs.installer_source != 'published' && github.workspace || '' }} run: | set -a; . ./clean-machine.env; set +a set -o pipefail rc=0 FLAGS="${{ matrix.flags }}" + # A `published` dispatch asks whether unsloth.ai's script works. Only `pipe` + # honoured it, so six of the eight macOS rows ran the checked-out script and + # were still reported as published coverage. Resolve it once, here, for every + # delivery. Empty on pull_request/push, so automatic runs stay on this ref. + SCRIPT=install.sh + if [ "${{ inputs.installer_source }}" = "published" ]; then + curl -fsSL https://unsloth.ai/install.sh -o published-install.sh + SCRIPT=published-install.sh + echo "installer: published (unsloth.ai)" + else + echo "installer: this ref ($GITHUB_SHA)" + fi case "${{ matrix.delivery }}" in file) # Plain file execution isolates "installer logic broken" from # "curl-pipe delivery broken". - bash install.sh $FLAGS 2>&1 | tee logs/install.log || rc=$? + bash "$SCRIPT" $FLAGS 2>&1 | tee logs/install.log || rc=$? ;; pipe) # The shape users actually run. install.sh is ~150KB of top-level # statements, so an early `exit` leaves the writer with a closed pipe # -> `curl: (56)`. Piping a local file reproduces that faithfully - # without depending on unsloth.ai being current. This input is empty on - # pull_request/push, so only an explicit dispatch tests unsloth.ai. + # without depending on unsloth.ai being current. The published case + # re-fetches rather than piping $SCRIPT: the live transport is half of + # what this delivery tests. if [ "${{ inputs.installer_source }}" = "published" ]; then curl -fsSL https://unsloth.ai/install.sh | sh -s -- $FLAGS 2>&1 | tee logs/install.log || rc=$? else @@ -243,7 +253,7 @@ jobs: # before doing any work. The runner is ephemeral, so the real home is # as disposable as the override. env -u UNSLOTH_STUDIO_HOME \ - bash install.sh --tauri $FLAGS < /dev/null 2>&1 | tee logs/install.log || rc=$? + bash "$SCRIPT" --tauri $FLAGS < /dev/null 2>&1 | tee logs/install.log || rc=$? ;; esac echo "install_rc=$rc" >> "$GITHUB_OUTPUT" @@ -255,9 +265,9 @@ jobs: fi exit "$rc" - # Without this the gap comes back silently: install.sh ignores an unset - # UNSLOTH_CI_SOURCE_OVERLAY, so a typo in the matrix or the expression would - # put every leg back on the released wheel and nothing in the run would say so. + # Without this the gap returns silently: install.sh ignores an unset + # UNSLOTH_CI_SOURCE_OVERLAY, so a typo in the matrix or the expression puts every + # leg back on the released wheel with nothing in the run saying so. - name: Assert this ref's Python was really put under test if: matrix.overlay && inputs.installer_source != 'published' && steps.install.outcome == 'success' run: | @@ -272,10 +282,10 @@ jobs: run: | set -a; . ./clean-machine.env; set +a checks="nobuild" - # `absent` ran only BEFORE the install, so an installer that quietly - # selected the CLT or installed a compiler left the leg green while every - # later source build could succeed -- the exact behaviour the assert script - # says `absent` guards the whole run against. Re-run it after the install. + # `absent` ran only BEFORE the install, so an installer that quietly selected + # the CLT or installed a compiler left the leg green while every later source + # build could succeed, the exact behaviour `absent` claims to guard the whole + # run against. Re-run it afterwards. [ "${{ matrix.mode }}" = "mask" ] && checks="$checks absent" [ "${{ matrix.mode }}" = "trace" ] && checks="$checks notools" UNSLOTH_CLEAN_ALLOW_WORKING='${{ matrix.allow_working }}' \ @@ -285,9 +295,9 @@ jobs: if: steps.install.outcome == 'success' run: | set -a; . ./clean-machine.env; set +a - # The tauri leg cannot honour UNSLOTH_STUDIO_HOME (see Install), so it went - # to the legacy root: llama.cpp sits at /llama.cpp and the venv at - # /studio, so this is ~/.unsloth, not ~/.unsloth/studio. + # The tauri leg cannot honour UNSLOTH_STUDIO_HOME (see Install) and went to + # the legacy root, where llama.cpp sits at /llama.cpp and the venv at + # /studio: so ~/.unsloth, not ~/.unsloth/studio. if [ "${{ matrix.delivery }}" = "tauri" ]; then HOME_DIR="$HOME/.unsloth" else @@ -303,8 +313,8 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - # Two matrix rows differ only in `flags`, so flags must be in the name: - # artifacts are immutable per run and the second upload 409s. + # Two rows differ only in `flags`, so flags must be in the name: artifacts + # are immutable per run and the second upload 409s. name: clean-mac-${{ matrix.os }}-${{ matrix.mode }}-${{ matrix.delivery }}${{ matrix.flags && format('-{0}', matrix.flags) || '' }} path: | logs/ @@ -321,8 +331,8 @@ jobs: container: ${{ matrix.image }} timeout-minutes: 40 continue-on-error: ${{ matrix.experimental }} - # Container jobs default to `sh -e` (dash), where `set -o pipefail` is an - # "Illegal option" that kills the step before the installer even starts. + # Container jobs default to `sh -e` (dash), where `set -o pipefail` is an "Illegal + # option" that kills the step before the installer starts. defaults: run: shell: bash @@ -330,8 +340,8 @@ jobs: fail-fast: false matrix: include: - # Root + apt available: install.sh's _smart_apt_install should self-heal - # from a base image with no curl, git, gcc or cmake at all. + # Root + apt: install.sh's _smart_apt_install should self-heal from a base + # image with no curl, git, gcc or cmake at all. - label: ubuntu2404-root image: ubuntu:24.04 runner: ubuntu-latest @@ -342,17 +352,16 @@ jobs: runner: ubuntu-24.04-arm experimental: false overlay: true - # No elevation: today this hard-fails at install.sh:856-861. Expected - # failure; the point is to pin the message and prove it is actionable - # rather than a bare `curl: (56)`. No overlay: it never gets as far as a - # venv, so there would be nothing to overlay into. + # No elevation: today this hard-fails at install.sh:856-861. Expected; the + # point is to pin the message and prove it is actionable rather than a bare + # `curl: (56)`. No overlay: it never reaches a venv to overlay into. - label: ubuntu2404-nonroot image: ubuntu:24.04 runner: ubuntu-latest experimental: true overlay: false - # Non-apt: today this hard-fails at install.sh:2034. Expected failure; - # forces the decision on whether dnf/pacman/zypper get supported. + # Non-apt: today this hard-fails at install.sh:2034. Expected; forces the + # decision on whether dnf/pacman/zypper get supported. - label: fedora41 image: fedora:41 runner: ubuntu-latest @@ -366,16 +375,15 @@ jobs: printf '%-8s %s\n' "$t" "$(command -v $t 2>/dev/null || echo ABSENT)" done | tee /tmp/container-baseline.txt - # The advertised `curl | sh` cannot even start on an image without curl, so the + # The advertised `curl | sh` cannot start on an image without curl, so the # transport is provisioned apart from the installer's own dependencies. # Everything else stays absent. - name: Provision only the bootstrap transport run: | # tar and gzip ride along on the overlay legs: with no actions/checkout here - # (it needs git) the only way to get this ref's source into the container is - # to fetch and unpack an archive over the same transport. Neither is a - # compiler, git or cmake, so the clean-machine premise is untouched. Both - # are usually already in the base image; naming them just makes it certain. + # (it needs git) the only way in for this ref's source is an archive over the + # same transport. Neither is a compiler, git or cmake, so the premise holds. + # Both are usually in the base image already; naming them makes it certain. pkgs="ca-certificates curl" if [ "${{ matrix.overlay }}" = "true" ]; then pkgs="$pkgs tar gzip"; fi if command -v apt-get >/dev/null 2>&1; then @@ -386,8 +394,8 @@ jobs: # No actions/checkout on purpose: it needs git, and a container with git # preinstalled is not the clean machine under test. Fetch over the transport - # above, and fetch the INSTALLER from the same ref so these legs can validate a - # fix instead of only the published script. + # above, INSTALLER included, so these legs validate a fix and not just the + # published script. - name: Fetch installer + assert script for this ref run: | mkdir -p logs .github/scripts @@ -403,9 +411,9 @@ jobs: fi wc -l install.sh - # The overlay needs a source tree, and these legs deliberately have no + # The overlay needs a source tree and these legs deliberately have no # actions/checkout. codeload serves the same commit as a tarball over plain - # HTTPS, so this ref's Python code gets in without a git client. + # HTTPS, so this ref's Python gets in without a git client. - name: Fetch this ref's source tree for the overlay if: matrix.overlay && inputs.installer_source != 'published' run: | @@ -420,11 +428,11 @@ jobs: if: matrix.label == 'ubuntu2404-nonroot' run: | useradd -m tester - # Switching user without a login shell keeps the caller's environment, so - # the workflow-wide UNSLOTH_STUDIO_HOME follows tester in, and install.sh + # Switching user without a login shell keeps the caller's environment, so the + # workflow-wide UNSLOTH_STUDIO_HOME follows tester in, and install.sh # validates that override in _resolve_studio_destinations (503-559), long - # before the elevation gate (840-861). Without a writable target this leg - # dies on "cannot be created" instead of on "cannot elevate". + # before the elevation gate (840-861). Without a writable target this leg dies + # on "cannot be created" rather than "cannot elevate". mkdir -p "$UNSLOTH_STUDIO_HOME" # No sudo installed and not root -> exercises the "cannot elevate" branch. chown -R tester logs install.sh "$UNSLOTH_STUDIO_HOME" @@ -434,16 +442,16 @@ jobs: if: matrix.label != 'ubuntu2404-nonroot' run: | set -o pipefail - # Resolved here rather than in `env:` so it tracks the step's real working - # directory: a container job remaps the workspace and github.workspace is - # not something this needs to depend on. + # Resolved here, not in `env:`, so it tracks the step's real working + # directory: a container job remaps the workspace, and github.workspace is not + # something this needs to depend on. if [ -d ci-source ]; then export UNSLOTH_CI_SOURCE_OVERLAY="$PWD/ci-source" echo "overlaying this ref's source from $UNSLOTH_CI_SOURCE_OVERLAY" fi rc=0 - # Piped: the advertised command, and the shape that turns an early exit - # into curl:(56). + # Piped: the advertised command, and the shape that turns an early exit into + # curl:(56). cat install.sh | sh 2>&1 | tee logs/install.log || rc=$? echo "installer exit code: $rc" exit "$rc" @@ -456,16 +464,15 @@ jobs: su tester -c 'cat install.sh | sh' > logs/install.log 2>&1 || rc=$? echo "installer exit code: $rc" tail -40 logs/install.log - # It may legitimately fail; what must NOT happen is an unexplained exit or - # a bare broken-pipe error standing in for a real diagnosis. + # It may legitimately fail; what must NOT happen is an unexplained exit or a + # bare broken-pipe error standing in for a real diagnosis. if [ "$rc" != "0" ] && ! grep -qiE "sudo is not available|apt-get install|missing:|permission" logs/install.log; then echo "::error::unprivileged install failed with no actionable message" exit 1 fi - # This leg is continue-on-error like the nonroot one, so without the same check - # a bootstrap outage or an unrelated early exit would be tolerated exactly like - # the intentional diagnostic. + # continue-on-error like the nonroot leg, so without the same check a bootstrap + # outage or an unrelated early exit is tolerated like the intended diagnostic. - name: Assert the Fedora outcome is a known one if: always() && matrix.label == 'fedora41' run: | @@ -478,30 +485,28 @@ jobs: # install.sh comes from this ref, so which of the two accepted outcomes # applies depends on which dependency gate this ref carries. if grep -q "using prebuilt llama.cpp (missing:" logs/install.log; then - # The gate no longer hard-stops on a non-apt distro: it warns that the - # optional build tools are absent and carries on. Reaching this warning is - # what proves the Linux gate did not stop the install. - # Past that point the accepted failure used to be release lag: install.sh - # came from this ref but unsloth from PyPI, and the released - # studio/install_python_stack.py has no "skip the triton kernels when git - # is missing" guard, so it fetched the git+https triton_kernels - # requirement on a machine with no git. This leg is now overlaid with this - # ref's Python (see the header), so that guard is this ref's own code and - # the triton failure must NOT come back. Accepting it here would be - # accepting a regression in the guard as if it were release lag. + # The gate no longer hard-stops on a non-apt distro: it warns the optional + # build tools are absent and carries on, and reaching that warning is what + # proves the Linux gate did not stop the install. Past it, the accepted + # failure used to be release lag: install.sh came from this ref but unsloth + # from PyPI, and the released install_python_stack.py has no "skip the + # triton kernels when git is missing" guard, so it fetched the git+https + # triton_kernels requirement with no git. The overlay makes that guard this + # ref's own code, so the triton failure must NOT come back: accepting it + # would be accepting a regression in the guard as release lag. if grep -q "Installing triton kernels (pip) failed" logs/install.log; then echo "::error::triton kernels still failed with this ref's install_python_stack.py overlaid, so its no-git skip did not hold" exit 1 fi # Nothing past the dependency warning is acceptable any more: the one # tolerated failure was the released package lagging this ref, and the - # overlay removes that difference. A failure here is this ref's own. + # overlay removes that difference. echo "::error::fedora got past the dependency warning and still failed, with this ref's Python overlaid; there is no known-good outcome left to accept" exit 1 fi # This ref still hard-exits on a non-apt package manager. Pin that message so - # a bootstrap outage or an unrelated early exit is not tolerated as if it - # were the intentional diagnostic. + # a bootstrap outage or an unrelated early exit is not tolerated as the + # intended diagnostic. if ! grep -qiE "Automatic system package installation is supported on apt-based|Fedora/RHEL: sudo dnf install" logs/install.log; then echo "::error::fedora leg failed neither at the unsupported-package-manager gate nor past the dependency warning" exit 1 @@ -518,8 +523,8 @@ jobs: echo "overlay applied; this leg exercised this ref's Python" # nobuild only reads the log, so an installer that exits 0 having done nothing - # satisfies it. These required Linux rows had no check that the install - # produced anything runnable, unlike the WSL and Windows jobs. + # satisfies it. Unlike WSL and Windows, these required Linux rows had no check + # that the install produced anything runnable. - name: Assert the install is actually usable if: steps.install_root.outcome == 'success' run: | @@ -555,8 +560,8 @@ jobs: # environment, which cannot catch anything about a real WSL. # # No third-party action: the official Ubuntu rootfs plus `wsl --import` is - # deterministic and checksum-verifiable, and adds no supply-chain dependency to a - # repo that audits its lockfiles. + # deterministic and checksum-verifiable, adding no supply-chain dependency to a repo + # that audits its lockfiles. wsl: name: wsl ubuntu-24.04 runs-on: windows-latest @@ -584,20 +589,20 @@ jobs: } wsl --import unsloth-ci "$PWD/wsl-dist/instance" "$PWD/wsl-dist/rootfs.tar.gz" --version 2 wsl -d unsloth-ci -- uname -a - # A freshly imported rootfs is genuinely bare: no curl, git or compiler. - # That is the clean machine, not a simulation of one. + # A freshly imported rootfs is genuinely bare: no curl, git or compiler. The + # clean machine, not a simulation of one. wsl -d unsloth-ci -- sh -c 'for t in curl wget git gcc cmake python3 sudo; do printf "%-8s %s\n" "$t" "$(command -v $t || echo ABSENT)"; done' - name: Install inside WSL, piped exactly as documented shell: pwsh run: | - # Only ca-certificates + curl: the advertised one-liner cannot start without - # a transport. Everything else must come from the installer. + # Only ca-certificates + curl: the advertised one-liner cannot start without a + # transport. Everything else must come from the installer. wsl -d unsloth-ci -u root -- sh -c 'apt-get update -qq && apt-get install -y -qq --no-install-recommends ca-certificates curl' 2>&1 | Tee-Object -FilePath logs/wsl-bootstrap.log - # A dispatch that selects `published` is asking whether the script on - # unsloth.ai works; running the checked-out one here and reporting the leg - # green answered a different question under the same name. Empty on - # pull_request/push, so automatic runs stay on this ref. + # A dispatch selecting `published` asks whether unsloth.ai's script works; + # running the checked-out one and reporting the leg green answered a different + # question under the same name. Empty on pull_request/push, so automatic runs + # stay on this ref. if ('${{ inputs.installer_source }}' -eq 'published') { Write-Host 'installer: published (unsloth.ai)' wsl -d unsloth-ci -u root -- sh -c 'curl -fsSL https://unsloth.ai/install.sh -o /root/install.sh' @@ -606,22 +611,21 @@ jobs: exit 1 } } else { - # Copy the script in rather than reaching across /mnt/c: a DrvFs path - # brings Windows permissions and CRLF risk, neither of which a real WSL - # user has. + # Copy the script in rather than reach across /mnt/c: a DrvFs path brings + # Windows permissions and CRLF risk, neither of which a real WSL user has. $wslPath = (wsl -d unsloth-ci -- wslpath -a "$($env:GITHUB_WORKSPACE -replace '\\','/')/install.sh").Trim() Write-Host "installer source in WSL: $wslPath" wsl -d unsloth-ci -u root -- cp "$wslPath" /root/install.sh } # Piped, same shape as `curl ... | sh`, so an early exit still exposes the - # broken pipe, but the script under test is this ref not production's. + # broken pipe, but on the script this dispatch selected. wsl -d unsloth-ci -u root -- sh -c 'cd /root && cat install.sh | sh' 2>&1 | Tee-Object -FilePath logs/wsl-install.log $installRc = $LASTEXITCODE Write-Host "installer exit: $installRc" - # Printing the code discarded it. The CLI check in the next step does not - # compensate: install.sh links the `unsloth` shim (4174-4182) BEFORE it - # reports a failing studio/setup.sh (4219-4230), so a late setup failure - # leaves a shim whose --version succeeds and the whole job looked green. + # Printing the code discarded it, and the next step's CLI check does not + # compensate: install.sh links the `unsloth` shim (4174-4182) BEFORE it reports + # a failing studio/setup.sh (4219-4230), so a late setup failure leaves a shim + # whose --version succeeds and the job looked green. if ($installRc -ne 0) { Write-Host "::error::WSL installer exited $installRc" exit $installRc @@ -634,11 +638,10 @@ jobs: # The platform line proves the wsl branch was taken rather than plain linux. Select-String -Path logs/wsl-install.log -Pattern 'platform|\[TAURI:DIAG\]|wsl' -ErrorAction SilentlyContinue | Select-Object -First 10 - # Printing could not fail, and that alternation also matches - # `platform linux`: if detection regresses, every WSL branch is skipped and - # this job still passes as a plain-Linux install, the one thing no other job - # covers. `step` writes the label in reverse video, so strip ANSI first or - # an anchored match can never hit. + # Printing could not fail, and that alternation also matches `platform linux`: + # if detection regresses, every WSL branch is skipped and this job still passes + # as a plain-Linux install, the one thing no other job covers. `step` writes + # the label in reverse video, so strip ANSI or an anchored match never hits. $esc = [char]27 $platformLines = @( Get-Content logs/wsl-install.log -ErrorAction SilentlyContinue | @@ -691,33 +694,31 @@ jobs: overlay: true # The no-winget path (LTSC / Server / managed corporate machines) falls back # to python.org + astral.sh and is untested today. It is also where - # Ensure-VCRedist silently does not run, leaving torch unable to load -- - # hence the explicit `import torch` assert below. + # Ensure-VCRedist silently does not run, leaving torch unable to load, hence + # the explicit `import torch` assert below. # - # This leg fails at studio/setup.ps1:1652-1670, the unconditional - # "Git is required but could not be installed automatically" gate: no winget - # means no way to fetch git, and setup.ps1 refuses to continue without it. - # Before the overlay that failure came out of the RELEASED setup.ps1 and said - # nothing about this ref. It now comes out of this ref's own copy, which - # still carries the same gate, so the leg is red for a real and currently - # unfixed reason rather than for an untestable one. Relaxing that gate to - # `--local` and llama.cpp source builds is what turns it green, and this - # overlay is what will let this workflow see it happen. + # It fails at studio/setup.ps1:1652-1670, the unconditional "Git is required + # but could not be installed automatically" gate: no winget means no way to + # fetch git. Before the overlay that failure came out of the RELEASED setup.ps1 + # and said nothing about this ref; it now comes out of this ref's own copy, + # which carries the same gate, so the leg is red for a real and currently + # unfixed reason rather than an untestable one. Relaxing that gate to --local + # and llama.cpp source builds turns it green, and the overlay is what lets this + # workflow see that happen. - os: windows-latest winget: 'masked' experimental: false overlay: true - # Windows on ARM gets as far as the dependency install and then stops on - # two packages that publish no win_arm64 wheel at all: - # pyarrow==25.0.0 (pulled in by datasets) -- PyPI has win_amd64 only, - # so uv falls back to the sdist and its CMake configure fails - # hf-transfer==0.1.9 -- a maturin/Rust sdist whose openssl-sys build - # script wants perl, which the image does not have - # The ARM-specific handling added for this platform is working: the log - # shows "windows on arm: skipping torchaudio", and torch 2.10.0+cpu and - # torchvision both install from wheels. The redness that remains is a - # real product gap on this platform, not a gap in the simulation, so the - # leg stays experimental and keeps reporting it rather than hiding it. + # Windows on ARM reaches the dependency install and stops on two packages + # that publish no win_arm64 wheel at all: + # pyarrow==25.0.0 (via datasets) -- PyPI has win_amd64 only, so uv falls + # back to the sdist and its CMake configure fails + # hf-transfer==0.1.9 -- a maturin/Rust sdist whose openssl-sys build script + # wants perl, which the image does not have + # The ARM handling itself works: the log shows "windows on arm: skipping + # torchaudio", and torch 2.10.0+cpu and torchvision install from wheels. What + # is left is a real product gap on this platform, not a gap in the simulation, + # so the leg stays experimental and keeps reporting it. - os: windows-11-arm winget: 'visible' experimental: true @@ -735,19 +736,18 @@ jobs: run: | New-Item -ItemType Directory -Force -Path logs | Out-Null # Drop preinstalled Python, git, CMake, VS/LLVM and the WindowsApps aliases - # from PATH. A full Visual Studio uninstall is not realistic in CI (registry - # + vswhere discovery, slow, may need a reboot), so PATH and env scrubbing - # is the honest approximation, recorded as such. + # from PATH. A full Visual Studio uninstall is not realistic in CI (registry + + # vswhere discovery, slow, may need a reboot), so PATH and env scrubbing is the + # honest approximation, recorded as such. $drop = @('hostedtoolcache\windows\Python', 'WindowsApps', '\Git\', 'CMake', 'Microsoft Visual Studio', 'BuildTools', 'LLVM', 'MSYS', 'mingw', 'Strawberry') # winget is an app-execution alias in ...\Local\Microsoft\WindowsApps, which - # the blanket drop above removes on EVERY leg -- so winget=visible was - # silently running the same no-winget fallback as winget=masked. Resolve it - # before the scrub and hand it back through a shim, so the visible leg gets - # winget without the Store's python.exe alias back. windows-11-arm has no - # winget on the hosted image (actions/runner-images#14083), so only - # windows-latest can carry it. + # the blanket drop removes on EVERY leg, so winget=visible was silently running + # the same no-winget fallback as winget=masked. Resolve it before the scrub and + # hand it back through a shim: the visible leg gets winget without the Store's + # python.exe alias returning. windows-11-arm has no winget on the hosted image + # (actions/runner-images#14083), so only windows-latest can carry it. $wantWinget = ('${{ matrix.winget }}' -ne 'masked') -and ('${{ matrix.os }}' -eq 'windows-latest') $wingetCmd = Get-Command winget -ErrorAction SilentlyContinue $scrub = { @@ -774,7 +774,7 @@ jobs: } # Take the toolcache Python off disk, not just off PATH: py.exe lives in # C:\Windows (which must stay) and uv does its own interpreter discovery, so - # both reach the toolcache whatever PATH says -- which is how a leg printing + # both reach the toolcache whatever PATH says. That is how a leg printing # `python ABSENT` still installed with the runner's 3.13.14. foreach ($tc in @("$env:AGENT_TOOLSDIRECTORY\Python", 'C:\hostedtoolcache\windows\Python')) { if ($tc -and (Test-Path $tc)) { @@ -786,11 +786,11 @@ jobs: $newPath = ($kept -join ';') "PATH=$newPath" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 # install.ps1's Refresh-SessionPath (318-337, called at 1246/1278/1295/1360/ - # 1369/2797) rebuilds $env:Path from the Machine and User registry values, so - # scrubbing only the process PATH lasts until the first bootstrap refresh, - # after which Git/CMake/VS/LLVM are back and the rest of the install is no - # longer clean. The runner is ephemeral, so rewrite the registry copies too. - # Expand first: SetEnvironmentVariable rewrites REG_EXPAND_SZ as REG_SZ + # 1369/2797) rebuilds $env:Path from the Machine and User registry values, so a + # process-only scrub lasts until the first bootstrap refresh, after which + # Git/CMake/VS/LLVM are back and the rest of the install is not clean. The + # runner is ephemeral, so rewrite the registry copies too. Expand first: + # SetEnvironmentVariable rewrites REG_EXPAND_SZ as REG_SZ # (dotnet/runtime#1442). foreach ($scope in 'Machine','User') { $raw = [System.Environment]::GetEnvironmentVariable('Path', $scope) @@ -813,22 +813,21 @@ jobs: run: | $leaked = @() # `py` too: the launcher lives in C:\Windows, which the scrub keeps, and it - # finds the toolcache Python that the scrub only removed from PATH. + # finds the toolcache Python the scrub only removed from PATH. foreach ($t in 'python','py','git','cmake','cl') { $f = Get-Command $t -ErrorAction SilentlyContinue Write-Host ("{0,-8} {1}" -f $t, $(if ($f) { $f.Source } else { 'ABSENT' })) if ($f -and $t -ne 'py') { $leaked += "$t -> $($f.Source)" } } # The launcher binary may stay, but an interpreter it can still START is a - # leak: Find-CompatiblePython (install.ps1:1130-1153) probes `py` first, so - # any version registered outside the two renamed toolcache directories gets - # reused and Python bootstrap never runs. Exempting `py` without running it - # left that unchecked. + # leak: Find-CompatiblePython (install.ps1:1130-1153) probes `py` first, so any + # version registered outside the two renamed toolcache directories gets reused + # and Python bootstrap never runs. Exempting `py` left that unchecked. if (Get-Command py -ErrorAction SilentlyContinue) { - # -0p prints the launcher's REGISTRY view. The mask step renames the - # toolcache directory on disk but cannot rewrite those registry entries, - # so -0p keeps naming paths that no longer exist. It is context for a - # failure, never evidence of one -- only a probe that starts counts. + # -0p prints the launcher's REGISTRY view. The mask renames the toolcache + # directory on disk but cannot rewrite those entries, so -0p keeps naming + # paths that no longer exist: context for a failure, never evidence of one. + # Only a probe that actually STARTS counts. Write-Host "py -0p (stale registry entries; masked paths no longer exist on disk):" & py -0p 2>&1 | ForEach-Object { Write-Host " $_" } foreach ($v in '-3.11', '-3.12', '-3.13') { @@ -838,13 +837,12 @@ jobs: Write-Host ("py {0} -> exit {1}: {2}" -f $v, $rc, (($out | Out-String).Trim() -replace '\r?\n', ' / ')) if ($rc -eq 0) { $leaked += "py $v -> $out" } } - # A probe that FAILS is the outcome we want, but it leaves $LASTEXITCODE - # non-zero, and Get-Command/Write-Host are cmdlets that never reset it. - # The runner appends + # A FAILING probe is the outcome we want, but it leaves $LASTEXITCODE + # non-zero and cmdlets never reset it. The runner appends # if ((Test-Path -LiteralPath variable:\LASTEXITCODE)) { exit $LASTEXITCODE } - # to every pwsh step (actions/runner#351), so all three Windows legs - # exited 1 with no ::error:: printed, on machines that were in fact clean - # -- and never reached the Install step at all. + # to every pwsh step (actions/runner#351), so all three Windows legs exited 1 + # with no ::error:: printed, on machines that were in fact clean, and never + # reached the Install step. $global:LASTEXITCODE = 0 } # Printing alone could not fail, and the leg was green while not clean: run @@ -862,17 +860,16 @@ jobs: exit 1 } } elseif ('${{ matrix.os }}' -eq 'windows-latest' -and -not $winget) { - # Without this the visible leg quietly degrades into a second masked leg - # and no job in this workflow exercises the normal winget bootstrap. + # Without this the visible leg quietly degrades into a second masked leg and + # nothing in this workflow exercises the normal winget bootstrap. Write-Host '::error::winget is not resolvable on the visible leg; the winget bootstrap is not under test' exit 1 } foreach ($scope in 'Machine','User') { Write-Host ("{0} PATH after scrub: {1}" -f $scope, [System.Environment]::GetEnvironmentVariable('Path', $scope)) } - # Every failure above exits 1 explicitly, so reaching here means the machine - # is clean. Be explicit rather than leaving the runner's appended - # `exit $LASTEXITCODE` to decide. + # Every failure above exits 1 explicitly, so reaching here means clean. Be + # explicit rather than let the runner's appended `exit $LASTEXITCODE` decide. exit 0 - name: Install @@ -883,10 +880,10 @@ jobs: UNSLOTH_CI_SOURCE_OVERLAY: ${{ matrix.overlay && inputs.installer_source != 'published' && github.workspace || '' }} run: | $ErrorActionPreference = 'Continue' - # Windows ships its own published script (install.ps1:3), so `published` - # has a meaning here too. Running the checked-out one regardless made a - # dispatch that asked about unsloth.ai report on this ref instead. Empty on - # pull_request/push, so automatic runs stay on this ref. + # Windows ships its own published script (install.ps1:3), so `published` means + # something here too. Running the checked-out one regardless made a dispatch + # asking about unsloth.ai report on this ref. Empty on pull_request/push, so + # automatic runs stay on this ref. $script = './install.ps1' if ('${{ inputs.installer_source }}' -eq 'published') { Invoke-WebRequest -Uri https://unsloth.ai/install.ps1 ` @@ -898,8 +895,8 @@ jobs: } # No -SkipTorch: install.ps1 has no param block and its parser matches # `--no-torch` only (112-142), so the token was silently dropped and every - # Windows leg installed torch anyway. The assert below needs torch, so ask - # for it explicitly rather than by accident. + # Windows leg installed torch anyway. The assert below needs torch, so get it + # on purpose rather than by accident. & $script *>&1 | Tee-Object -FilePath logs/install.log $rc = $LASTEXITCODE Write-Host "installer exit code: $rc" @@ -923,10 +920,10 @@ jobs: # HONESTY NOTE: the hosted image ships the VC++ 2015-2022 runtime in System32 # and it cannot be removed without breaking the runner, so `import torch` # succeeding here does NOT prove a genuinely clean no-winget machine has the - # runtime: Test-VCRedistInstalled (studio/setup.ps1:875) finds the - # preinstalled DLL and Ensure-VCRedist (891) short-circuits before it needs - # winget. Record that, then assert what CAN fail: torch imports, and the - # masked leg really did take the no-winget path. + # runtime: Test-VCRedistInstalled (setup.ps1:875) finds the preinstalled DLL + # and Ensure-VCRedist (891) short-circuits before it needs winget. Record that, + # then assert what CAN fail: torch imports, and the masked leg really did take + # the no-winget path. $sys32 = Join-Path $env:WINDIR 'System32\vcruntime140_1.dll' Write-Host "preinstalled System32 vcruntime140_1.dll: $(Test-Path $sys32)" $py = Join-Path $env:UNSLOTH_STUDIO_HOME 'unsloth_studio\Scripts\python.exe' diff --git a/.github/workflows/desktop-app-clean-machine-ci.yml b/.github/workflows/desktop-app-clean-machine-ci.yml index 2079857d80..0bbcfc5938 100644 --- a/.github/workflows/desktop-app-clean-machine-ci.yml +++ b/.github/workflows/desktop-app-clean-machine-ci.yml @@ -11,17 +11,16 @@ # # Hosted runners have no interactive desktop session, so "runs" means: the bundle # installs / mounts / extracts, the binary is present, of the right architecture, and -# passes the gatekeeper checks a user would hit (macOS quarantine + codesign, Windows -# installer exit), the process STAYS UP past its preflight (where an unhappy app dies), -# and it writes tauri.log showing the preflight disposition -- the field that read +# clears the gatekeeper checks a user would hit (macOS quarantine + codesign, Windows +# installer exit); the process STAYS UP past its preflight (where an unhappy app dies); +# and it writes tauri.log with a preflight disposition -- the field that read # `ManagedReady` over an unbootable venv in the bug report. Linux gets the strongest # check: a real webview under Xvfb. name: Desktop app clean machine on: - # Also on PRs touching this job or the machine-stripping scripts: workflow_dispatch - # alone cannot validate a change to the job itself, because dispatch resolves the + # Also on PRs touching this job or the stripping scripts: dispatch resolves the # workflow from the DEFAULT branch, so a new or edited file on a feature branch can # never be dispatched and would first run only after merging blind. pull_request: @@ -55,14 +54,13 @@ permissions: contents: read env: - # release-desktop.yml publishes into github.repository, so a nightly aimed - # anywhere else goes green over a broken production bundle. - # unsloth-test/unsloth-test holds one frozen release, so the schedule was - # re-testing the same fixture forever. + # release-desktop.yml publishes into github.repository, so a nightly aimed anywhere + # else goes green over a broken production bundle. unsloth-test/unsloth-test holds + # one frozen release, so the schedule was re-testing the same fixture forever. REL_REPO: ${{ inputs.release_repo || github.repository }} - # Empty unless dispatched: a pinned tag is an immutable fixture, so a nightly - # against it could never catch a newly published broken bundle. Each download step - # resolves the newest desktop-v* release when this is empty. + # Empty unless dispatched: a pinned tag is an immutable fixture, so a nightly against + # it could never catch a newly published broken bundle. Each download step resolves + # the newest desktop-v* release when this is empty. REL_TAG: ${{ inputs.release_tag || '' }} UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home UNSLOTH_STUDIO_DISABLE_PUBLIC_CHECK: '1' @@ -108,10 +106,10 @@ jobs: ls -la dl - name: Strip the developer toolchain - # `inputs` exists only for workflow_dispatch, so on pull_request and schedule - # `inputs.strip_toolchain` is '' -- and loose equality coerces both '' and - # false to 0, making `!= false` FALSE, so automatic runs would keep the hosted - # toolchain this workflow exists to remove. Gate on the event instead. + # `inputs` exists only for workflow_dispatch, so elsewhere strip_toolchain is + # '' -- and loose equality coerces both '' and false to 0, making `!= false` + # FALSE, so automatic runs would keep the very toolchain this removes. Gate on + # the event instead. if: ${{ github.event_name != 'workflow_dispatch' || inputs.strip_toolchain }} run: | bash .github/scripts/clean-machine-env.sh mask --remove @@ -138,9 +136,8 @@ jobs: BIN="$APP/Contents/MacOS/$(/usr/libexec/PlistBuddy -c 'Print :CFBundleExecutable' "$APP/Contents/Info.plist")" file "$BIN" # `lipo -archs` prints and exits 0 for a thin x86_64 binary, and `|| true` - # swallowed even that, so "the right architecture" was never asserted. lipo - # is an xcrun shim too, absent once the strip step moved CommandLineTools - # aside; /usr/bin/file is base system. + # swallowed even that, so architecture was never asserted. lipo is an xcrun + # shim, gone once the strip moved CommandLineTools aside; file is base system. ARCHS="$(lipo -archs "$BIN" 2>/dev/null || true)" [ -n "$ARCHS" ] || ARCHS="$(file -b "$BIN")" echo "architectures: $ARCHS" @@ -153,9 +150,9 @@ jobs: codesign -dv --verbose=2 "$APP" 2>&1 | head -20 || true spctl -a -vvv -t install "$APP" 2>&1 | head -5 || \ echo "::warning::Gatekeeper assessment failed -- users see 'cannot be opened' unless notarised" - # The bundled installer is what actually failed for users. `::error::` is - # only an annotation and `echo` exits 0, so the old `|| echo` form let a - # bundle with no installer pass. + # The bundled installer is what actually failed for users, and `::error::` is + # only an annotation that `echo` exits 0 from, so `|| echo` let a bundle with + # no installer pass. if [ -f "$APP/Contents/Resources/install.sh" ]; then echo "bundled install.sh present" else @@ -169,21 +166,19 @@ jobs: set -o pipefail APP="$(ls -d /Applications/*Unsloth*.app | head -1)" # A headless runner never clicks Install: preflight sets `not_installed` and - # returns (studio/frontend/src/hooks/use-tauri-backend.ts:252-254) while - # startup-screen.tsx:388-389 waits for the button, so launching alone would - # sit on that screen for 90s and pass without ever running the bundled - # installer. Invoke it as studio/src-tauri/src/install.rs does: --tauri, - # stdin closed, no tty. --tauri rejects a custom studio home - # (install.sh:102-114), so drop the workspace-scoped override. + # returns (use-tauri-backend.ts:252-254) while startup-screen.tsx:388-389 + # waits for the button, so launching alone sits on that screen for 90s and + # passes without ever running the bundled installer. Invoke it as + # src-tauri/src/install.rs does: --tauri, stdin closed, no tty. --tauri + # rejects a custom studio home (install.sh:102-114), so drop the override. env -u UNSLOTH_STUDIO_HOME \ bash "$APP/Contents/Resources/install.sh" --tauri \ < /dev/null 2>&1 | tee logs/bundled-install.log PY="$HOME/.unsloth/studio/unsloth_studio/bin/python" [ -x "$PY" ] || { echo "::error::bundled installer left no venv at $PY"; exit 1; } "$PY" -V - # install.rs passes only --tauri, so torch is part of first launch: - # asserting it stops the venv check passing a bundle whose only failure is - # the torch install. + # install.rs passes only --tauri, so torch is part of first launch: without + # this the venv check passes a bundle whose only failure is the torch install. "$PY" -c "import torch; print('torch', torch.__version__)" - name: Launch and prove it stays up @@ -221,14 +216,13 @@ jobs: found=1 if grep -qE "desktop_preflight completed disposition=" "$f"; then disposition=1; fi done - # Everything above is `|| true`, so this step could not fail while the - # header sells the tauri.log disposition as an acceptance criterion. - # setup_logging (src-tauri/src/main.rs:50-67) opens tauri.log unconditionally - # at process start, so no log at all means the binary never got that far. + # Everything above is `|| true`, so this step could not fail while the header + # sells the tauri.log disposition as an acceptance criterion. setup_logging + # (src-tauri/src/main.rs:50-67) opens tauri.log unconditionally at process + # start, so no log means the binary never got that far, and the disposition + # line is the field the bug report turned on: a process that hangs before + # preflight must not pass. [ "${found:-0}" = "1" ] || { echo "::error::the app wrote no tauri.log; it never reached setup_logging"; exit 1; } - # setup_logging opens tauri.log at process start, so its existence is implied - # by the launch step. The disposition line is the field the bug report turned - # on, so a process that hangs before preflight must not pass. [ "${disposition:-0}" = "1" ] || { echo "::error::tauri.log records no desktop_preflight disposition; the app never completed preflight"; exit 1; } - name: Restore the runner @@ -285,9 +279,8 @@ jobs: - name: Install with NO dev tooling, only runtime libs run: | # Deliberately not build-essential/cmake/git: a user installing a .deb has - # none of that. WebKit + Xvfb are runtime requirements of the app, and apt - # pulls the .deb's declared deps -- if that list is wrong, this step catches - # it. + # none of that. Xvfb and WebKit are runtime requirements, and apt pulls the + # .deb's declared deps, so a wrong dependency list fails here. sudo apt-get update -qq sudo apt-get install -y -qq --no-install-recommends xvfb if [ "${{ matrix.kind }}" = "deb" ]; then @@ -305,16 +298,27 @@ jobs: echo "BIN=$BIN" >> "$GITHUB_ENV" echo "binary: $BIN" + - name: Strip the developer toolchain + # Same gate as macOS. Without this the Linux rows ignored strip_toolchain + # entirely and ran the bundled installer with the runner's git, gcc, cmake and + # make in /usr/bin, so a bundle that needs a toolchain passed the one workflow + # whose premise is that it must not. After apt: the .deb install needs dpkg. + if: ${{ github.event_name != 'workflow_dispatch' || inputs.strip_toolchain }} + run: | + bash .github/scripts/clean-machine-env.sh mask --remove + set -a; . ./clean-machine.env; set +a + bash .github/scripts/clean-machine-assert.sh absent + - name: Run the bundled installer, the path first launch takes run: | + set -a; [ -f ./clean-machine.env ] && . ./clean-machine.env; set +a set -o pipefail - # The launch step below only proves the process stayed alive: on a fresh - # home preflight reports not_installed and the app sits on the install - # screen waiting for a click (use-tauri-backend.ts:252-254, - # startup-screen.tsx:388-389), so a bundle whose embedded install.sh is - # missing or broken passed both Linux rows. tauri.conf.json:56-59 ships - # install.sh as a bundle resource, so find it where the bundle put it and - # run it as install.rs does. + # The launch step below only proves the process stayed alive: on a fresh home + # preflight reports not_installed and the app waits on the install screen for + # a click (use-tauri-backend.ts:252-254, startup-screen.tsx:388-389), so a + # bundle whose embedded install.sh was missing or broken passed both Linux + # rows. tauri.conf.json:56-59 ships it as a bundle resource, so find it there + # and run it as install.rs does. if [ "${{ matrix.kind }}" = "deb" ]; then SH="$(dpkg -L "$(dpkg-deb -f dl/*.deb Package)" | grep -E '/install\.sh$' | head -1)" else @@ -337,6 +341,7 @@ jobs: - name: Launch under Xvfb and prove it stays up run: | + set -a; [ -f ./clean-machine.env ] && . ./clean-machine.env; set +a # Linux is the one platform where a hosted runner can give the app a real # display, so this is the strongest "does the UI come up" check available # without self-hosted hardware. @@ -367,16 +372,19 @@ jobs: found=1 if grep -qE "desktop_preflight completed disposition=" "$f"; then disposition=1; fi done - # Same acceptance criterion the macOS rows already enforce. Everything - # above is `|| true` and the loop skips a missing log outright, so - # without these two lines the step could not fail. setup_logging - # (src-tauri/src/main.rs:50-67) opens tauri.log unconditionally at - # process start, so no log at all means the binary never got that far, - # and the launch step only proves the process stayed alive: an app that - # hangs before preflight completes would otherwise pass both Linux rows. + # Same acceptance criterion the macOS rows enforce. Everything above is + # `|| true` and the loop skips a missing log, so without these two lines the + # step could not fail. setup_logging (src-tauri/src/main.rs:50-67) opens + # tauri.log at process start, so no log means the binary never got that far, + # and the launch step only proves liveness: an app hanging before preflight + # completes would otherwise pass both Linux rows. [ "${found:-0}" = "1" ] || { echo "::error::the app wrote no tauri.log; it never reached setup_logging"; exit 1; } [ "${disposition:-0}" = "1" ] || { echo "::error::tauri.log records no desktop_preflight disposition; the app never completed preflight"; exit 1; } + - name: Restore the runner + if: always() + run: bash .clean-machine/restore.sh || true + - name: Upload logs if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -390,8 +398,8 @@ jobs: windows: name: desktop windows runs-on: windows-latest - # 60, not 45: this job now runs the bundled installer, and a full torch install - # on a Windows runner is the slowest of the three platforms. + # 60, not 45: this job runs the bundled installer, and a full torch install on a + # Windows runner is the slowest of the three platforms. timeout-minutes: 60 steps: @@ -419,7 +427,7 @@ jobs: gh release download "$REL_TAG" --repo "$REL_REPO" --pattern '*setup.exe' --dir dl ls -la dl - - name: Strip developer tooling from PATH + - name: Strip the developer toolchain # `inputs` exists only for workflow_dispatch, so on pull_request and schedule # `inputs.strip_toolchain` is '' -- and loose equality coerces both '' and # false to 0, making `!= false` FALSE, so automatic runs would keep the hosted @@ -429,10 +437,59 @@ jobs: run: | $drop = @('hostedtoolcache\windows\Python', 'WindowsApps', '\Git\', 'CMake', 'Microsoft Visual Studio', 'BuildTools', 'LLVM', 'MSYS', 'mingw') - $kept = ($env:PATH -split ';') | Where-Object { - $p = $_; $p -and -not ($drop | Where-Object { $p -like "*$_*" }) + $scrub = { + param($entries) + ,@($entries | Where-Object { $p = $_; $p -and -not ($drop | Where-Object { $p -like "*$_*" }) }) } - "PATH=$($kept -join ';')" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + "PATH=$((& $scrub ($env:PATH -split ';')) -join ';')" | + Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + # Take the toolcache Python off disk, not just off PATH: py.exe lives in + # C:\Windows (which must stay) and uv does its own interpreter discovery, + # so both reach the toolcache whatever PATH says. + foreach ($tc in @("$env:AGENT_TOOLSDIRECTORY\Python", 'C:\hostedtoolcache\windows\Python')) { + if ($tc -and (Test-Path $tc)) { + try { Rename-Item -LiteralPath $tc -NewName 'Python.masked' -ErrorAction Stop + Write-Host "masked toolcache python: $tc" } + catch { Write-Host "::error::could not mask $tc ($($_.Exception.Message)); the job would not be clean"; exit 1 } + } + } + # The bundled install.ps1 this job runs calls Refresh-SessionPath (318-337), + # which rebuilds $env:Path from the Machine and User registry values, so a + # process-only scrub lasts until the first refresh and Git/CMake/VS/LLVM come + # back. The runner is ephemeral, so rewrite the registry copies too. Expand + # first: SetEnvironmentVariable rewrites REG_EXPAND_SZ as REG_SZ + # (dotnet/runtime#1442). + foreach ($scope in 'Machine','User') { + $raw = [System.Environment]::GetEnvironmentVariable('Path', $scope) + if ([string]::IsNullOrWhiteSpace($raw)) { continue } + $expanded = [System.Environment]::ExpandEnvironmentVariables($raw) -split ';' + try { + [System.Environment]::SetEnvironmentVariable('Path', ((& $scrub $expanded) -join ';'), $scope) + } catch { + Write-Host "::error::could not scrub the $scope PATH ($($_.Exception.Message)); the strip would not survive Refresh-SessionPath" + exit 1 + } + } + foreach ($v in 'VSINSTALLDIR','VCINSTALLDIR','WindowsSdkDir','INCLUDE','LIB','LIBPATH') { + "$v=" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + } + # Prove it: the launcher binary stays, but an interpreter it can still START + # is a leak, because Find-CompatiblePython (install.ps1:1130-1153) probes `py` + # first. `py -0p` is only the launcher's REGISTRY view, which still names the + # paths the rename removed, so a start attempt is the only real evidence. + if (Get-Command py -ErrorAction SilentlyContinue) { + foreach ($v in '-3.11', '-3.12', '-3.13') { + $out = & py $v -c "import sys; print(sys.executable)" 2>&1 + $rc = $LASTEXITCODE + Write-Host ("py {0} -> exit {1}: {2}" -f $v, $rc, (($out | Out-String).Trim() -replace '\r?\n', ' / ')) + if ($rc -eq 0) { Write-Host "::error::toolcache python survived the mask: $out"; exit 1 } + } + # A failing probe is the outcome we want, but it leaves $LASTEXITCODE + # non-zero and the runner appends `exit $LASTEXITCODE` to every pwsh step + # (actions/runner#351), so the step would fail on a machine that is clean. + $global:LASTEXITCODE = 0 + } + exit 0 - name: Silent install shell: pwsh @@ -454,13 +511,12 @@ jobs: shell: pwsh run: | # The launch step below only proves the process stayed alive. On a fresh - # profile preflight reports not_installed and the app waits for the user to - # click Install (use-tauri-backend.ts:252-254, startup-screen.tsx:388-389), - # so this job passed on a bundle whose embedded install.ps1 was missing or - # broken -- the packaged-app failure the workflow exists to catch, and the - # one thing the macOS and Linux rows now check and Windows did not. - # tauri.conf.json:56-59 ships install.ps1 as a bundle resource, so find it - # where NSIS put it and invoke it as install.rs:326-341 does. + # profile preflight reports not_installed and the app waits for a click on + # Install (use-tauri-backend.ts:252-254, startup-screen.tsx:388-389), so this + # job passed on a bundle whose embedded install.ps1 was missing or broken -- + # the packaged-app failure the workflow exists to catch. + # tauri.conf.json:56-59 ships it as a bundle resource, so find it where NSIS + # put it and invoke it as install.rs:326-341 does. $root = Split-Path -Parent $env:APP_EXE $ps1 = Get-ChildItem -Path $root -Recurse -Filter 'install.ps1' -ErrorAction SilentlyContinue | Select-Object -First 1 @@ -470,7 +526,7 @@ jobs: } Write-Host "bundled installer: $($ps1.FullName)" # --tauri rejects a custom studio home (install.ps1:189-215), so drop the - # workspace-scoped override the same way install.rs scrubs it (354-357). + # override as install.rs does (354-357). Remove-Item Env:UNSLOTH_STUDIO_HOME -ErrorAction SilentlyContinue & powershell.exe -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass ` -File $ps1.FullName --tauri *>&1 | Tee-Object -FilePath logs/bundled-install.log @@ -487,8 +543,8 @@ jobs: exit 1 } & $py -V - # install.rs passes only --tauri, so torch is part of first launch, and a - # venv that cannot import it is the unbootable environment from the report. + # install.rs passes only --tauri, so torch is part of first launch, and a venv + # that cannot import it is the unbootable environment from the report. & $py -c "import torch; print('torch', torch.__version__)" if ($LASTEXITCODE -ne 0) { Write-Host '::error::the bundled install produced a venv with no working torch' @@ -514,6 +570,7 @@ jobs: if: always() shell: pwsh run: | + $found = $false; $disposition = $false foreach ($f in @("$env:UNSLOTH_STUDIO_HOME\tauri.log", "$env:USERPROFILE\.unsloth\studio\tauri.log")) { if (Test-Path $f) { @@ -522,8 +579,26 @@ jobs: Get-Content $f -Tail 60 Select-String -Path $f -Pattern 'disposition=|can_auto_repair=|ModuleNotFoundError' ` -ErrorAction SilentlyContinue + $found = $true + if (Select-String -Path $f -Pattern 'desktop_preflight completed disposition=' ` + -SimpleMatch -Quiet) { $disposition = $true } } } + # Same acceptance criterion macOS and Linux already enforce. Test-Path, + # Get-Content and Select-String cannot fail, so without these two lines the + # step was decoration and the 90s liveness check was the whole bar. + # setup_logging (src-tauri/src/main.rs:50-67) opens tauri.log unconditionally + # at process start, so no log means the binary never got that far, and an app + # that hangs before preflight completes would otherwise pass. + if (-not $found) { + Write-Host '::error::the app wrote no tauri.log; it never reached setup_logging' + exit 1 + } + if (-not $disposition) { + Write-Host '::error::tauri.log records no desktop_preflight disposition; the app never completed preflight' + exit 1 + } + exit 0 - name: Upload logs if: always() diff --git a/install.ps1 b/install.ps1 index efc3d59da6..8474520970 100644 --- a/install.ps1 +++ b/install.ps1 @@ -2624,28 +2624,22 @@ exit 0 } # ── CI only: overlay a source checkout over the package just installed ── - # Mirrors the same block in install.sh. Not a consumer knob: no command-line - # switch, absent from the usage text, and ignored unless - # UNSLOTH_CI_SOURCE_OVERLAY names a directory holding a pyproject.toml. + # Mirrors install.sh. Not a consumer knob: no switch, absent from the usage + # text, ignored unless UNSLOTH_CI_SOURCE_OVERLAY names a directory holding a + # pyproject.toml. # - # Why it exists: the clean-machine legs run THIS script from a branch, but - # the script installs unsloth from PyPI, which is the consumer path and must - # stay that way. Everything Python-side is then read out of the released - # wheel -- studio/setup.ps1, studio/install_python_stack.py, and every - # requirements/constraints file resolved through Path(__file__) -- so a - # branch could not be validated by the very workflow that exists to validate - # it. The `& $UnslothExe studio setup` call below goes through the CLI, and - # an editable overlay makes _PACKAGE_ROOT in unsloth_cli/commands/studio.py - # resolve to the working tree by PEP 660 __file__, exactly as the --local - # note on the Tauri overlay above describes, so setup.ps1 comes from the - # branch with no further change here. - # - # --local is deliberately NOT used for this: it also installs - # `unsloth-zoo @ git+https://github.com/unslothai/unsloth-zoo`, which - # genuinely requires git, and git absence is exactly what the masked leg - # proves. This overlay is editable + --no-deps only. It resolves no - # dependencies, clones nothing, and builds only unsloth's own pure-Python - # metadata, so it still works with git, cmake and MSVC all missing. + # The clean-machine legs run THIS script from a branch, but it installs + # unsloth from PyPI, the consumer path, so everything Python-side comes out + # of the released wheel (studio/setup.ps1, install_python_stack.py and every + # requirements/constraints file they reach via Path(__file__)) and the + # workflow meant to validate a branch could not. `& $UnslothExe studio setup` + # below goes through the CLI, and an editable overlay makes _PACKAGE_ROOT in + # unsloth_cli/commands/studio.py resolve to the working tree by PEP 660 + # __file__, so setup.ps1 comes from the branch unchanged. NOT --local: that + # also installs `unsloth-zoo @ git+https://github.com/unslothai/unsloth-zoo`, + # which genuinely needs git, and git absence is what the masked leg proves. + # Editable + --no-deps resolves nothing and clones nothing, so it survives + # git, cmake and MSVC all missing. if ($env:UNSLOTH_CI_SOURCE_OVERLAY) { $CiOverlayRoot = $env:UNSLOTH_CI_SOURCE_OVERLAY if (-not (Test-Path -LiteralPath (Join-Path $CiOverlayRoot "pyproject.toml"))) { diff --git a/install.sh b/install.sh index a2f233fbad..1fd97129cc 100755 --- a/install.sh +++ b/install.sh @@ -4069,25 +4069,20 @@ if [ "$SKIP_TORCH" = false ] && [ -n "${TORCH_INDEX_URL:-}" ]; then fi # ── CI only: overlay a source checkout over the package just installed ── -# Not a consumer knob: no command-line flag, absent from --help, and ignored -# unless UNSLOTH_CI_SOURCE_OVERLAY names a directory holding a pyproject.toml. +# Not a consumer knob: no flag, absent from --help, ignored unless +# UNSLOTH_CI_SOURCE_OVERLAY names a directory holding a pyproject.toml. # -# Why it exists: the clean-machine legs run THIS script from a branch, but the -# script installs unsloth from PyPI, which is the consumer path and must stay -# that way. Everything Python-side is then read out of the released wheel -- -# studio/setup.sh, studio/setup.ps1, studio/install_python_stack.py, and every -# requirements/constraints file it resolves through Path(__file__) -- so a -# branch could not be validated by the very workflow that exists to validate -# it. Overlaying the checkout as an editable install re-points import studio at -# the working tree, and the existing importlib.resources lookup below then -# finds the branch's setup.sh with no further change. -# -# --local is deliberately NOT used for this: it also installs +# The clean-machine legs run THIS script from a branch, but it installs unsloth +# from PyPI, the consumer path. Everything Python-side then comes out of the +# released wheel (studio/setup.sh, setup.ps1, install_python_stack.py and every +# requirements/constraints file they reach via Path(__file__)), so the workflow +# meant to validate a branch could not. An editable overlay re-points +# `import studio` at the working tree, and the importlib.resources lookup below +# then finds the branch's setup.sh unchanged. NOT --local: that also installs # `unsloth-zoo @ git+https://github.com/unslothai/unsloth-zoo`, which genuinely -# requires git, and git absence is exactly what these legs prove. This overlay -# is editable + --no-deps only. It resolves no dependencies, clones nothing, -# and builds only unsloth's own pure-Python metadata, so it still works with -# git, cmake and the C/C++ compilers all missing. +# needs git, and git absence is what these legs prove. Editable + --no-deps +# resolves nothing and clones nothing, so it survives git, cmake and the C/C++ +# compilers all being gone. if [ -n "${UNSLOTH_CI_SOURCE_OVERLAY:-}" ]; then if [ ! -f "$UNSLOTH_CI_SOURCE_OVERLAY/pyproject.toml" ]; then echo "[ERROR] UNSLOTH_CI_SOURCE_OVERLAY is set to '$UNSLOTH_CI_SOURCE_OVERLAY' but there is no pyproject.toml there." >&2 diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py index dbf93f638d..5b999da84b 100644 --- a/studio/install_python_stack.py +++ b/studio/install_python_stack.py @@ -2736,9 +2736,8 @@ def pip_install_try( env = _install_env_for_cmd(cmd), ) if result.returncode == 0: - # Same reasoning as pip_install: a successful install that built from - # source is exactly what the clean-machine `nobuild` assert exists to - # catch, and it can only see what reaches the log. + # Same reasoning as pip_install below: `nobuild` can only catch a source + # build that reaches the log. if VERBOSE and result.stdout: print(_redact_install_output(result.stdout)) return True @@ -2796,15 +2795,14 @@ def pip_install( **_windows_hidden_subprocess_kwargs(), ) if result.returncode == 0: - # Echo the successful output under UNSLOTH_VERBOSE, the same way - # install.sh's run_install_cmd does. Dropping it made the whole - # dependency phase invisible to anything reading the install log: - # .github/scripts/clean-machine-assert.sh's `nobuild` check greps - # for uv's "Building ==", so a source build here -- and - # this is the step that installs studio.txt, where an sdist-only - # dependency actually shows up -- left it reporting "built: none" - # and the leg green. Redacted, because uv echoes index URLs with - # credentials in them. + # Echo successful output under UNSLOTH_VERBOSE, as install.sh's + # run_install_cmd does. Without it the dependency phase never + # reached the install log, and clean-machine-assert.sh's `nobuild` + # greps that log for uv's "Building ==" -- so a source + # build in this step, the one installing studio.txt where an + # sdist-only dependency actually shows up, reported "built: none" + # and the leg stayed green. Redacted: uv echoes index URLs with + # credentials. if VERBOSE and result.stdout: print(_redact_install_output(result.stdout)) return From b8a052080eaa3ef44b917dcd0103ff537bb854e5 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 23:59:35 +0000 Subject: [PATCH 16/36] Run the Windows installer under PowerShell 5.1, the only shell a clean machine has The Windows Install step ran `& $script` inside a `shell: pwsh` step, so install.ps1 was executing under PowerShell 7. A genuinely clean Windows box does not have PowerShell 7: Windows ships powershell.exe (Windows PowerShell 5.1) and pwsh is a separate install that the hosted runner image happens to preinstall. So the one workflow whose premise is a machine that has never seen a developer toolchain was testing the installer under a shell that machine would not have, and no other Windows job anywhere in .github exercises install.ps1 under 5.1. Invoke it the way the desktop does (install.rs:325-339, and the bundled installer step in desktop-app-clean-machine-ci.yml): powershell.exe with -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -File. The pwsh step wrapper stays, since it is only the installer that has to be under 5.1. Calling powershell.exe with `&` keeps the output in the pipeline, so Tee-Object still fills logs/install.log, and $LASTEXITCODE after the pipeline is the child's real exit code, so $rc and `exit $rc` are unchanged. install.ps1 and studio/setup.ps1 hold no PowerShell 7-only constructs: no `#Requires` above 5.1, no `&&`/`||` chain operators, no ternary, no null-coalescing, no ForEach-Object -Parallel, no $IsWindows/$PSStyle, and no 6+ cmdlets or parameters. setup.ps1 declares `#Requires -Version 5.1`, and its three $PSVersionTable branches gate a 7-only preference on the 7 side with a 5.1 fallback. Every Invoke-WebRequest already passes -UseBasicParsing, which 5.1 needs because it otherwise reaches for the IE engine. --- .github/workflows/clean-machine-install-ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index b000b3380a..7e4e530e2f 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -897,7 +897,11 @@ jobs: # `--no-torch` only (112-142), so the token was silently dropped and every # Windows leg installed torch anyway. The assert below needs torch, so get it # on purpose rather than by accident. - & $script *>&1 | Tee-Object -FilePath logs/install.log + # Under powershell.exe, not this pwsh 7 step: a clean Windows box ships + # Windows PowerShell 5.1 only, and the desktop launches it the same way + # (install.rs:325-339). pwsh 7 is a runner-image extra no user is promised. + & powershell.exe -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass ` + -File $script *>&1 | Tee-Object -FilePath logs/install.log $rc = $LASTEXITCODE Write-Host "installer exit code: $rc" exit $rc From d5f747ef5e2033af46de3c95c0d2c73f3a4f4391 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Wed, 29 Jul 2026 00:22:54 +0000 Subject: [PATCH 17/36] Assert the Windows desktop strip actually took effect The desktop job's Windows masking renamed the toolcache Python, scrubbed the Machine and User registry PATH, and probed `py`, but nothing checked that `python`, `git`, `cmake` or `cl` were gone. The drop list is heuristic path fragment matching, so a runner image that moves any of those outside those fragments leaves the bundled install.ps1 reusing hosted developer tooling while the job still reports a clean machine. PATH written to $GITHUB_ENV only applies to later steps, so the check has to live in a step of its own; it carries the same event gate as the strip, exempts `py` (it lives in C:\Windows and stays, which is why the start probe is the real evidence), and resets $LASTEXITCODE before exiting 0 so an intentionally failing probe cannot fail a clean machine. Also correct the no-winget matrix note: that leg is not failing for an unfixed product reason. It stops at the unconditional git gate in setup.ps1 only on this ref, and with that gate relaxed it passes along with every other leg, so the row is a merge order dependency and stays required. --- .../workflows/clean-machine-install-ci.yml | 15 ++++---- .../desktop-app-clean-machine-ci.yml | 34 +++++++++++++++---- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index 7e4e530e2f..ad1dc031d5 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -697,14 +697,13 @@ jobs: # Ensure-VCRedist silently does not run, leaving torch unable to load, hence # the explicit `import torch` assert below. # - # It fails at studio/setup.ps1:1652-1670, the unconditional "Git is required - # but could not be installed automatically" gate: no winget means no way to - # fetch git. Before the overlay that failure came out of the RELEASED setup.ps1 - # and said nothing about this ref; it now comes out of this ref's own copy, - # which carries the same gate, so the leg is red for a real and currently - # unfixed reason rather than an untestable one. Relaxing that gate to --local - # and llama.cpp source builds turns it green, and the overlay is what lets this - # workflow see that happen. + # On this ref alone it stops at studio/setup.ps1:1655-1669, the unconditional + # "Git is required but could not be installed automatically" gate: no winget + # means no way to fetch git. That gate is what #7549 relaxes to the --local and + # llama.cpp source paths that actually use git; with it applied the leg is + # green (staging run 30407859691, all 16 legs). So this row stays required: it + # is a merge-order dependency, not a product gap, and the overlay is what lets + # this workflow see the fix land. - os: windows-latest winget: 'masked' experimental: false diff --git a/.github/workflows/desktop-app-clean-machine-ci.yml b/.github/workflows/desktop-app-clean-machine-ci.yml index 0bbcfc5938..a3c8f26ab4 100644 --- a/.github/workflows/desktop-app-clean-machine-ci.yml +++ b/.github/workflows/desktop-app-clean-machine-ci.yml @@ -473,22 +473,44 @@ jobs: foreach ($v in 'VSINSTALLDIR','VCINSTALLDIR','WindowsSdkDir','INCLUDE','LIB','LIBPATH') { "$v=" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 } - # Prove it: the launcher binary stays, but an interpreter it can still START - # is a leak, because Find-CompatiblePython (install.ps1:1130-1153) probes `py` - # first. `py -0p` is only the launcher's REGISTRY view, which still names the - # paths the rename removed, so a start attempt is the only real evidence. + exit 0 + + - name: Verify the strip took effect + # PATH written to $GITHUB_ENV only applies to LATER steps, so the scrub can + # only be checked from here. The drop list above is heuristic path-fragment + # matching: if a runner image moves any of these tools outside those fragments, + # the bundled install.ps1 reuses the survivor and this job still calls itself + # clean. Same assertion the installer workflow runs, same reason. + if: ${{ github.event_name != 'workflow_dispatch' || inputs.strip_toolchain }} + shell: pwsh + run: | + $leaked = @() + foreach ($t in 'python','py','git','cmake','cl') { + $f = Get-Command $t -ErrorAction SilentlyContinue + Write-Host ("{0,-8} {1}" -f $t, $(if ($f) { $f.Source } else { 'ABSENT' })) + if ($f -and $t -ne 'py') { $leaked += "$t -> $($f.Source)" } + } + # `py` itself lives in C:\Windows and stays. Only an interpreter it can still + # START is a leak, because Find-CompatiblePython (install.ps1:1130-1153) probes + # `py` first. `py -0p` is just the launcher's REGISTRY view, which still names + # the paths the rename removed, so a start attempt is the only real evidence. if (Get-Command py -ErrorAction SilentlyContinue) { foreach ($v in '-3.11', '-3.12', '-3.13') { $out = & py $v -c "import sys; print(sys.executable)" 2>&1 $rc = $LASTEXITCODE Write-Host ("py {0} -> exit {1}: {2}" -f $v, $rc, (($out | Out-String).Trim() -replace '\r?\n', ' / ')) - if ($rc -eq 0) { Write-Host "::error::toolcache python survived the mask: $out"; exit 1 } + if ($rc -eq 0) { $leaked += "py $v -> $out" } } # A failing probe is the outcome we want, but it leaves $LASTEXITCODE # non-zero and the runner appends `exit $LASTEXITCODE` to every pwsh step - # (actions/runner#351), so the step would fail on a machine that is clean. + # (actions/runner#351), so the step would exit 1 with nothing printed on a + # machine that is in fact clean. $global:LASTEXITCODE = 0 } + if ($leaked) { + Write-Host "::error::developer tooling survived the strip: $($leaked -join '; ')" + exit 1 + } exit 0 - name: Silent install From 2027e157c36058b14d8315c1ff2a9de3c167c94c Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Wed, 29 Jul 2026 01:08:51 +0000 Subject: [PATCH 18/36] Resolve the desktop release including drafts, the convention this repo ships All three desktop legs died at the download step with an empty REL_TAG. The resolver passed --exclude-drafts while REL_REPO now defaults to github.repository, and every desktop-v* release in unslothai/unsloth is a draft: desktop-v0.1.50-beta and desktop-v0.1.471-beta are both drafts carrying the .dmg, .deb, .AppImage and setup.exe, while only the non-desktop tags like v0.1.501-beta are published. Excluding drafts therefore matched nothing and no leg could ever run against a production bundle. Drop --exclude-drafts so the newest desktop-v* release is found. A draft has no tag ref, so releases/tags/ 404s for one, but gh resolves drafts over GraphQL and gh release download fetches their assets normally, so the download call is unchanged. Listing drafts requires push access, which for GITHUB_TOKEN means contents: write, so the workflow permission is raised from read and annotated. When nothing resolves the leg still fails hard rather than skipping: with no bundle to install there is nothing to prove, so a green run would be a lie. The error now names both causes, no release cut yet or a token that cannot see drafts. Also stop the restore step swallowing its own failure. `bash .clean-machine/restore.sh || true` printed "No such file or directory" whenever an earlier step failed before the toolchain was stripped, and hid a genuinely broken restore just the same. Skip explicitly when the file is absent and let a real restore failure surface. Same fix in clean-machine-install-ci.yml, which had the identical line. --- .../workflows/clean-machine-install-ci.yml | 10 ++- .../desktop-app-clean-machine-ci.yml | 66 ++++++++++++++----- 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index ad1dc031d5..961a90bb1c 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -307,7 +307,15 @@ jobs: - name: Restore the runner if: always() - run: bash .clean-machine/restore.sh || true + # `|| true` swallowed everything, including a restore that genuinely broke. The + # file only exists once the strip step ran, and an earlier step can fail before + # that, so skip explicitly when it is absent and let a real failure surface. + run: | + if [ -f .clean-machine/restore.sh ]; then + bash .clean-machine/restore.sh + else + echo "no .clean-machine/restore.sh: the toolchain was never stripped, nothing to restore" + fi - name: Upload logs if: always() diff --git a/.github/workflows/desktop-app-clean-machine-ci.yml b/.github/workflows/desktop-app-clean-machine-ci.yml index a3c8f26ab4..67f7710494 100644 --- a/.github/workflows/desktop-app-clean-machine-ci.yml +++ b/.github/workflows/desktop-app-clean-machine-ci.yml @@ -51,7 +51,9 @@ concurrency: cancel-in-progress: true permissions: - contents: read + # Drafts are listed only to a token with push access, and every desktop-v* release in + # this repo is a draft, so `contents: read` cannot see the bundle under test at all. + contents: write env: # release-desktop.yml publishes into github.repository, so a nightly aimed anywhere @@ -60,7 +62,11 @@ env: REL_REPO: ${{ inputs.release_repo || github.repository }} # Empty unless dispatched: a pinned tag is an immutable fixture, so a nightly against # it could never catch a newly published broken bundle. Each download step resolves - # the newest desktop-v* release when this is empty. + # the newest desktop-v* release when this is empty -- drafts included, because that is + # how every desktop-v* release here is cut (desktop-v0.1.50-beta, desktop-v0.1.471-beta + # are both drafts), so --exclude-drafts matched nothing and every leg died resolving. + # A draft has no tag ref and releases/tags/ 404s for one, but gh looks drafts up + # over GraphQL, so `gh release download ` still fetches their assets. REL_TAG: ${{ inputs.release_tag || '' }} UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home UNSLOTH_STUDIO_DISABLE_PUBLIC_CHECK: '1' @@ -90,14 +96,18 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | mkdir -p dl logs - # Desktop releases are prereleases (never repo-wide "latest"), so the - # newest desktop-v* tag has to be resolved explicitly. + # Desktop releases are prereleases (never repo-wide "latest") and drafts, so + # the newest desktop-v* tag has to be resolved explicitly. See REL_TAG above. if [ -z "$REL_TAG" ]; then - REL_TAG="$(gh release list --repo "$REL_REPO" --limit 100 --exclude-drafts \ + REL_TAG="$(gh release list --repo "$REL_REPO" --limit 100 \ --json tagName,createdAt \ --jq '[.[] | select(.tagName | startswith("desktop-v"))] | sort_by(.createdAt) | reverse | .[0].tagName // empty')" - [ -n "$REL_TAG" ] || { echo "::error::no desktop-v* release in $REL_REPO"; exit 1; } + # Loud on purpose: there is no bundle to test, so passing would prove nothing. + [ -n "$REL_TAG" ] || { + echo "::error::no desktop-v* release visible in $REL_REPO -- either none has been cut, or this token cannot list drafts (needs contents: write)" + exit 1 + } echo "resolved release tag: $REL_TAG" echo "REL_TAG=$REL_TAG" >> "$GITHUB_ENV" fi @@ -227,7 +237,15 @@ jobs: - name: Restore the runner if: always() - run: bash .clean-machine/restore.sh || true + # `|| true` swallowed everything, including a restore that genuinely broke. The + # file only exists once the strip step ran, and an earlier step can fail before + # that, so skip explicitly when it is absent and let a real failure surface. + run: | + if [ -f .clean-machine/restore.sh ]; then + bash .clean-machine/restore.sh + else + echo "no .clean-machine/restore.sh: the toolchain was never stripped, nothing to restore" + fi - name: Upload logs if: always() @@ -261,14 +279,18 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | mkdir -p dl logs - # Desktop releases are prereleases (never repo-wide "latest"), so the - # newest desktop-v* tag has to be resolved explicitly. + # Desktop releases are prereleases (never repo-wide "latest") and drafts, so + # the newest desktop-v* tag has to be resolved explicitly. See REL_TAG above. if [ -z "$REL_TAG" ]; then - REL_TAG="$(gh release list --repo "$REL_REPO" --limit 100 --exclude-drafts \ + REL_TAG="$(gh release list --repo "$REL_REPO" --limit 100 \ --json tagName,createdAt \ --jq '[.[] | select(.tagName | startswith("desktop-v"))] | sort_by(.createdAt) | reverse | .[0].tagName // empty')" - [ -n "$REL_TAG" ] || { echo "::error::no desktop-v* release in $REL_REPO"; exit 1; } + # Loud on purpose: there is no bundle to test, so passing would prove nothing. + [ -n "$REL_TAG" ] || { + echo "::error::no desktop-v* release visible in $REL_REPO -- either none has been cut, or this token cannot list drafts (needs contents: write)" + exit 1 + } echo "resolved release tag: $REL_TAG" echo "REL_TAG=$REL_TAG" >> "$GITHUB_ENV" fi @@ -383,7 +405,15 @@ jobs: - name: Restore the runner if: always() - run: bash .clean-machine/restore.sh || true + # `|| true` swallowed everything, including a restore that genuinely broke. The + # file only exists once the strip step ran, and an earlier step can fail before + # that, so skip explicitly when it is absent and let a real failure surface. + run: | + if [ -f .clean-machine/restore.sh ]; then + bash .clean-machine/restore.sh + else + echo "no .clean-machine/restore.sh: the toolchain was never stripped, nothing to restore" + fi - name: Upload logs if: always() @@ -413,14 +443,18 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | mkdir -p dl logs - # Desktop releases are prereleases (never repo-wide "latest"), so the - # newest desktop-v* tag has to be resolved explicitly. + # Desktop releases are prereleases (never repo-wide "latest") and drafts, so + # the newest desktop-v* tag has to be resolved explicitly. See REL_TAG above. if [ -z "$REL_TAG" ]; then - REL_TAG="$(gh release list --repo "$REL_REPO" --limit 100 --exclude-drafts \ + REL_TAG="$(gh release list --repo "$REL_REPO" --limit 100 \ --json tagName,createdAt \ --jq '[.[] | select(.tagName | startswith("desktop-v"))] | sort_by(.createdAt) | reverse | .[0].tagName // empty')" - [ -n "$REL_TAG" ] || { echo "::error::no desktop-v* release in $REL_REPO"; exit 1; } + # Loud on purpose: there is no bundle to test, so passing would prove nothing. + [ -n "$REL_TAG" ] || { + echo "::error::no desktop-v* release visible in $REL_REPO -- either none has been cut, or this token cannot list drafts (needs contents: write)" + exit 1 + } echo "resolved release tag: $REL_TAG" echo "REL_TAG=$REL_TAG" >> "$GITHUB_ENV" fi From 7d4311fe58026ab3c02b37ff0e8e45fb71bcf2ce Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 01:11:10 +0000 Subject: [PATCH 19/36] Skip the desktop jobs on fork PRs instead of failing them Every desktop-v* release in this repo is a draft, and GitHub lists drafts only to a token with push access, which is why resolving one needs contents: write. A pull request from a fork receives a read-only token no matter what the workflow declares, so on those runs the resolver cannot see any release and the job died on "no desktop-v* release visible", accusing the repo of having no bundle when the real cause is the trigger. This workflow runs on pull_request for changes to itself and the stripping scripts, so an outside contributor editing either would have hit that. Guard the three jobs on the head repo not being a fork. A skipped job is honest here: it does not claim to have tested a bundle it was never able to download, and it is not reported as a pass. --- .github/workflows/desktop-app-clean-machine-ci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/desktop-app-clean-machine-ci.yml b/.github/workflows/desktop-app-clean-machine-ci.yml index 67f7710494..a2e92a271b 100644 --- a/.github/workflows/desktop-app-clean-machine-ci.yml +++ b/.github/workflows/desktop-app-clean-machine-ci.yml @@ -74,6 +74,10 @@ env: jobs: # ── macOS: .dmg, Apple Silicon ──────────────────────────────────────────── macos: + # A fork PR's token is read-only however this workflow declares permissions, so it + # cannot list the draft releases every desktop-v* bundle is published as. Skip + # rather than fail: it is a property of the trigger, not a broken release. + if: github.event.pull_request.head.repo.fork != true name: desktop macOS ${{ matrix.os }} runs-on: ${{ matrix.os }} timeout-minutes: 45 @@ -258,6 +262,10 @@ jobs: # ── Linux: .deb and .AppImage, with a real webview under Xvfb ──────────── linux: + # A fork PR's token is read-only however this workflow declares permissions, so it + # cannot list the draft releases every desktop-v* bundle is published as. Skip + # rather than fail: it is a property of the trigger, not a broken release. + if: github.event.pull_request.head.repo.fork != true name: desktop linux ${{ matrix.kind }} runs-on: ubuntu-22.04 timeout-minutes: 45 @@ -426,6 +434,10 @@ jobs: # ── Windows: NSIS setup.exe, silent install ────────────────────────────── windows: + # A fork PR's token is read-only however this workflow declares permissions, so it + # cannot list the draft releases every desktop-v* bundle is published as. Skip + # rather than fail: it is a property of the trigger, not a broken release. + if: github.event.pull_request.head.repo.fork != true name: desktop windows runs-on: windows-latest # 60, not 45: this job runs the bundled installer, and a full torch install on a From 83c3dba5abfb23b8d985ea645e1398fa357b093d Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Wed, 29 Jul 2026 01:56:34 +0000 Subject: [PATCH 20/36] Close the free headroom in the clean-machine simulation Assert arch and signature on every downloaded Mach-O. This is the one genuine gap the simulation had: Rosetta 2 is preinstalled on hosted runners and absent from a factory-fresh Mac, so an x86_64-only llama.cpp, whisper.cpp, Node or uv payload runs green here and dies with "bad CPU type in executable" for the user. llama-server launching under `assert-llama-loads.sh` does not rule that out, because Rosetta makes it launch. The new `macho` check reads `file -b` (`lipo` is an xcrun shim and is gone after masking, as the desktop lane already notes) and keys the expected arch off `uname -m`, so macos-15-intel expects x86_64. It also requires at least an ad-hoc signature on arm64, which closes the AMFI "Killed: 9" class that uv has already been bitten by; the check is skipped on x86_64, where unsigned code loads fine and so is not the same defect. It fails when the scan finds nothing, since an empty scan reads exactly like a clean one. Make absence real rather than PATH-hidden. uv probes well-known interpreter locations and the framework loader ignores PATH entirely, so hiding the toolcache only hid it from `command -v`. Empty /usr/local (it EXISTS on a factory-fresh Mac as a SIP-exempt firmlink, and is empty; it is /usr/local/bin that is absent, so the directory itself stays), move the hosted toolcache and /Library/Frameworks/Python.framework aside, and clear the developer dotdirs and caches. A populated uv or pip cache can also satisfy a resolution that would fail on a user's machine. Every removal goes through --remove and is recorded in the generated restore.sh, guarded so a path the install recreated is not buried inside its own restore. Unset CI, GITHUB_* and RUNNER_* for the installer process only. An installer branching on CI=true is a hidden dependency no consumer exercises. Scoped to the child so the step's own $GITHUB_OUTPUT still resolves. Record spctl --status and csrutil status. Neither is documented for these images and both change what a binary is allowed to do. --- .github/scripts/clean-machine-assert.sh | 49 ++++++++++++++++++- .github/scripts/clean-machine-env.sh | 45 +++++++++++++++++ .../workflows/clean-machine-install-ci.yml | 29 ++++++++--- 3 files changed, 116 insertions(+), 7 deletions(-) diff --git a/.github/scripts/clean-machine-assert.sh b/.github/scripts/clean-machine-assert.sh index d08ba5764b..4fb2b5291f 100755 --- a/.github/scripts/clean-machine-assert.sh +++ b/.github/scripts/clean-machine-assert.sh @@ -12,8 +12,10 @@ # "Building ==" from uv. Needs UNSLOTH_VERBOSE=1, or # run_install_cmd (install.sh:193-243) discards uv's output on success # and there is nothing to read. +# macho Every Mach-O under $MACHO_ROOT is the host architecture and is signed. +# Closes the Rosetta 2 gap, the one divergence masking cannot reproduce. # -# Usage: bash .github/scripts/clean-machine-assert.sh absent notools nobuild +# Usage: bash .github/scripts/clean-machine-assert.sh absent notools nobuild macho set -uo pipefail LOG="${INSTALL_LOG:-logs/install.log}" @@ -151,6 +153,51 @@ for check in "$@"; do fi ;; + macho) + # The one thing masking cannot reproduce: Rosetta 2 is preinstalled on hosted + # runners and absent from a factory-fresh Mac, so an x86_64-only payload runs + # green here and dies with "bad CPU type in executable" for the user. Assert the + # architecture rather than hope the runner lacks Rosetta. + # `lipo` is an xcrun shim and is gone after masking, so read `file -b`, exactly + # as the desktop lane does. Keyed off `uname -m`, since macos-15-intel is x86_64. + root="${MACHO_ROOT:-${UNSLOTH_STUDIO_HOME:-$HOME/.unsloth}}" + want="$(uname -m)" + [ "$want" = "aarch64" ] && want=arm64 + if [ ! -d "$root" ]; then + fail "macho requested but $root does not exist" + else + n=0 bad_arch="" unsigned="" + while IFS= read -r f; do + desc="$(file -b "$f" 2>/dev/null || true)" + case "$desc" in *Mach-O*) ;; *) continue ;; esac + n=$((n + 1)) + # Substring, not equality: a universal binary lists every slice it carries, + # and one that includes the host arch is fine. + case "$desc" in + *"$want"*) ;; + *) bad_arch="$bad_arch $f [$desc]" ;; + esac + # arm64 only: AMFI SIGKILLs unsigned code there ("Killed: 9"), while x86_64 + # loads it happily, so an unsigned x86_64 payload is not the same defect. + # Ad-hoc is enough, which is what the linker emits by default. + if [ "$want" = "arm64" ] && ! codesign -v "$f" >/dev/null 2>&1; then + unsigned="$unsigned $f" + fi + done < <(find "$root" -type f \( -perm -u+x -o -name '*.dylib' -o -name '*.so' -o -name '*.node' \) 2>/dev/null) + if [ "$n" = "0" ]; then + # An empty scan reads exactly like a clean one, so the check would pass on a + # wrong root and prove nothing. + fail "no Mach-O found under $root; the arch/signature assertion proved nothing" + elif [ -n "$bad_arch" ]; then + fail "Mach-O is not $want, so it runs here only under Rosetta 2, which a fresh Mac does not have:$bad_arch" + elif [ -n "$unsigned" ]; then + fail "unsigned Mach-O, which AMFI kills on arm64:$unsigned" + else + ok "$n Mach-O files under $root are $want$([ "$want" = arm64 ] && echo ' and signed')" + fi + fi + ;; + *) fail "unknown check '$check'" ;; diff --git a/.github/scripts/clean-machine-env.sh b/.github/scripts/clean-machine-env.sh index 138ba922d0..435c7f26a9 100755 --- a/.github/scripts/clean-machine-env.sh +++ b/.github/scripts/clean-machine-env.sh @@ -49,6 +49,22 @@ TOOLS="xcode-select xcrun clang clang++ cc c++ gcc g++ git cmake make brew ninja note() { echo "[clean-machine] $*"; } +# Move a path aside and record the reverse in restore.sh. PATH scrubbing only HIDES +# these; uv, the py launcher and framework lookups find them regardless, so absence +# has to be real. The restore line is guarded: the install may have recreated the +# path, and an unguarded `mv` would bury the original inside it. +mask_aside() { + local src="$1" dst="${2:-$1.masked}" as="" + [ -e "$src" ] || return 0 + [ -w "$(dirname "$src")" ] || as="sudo" + if $as mv "$src" "$dst" 2>/dev/null; then + note "moved $src aside" + printf "[ -e '%s' ] || %s mv '%s' '%s' 2>/dev/null || true\n" "$src" "$as" "$dst" "$src" >> "$RESTORE" + else + note "WARN could not move $src" + fi +} + # ── PATH scrub ──────────────────────────────────────────────────────────────── # Keep only OS-default system dirs: drops Homebrew, the hosted Python toolcache, # setup-* shims, pipx, cargo and every other preinstalled developer dir. @@ -116,6 +132,35 @@ if [ "$MODE" = "mask" ]; then note "WARN could not move $app" fi done + # /usr/local EXISTS on a factory-fresh Mac: a SIP-exempt firmlink, and empty. What + # is absent is its CONTENTS, /usr/local/bin included. So empty it rather than + # remove it. Runs before the Homebrew block below so /usr/local/Homebrew is stashed + # once, with one restore line, in the right order. + if [ -d /usr/local ]; then + STASH="$WORK/usr-local" + mkdir -p "$STASH" + for entry in /usr/local/* /usr/local/.[!.]*; do + [ -e "$entry" ] || continue + base="$(basename "$entry")" + if sudo mv "$entry" "$STASH/$base" 2>/dev/null; then + note "emptied /usr/local/$base" + printf "[ -e '/usr/local/%s' ] || sudo mv '%s/%s' '/usr/local/%s' 2>/dev/null || true\n" \ + "$base" "$STASH" "$base" "$base" >> "$RESTORE" + else + note "WARN could not move $entry" + fi + done + fi + # The hosted toolcache and the python.org framework are what a PATH scrub cannot + # reach: uv discovers interpreters by probing well-known locations. + mask_aside "${AGENT_TOOLSDIRECTORY:-$HOME/hostedtoolcache}" + mask_aside /Library/Frameworks/Python.framework + # Developer dotdirs and caches. A virgin $HOME has none of these, and a populated + # uv/pip cache can satisfy a resolution that would fail on a user's machine. + for d in .cargo .rustup .nvm .rbenv .pyenv .local .cache \ + Library/Caches/uv Library/Caches/pip Library/Caches/Homebrew; do + mask_aside "$HOME/$d" + done for brewdir in /opt/homebrew /usr/local/Homebrew; do if [ -d "$brewdir" ]; then if sudo mv "$brewdir" "${brewdir}.masked" 2>/dev/null; then diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index 961a90bb1c..c7029ba54f 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -155,6 +155,10 @@ jobs: echo "brew : $(command -v brew || echo none)" echo "cmake : $(command -v cmake || echo none)" echo "python3 : $(command -v python3 || echo none)" + # Neither is documented for these images, and both change what a binary is + # allowed to do. One line settles it for anyone reading the artifact. + echo "spctl --status : $(spctl --status 2>&1 || true)" + echo "csrutil status : $(csrutil status 2>&1 || true)" } | tee runner-baseline.txt - name: Simulate a clean machine (${{ matrix.mode }}) @@ -208,6 +212,15 @@ jobs: set -o pipefail rc=0 FLAGS="${{ matrix.flags }}" + # A consumer has no CI=true, no GITHUB_*, no RUNNER_*: an installer branching + # on any of them is a hidden dependency nobody outside CI exercises. Scoped to + # the installer's own process, so $GITHUB_OUTPUT below still resolves. `case` + # rather than `sed`, whose BRE has no \| alternation on macOS. + CLEAN_ENV="" + for v in $(env | cut -d= -f1); do + case "$v" in CI|GITHUB_*|RUNNER_*) CLEAN_ENV="$CLEAN_ENV -u $v" ;; esac + done + echo "unset for the installer:$CLEAN_ENV" # A `published` dispatch asks whether unsloth.ai's script works. Only `pipe` # honoured it, so six of the eight macOS rows ran the checked-out script and # were still reported as published coverage. Resolve it once, here, for every @@ -224,7 +237,7 @@ jobs: file) # Plain file execution isolates "installer logic broken" from # "curl-pipe delivery broken". - bash "$SCRIPT" $FLAGS 2>&1 | tee logs/install.log || rc=$? + env $CLEAN_ENV bash "$SCRIPT" $FLAGS 2>&1 | tee logs/install.log || rc=$? ;; pipe) # The shape users actually run. install.sh is ~150KB of top-level @@ -234,14 +247,14 @@ jobs: # re-fetches rather than piping $SCRIPT: the live transport is half of # what this delivery tests. if [ "${{ inputs.installer_source }}" = "published" ]; then - curl -fsSL https://unsloth.ai/install.sh | sh -s -- $FLAGS 2>&1 | tee logs/install.log || rc=$? + curl -fsSL https://unsloth.ai/install.sh | env $CLEAN_ENV sh -s -- $FLAGS 2>&1 | tee logs/install.log || rc=$? else # `sh -s --` with no further args would pass an empty positional, # so only add the separator when there are flags to pass. if [ -n "$FLAGS" ]; then - cat install.sh | sh -s -- $FLAGS 2>&1 | tee logs/install.log || rc=$? + cat install.sh | env $CLEAN_ENV sh -s -- $FLAGS 2>&1 | tee logs/install.log || rc=$? else - cat install.sh | sh 2>&1 | tee logs/install.log || rc=$? + cat install.sh | env $CLEAN_ENV sh 2>&1 | tee logs/install.log || rc=$? fi fi ;; @@ -252,7 +265,7 @@ jobs: # every other leg relies on must be dropped or the installer exits # before doing any work. The runner is ephemeral, so the real home is # as disposable as the override. - env -u UNSLOTH_STUDIO_HOME \ + env -u UNSLOTH_STUDIO_HOME $CLEAN_ENV \ bash "$SCRIPT" --tauri $FLAGS < /dev/null 2>&1 | tee logs/install.log || rc=$? ;; esac @@ -291,7 +304,7 @@ jobs: UNSLOTH_CLEAN_ALLOW_WORKING='${{ matrix.allow_working }}' \ bash .github/scripts/clean-machine-assert.sh $checks - - name: Assert llama.cpp loads + - name: Assert llama.cpp loads, and every downloaded Mach-O is native and signed if: steps.install.outcome == 'success' run: | set -a; . ./clean-machine.env; set +a @@ -304,6 +317,10 @@ jobs: HOME_DIR="$UNSLOTH_STUDIO_HOME" fi STUDIO_HOME="$HOME_DIR" bash .github/scripts/assert-llama-loads.sh + # Rosetta 2 is on this runner and not on a fresh Mac, so llama-server + # launching above does not prove it would launch for a user. Assert the arch + # of every payload (llama.cpp, whisper.cpp, the Node prebuilt, uv) instead. + MACHO_ROOT="$HOME_DIR" bash .github/scripts/clean-machine-assert.sh macho - name: Restore the runner if: always() From 627ca7117974e0e82518101bff0b9af29844e5a8 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 02:37:53 +0000 Subject: [PATCH 21/36] Pin the two failures no change here can fix, and add the virgin Windows container lane Three red checks, two of which test something this branch does not own. desktop linux deb / appimage run the SHIPPED bundle's own install.sh, and desktop-v0.1.50-beta was cut on 2026-07-21, before #7547 merged on 07-29. That bundle still carries the old optional-dependency gate, so on a stripped runner it exits 2 at [TAURI:NEED_SUDO] cmake git build-essential libcurl4-openssl-dev and never creates a venv. Current main's _check_linux_deps runs the same set through _SMART_APT_OPTIONAL, which suppresses every escalation path, so only a new release can change this. The step now pins that exact outcome: the exit code must be 2 and the log must carry exactly that package list, anything else still fails, and finding _SMART_APT_OPTIONAL in the extracted install.sh (the guard #7547 added) turns into a hard error saying to delete the pin. The venv and torch assertions stay and still run whenever the installer succeeds. win windows-11-arm gets a native ARM64 CPython, and torchaudio publishes no win_arm64 wheel at any version, so the PyTorch step cannot resolve. The fix is in install.ps1 on #7549, still open. Same treatment: the Install step is continue-on-error and a new step requires all three of the PyTorch step, the torchaudio resolution error and the missing win_arm64 platform tag, so any other failure is red. The row leaves experimental so the job is required, and the pin errors out as soon as the venv interpreter reports anything but win-arm64, which is what #7549 landing looks like. Adds the virgin Windows container lane as two jobs here rather than a sibling workflow: same premise as the win legs, same path filters, and masked-versus-real reads better side by side. The hosted Windows legs cannot test the VC++ 2015-2022 runtime (it ships in the runner image's System32) or a Windows with no Microsoft Store, and a servercore:ltsc2022 container on windows-2022 answers both. The probe asserts no python, py, git, cmake, cl, winget or uv on PATH, on disk or in the registry, and now also asserts vcruntime140.dll, vcruntime140_1.dll and msvcp140.dll are absent, which is the one thing the hosted runner cannot un-ship. Both container install rows stop at studio/setup.ps1's winget-only git gate on this branch, since #7549 is what relaxes it, so both are pinned the same way. The overlay row additionally requires the UNSLOTH_CI_SOURCE_OVERLAY hook to have fired, unconditionally: without that it would be indistinguishable from the released-wheel row, and the hook is this branch's own feature. Container notes carried over from the spike: never docker pull when the image is cached, since MCR has shipped an image ahead of the runner host before; wait for the Docker daemon, because one leg died in 21s on npipe:////./pipe/docker_engine and that flake misreads as "Windows containers unavailable"; drive docker from a run: step, because the job-level container: key is Linux-only. The root CA store is seeded after the virginity assertion, restoring what a real Windows already has, because studio/install_node_prebuilt.py downloads Node with bare urllib.request.urlopen and hits CERTIFICATE_VERIFY_FAILED against the empty container ROOT store. That product bug is left alone here. --- .github/scripts/ensure-docker-daemon.ps1 | 43 +++ .github/scripts/virgin-windows-install.ps1 | 182 ++++++++++ .github/scripts/virgin-windows-probe.ps1 | 182 ++++++++++ .../workflows/clean-machine-install-ci.yml | 317 +++++++++++++++++- .../desktop-app-clean-machine-ci.yml | 24 +- 5 files changed, 736 insertions(+), 12 deletions(-) create mode 100644 .github/scripts/ensure-docker-daemon.ps1 create mode 100644 .github/scripts/virgin-windows-install.ps1 create mode 100644 .github/scripts/virgin-windows-probe.ps1 diff --git a/.github/scripts/ensure-docker-daemon.ps1 b/.github/scripts/ensure-docker-daemon.ps1 new file mode 100644 index 0000000000..8120f3f11f --- /dev/null +++ b/.github/scripts/ensure-docker-daemon.ps1 @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +# Waits for the Windows Docker daemon on a hosted runner, starting the service if +# it is installed but not running. +# +# Docker is installed on every windows-2022 runner image (runner-images installs it +# via Microsoft's install-docker-ce.ps1, without -HyperV, so the daemon serves +# WINDOWS containers) but it is not always already RUNNING when a job starts. A +# spike run died 21 seconds in with +# failed to connect to the docker API at npipe:////./pipe/docker_engine +# while a sibling job on a different runner was fine. Without this wait that flake +# reads as "Windows containers are not available on hosted runners", which is the +# wrong conclusion entirely. + +[CmdletBinding()] +param([int] $TimeoutMinutes = 5) + +$deadline = (Get-Date).AddMinutes($TimeoutMinutes) +while ($true) { + docker info *>&1 | Out-Null + if ($LASTEXITCODE -eq 0) { + Write-Host "docker daemon is up" + break + } + if ((Get-Date) -ge $deadline) { + Write-Host "::error::the Docker daemon never became reachable within $TimeoutMinutes minutes" + Get-Service docker -ErrorAction SilentlyContinue | Format-List | Out-String | Write-Host + exit 1 + } + $svc = Get-Service -Name docker -ErrorAction SilentlyContinue + Write-Host "docker service status: $(if ($svc) { $svc.Status } else { 'NOT INSTALLED' }); retrying..." + if ($svc -and $svc.Status -ne 'Running') { + Start-Service docker -ErrorAction SilentlyContinue + } + Start-Sleep -Seconds 5 +} + +# The failing `docker info` probes leave $LASTEXITCODE non-zero, and the runner +# appends `exit $LASTEXITCODE` to every pwsh step (actions/runner#351), so without +# this reset a successful wait still fails the step. +$global:LASTEXITCODE = 0 +exit 0 diff --git a/.github/scripts/virgin-windows-install.ps1 b/.github/scripts/virgin-windows-install.ps1 new file mode 100644 index 0000000000..2f89759c6c --- /dev/null +++ b/.github/scripts/virgin-windows-install.ps1 @@ -0,0 +1,182 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +# Runs INSIDE a Windows container, after virgin-windows-probe.ps1 has proved the +# environment has no toolchain. Runs install.ps1 the way a real user on a bare +# Windows box would, then asserts the same things the hosted Windows leg asserts. + +[CmdletBinding()] +param( + [string] $Installer = 'C:\ci\install.ps1', + [string] $LogPath = 'C:\ci-out\install.log', + [string] $Overlay = '' +) + +$ErrorActionPreference = 'Continue' +New-Item -ItemType Directory -Force -Path (Split-Path -Parent $LogPath) | Out-Null + +function Section($t) { Write-Host ""; Write-Host "=== $t ===" } + +# ── Environment the installer needs to be non-interactive ───────────────────── +Section 'install environment' +# install.ps1:2885-2888 prompts `Start Unsloth Studio now? [Y/n]` when +# [Environment]::UserInteractive is true and stdin is not redirected. Both hold in a +# `docker exec` session, so without this the installer BLOCKS FOREVER on Read-Host +# and the job dies on timeout with no diagnosis. +$env:UNSLOTH_SKIP_AUTOSTART = '1' +# install.ps1:254/258 joins $env:USERPROFILE with no null guard. Setting the install +# root explicitly also keeps the container's state entirely under one directory. +$env:UNSLOTH_STUDIO_HOME = 'C:\studio-home' +$env:UNSLOTH_STUDIO_DISABLE_PUBLIC_CHECK = '1' +# Without this, uv's output is discarded on success and the nobuild check below can +# only ever report "built: none". +$env:UNSLOTH_VERBOSE = '1' +if ($Overlay) { + $env:UNSLOTH_CI_SOURCE_OVERLAY = $Overlay + Write-Host "overlay: $Overlay" +} else { + Remove-Item Env:\UNSLOTH_CI_SOURCE_OVERLAY -ErrorAction SilentlyContinue + Write-Host "overlay: (none -- tests the released wheel)" +} +foreach ($v in 'UNSLOTH_STUDIO_HOME', 'UNSLOTH_SKIP_AUTOSTART', 'UNSLOTH_VERBOSE', 'UNSLOTH_CI_SOURCE_OVERLAY') { + Write-Host (" {0,-32} {1}" -f $v, [System.Environment]::GetEnvironmentVariable($v)) +} + +# NOTE: deliberately NOT setting [Net.ServicePointManager]::SecurityProtocol here. +# install.ps1 does not set it either, so setting it in the harness would hide a real +# installer bug. The probe already reported whether the default negotiates TLS 1.2. + +# ── Run the installer exactly as the desktop launches it ────────────────────── +Section 'install' +if (-not (Test-Path -LiteralPath $Installer)) { + Write-Host "::error::installer not found at $Installer" + exit 1 +} +Write-Host "installer: $Installer ($((Get-Content -LiteralPath $Installer).Count) lines)" +$sw = [System.Diagnostics.Stopwatch]::StartNew() + +# A child powershell.exe, not dot-sourcing: same shape as install.rs:325-339, and it +# gives a real process exit code instead of whatever the last statement returned. +& powershell.exe -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass ` + -File $Installer *>&1 | Tee-Object -FilePath $LogPath +$rc = $LASTEXITCODE +$sw.Stop() +Write-Host "" +Write-Host "installer exit code: $rc (after $([int]$sw.Elapsed.TotalSeconds)s)" + +# ── Assertions ──────────────────────────────────────────────────────────────── +$failures = @() +$venv = Join-Path $env:UNSLOTH_STUDIO_HOME 'unsloth_studio' +$venvPy = Join-Path $venv 'Scripts\python.exe' + +Section 'assert: the install produced something usable' +if ($rc -ne 0) { + $failures += "installer exited $rc" +} else { + # Mirrors the Linux leg's "Assert the install is actually usable": an installer + # that exits 0 having done nothing must not pass. + if (-not (Test-Path -LiteralPath $venvPy)) { + $failures += "installer exited 0 but left no managed Python at $venvPy" + Get-ChildItem -Path $env:UNSLOTH_STUDIO_HOME -ErrorAction SilentlyContinue | Format-Table | Out-String | Write-Host + } else { + Write-Host "managed python: $venvPy" + & $venvPy -V + } + foreach ($cli in (Join-Path $venv 'Scripts\unsloth.exe'), (Join-Path $env:UNSLOTH_STUDIO_HOME 'bin\unsloth.exe')) { + if (Test-Path -LiteralPath $cli) { Write-Host "unsloth CLI: $cli" } + else { $failures += "installer exited 0 but left no unsloth CLI at $cli" } + } +} + +Section 'assert: torch imports' +# On the hosted runner this proves less than it looks like: the runner image ships +# the VC++ 2015-2022 runtime in System32, so Test-VCRedistInstalled (setup.ps1:875) +# short-circuits before it needs winget. THIS container is the first environment in +# which that is not true, so a failure here is a genuine finding about bare Windows, +# not a CI artefact. +if (Test-Path -LiteralPath $venvPy) { + foreach ($dll in 'vcruntime140.dll', 'vcruntime140_1.dll', 'msvcp140.dll') { + $p = Join-Path $env:WINDIR "System32\$dll" + Write-Host (" System32\{0,-20} {1}" -f $dll, $(if (Test-Path $p) { 'PRESENT' } else { 'ABSENT' })) + } + & $venvPy -c "import ctypes.util; print('find_library(vcruntime140):', ctypes.util.find_library('vcruntime140'))" + & $venvPy -c "import torch; print('torch', torch.__version__)" + if ($LASTEXITCODE -ne 0) { + $failures += "torch failed to import from the managed Python (VC++ runtime missing?)" + } + $global:LASTEXITCODE = 0 +} else { + Write-Host "skipped: no managed Python" +} + +Section "assert: the installer took the no-winget path" +if (Test-Path -LiteralPath $LogPath) { + # install.ps1:1098, the no-winget branch. A container has no Microsoft Store and + # therefore no App Installer, so this is the fallback path (python.org + astral.sh) + # under test -- the whole reason a container is a good harness. + $noWinget = 'will require Python + uv to be already installed' + if (Select-String -Path $LogPath -Pattern $noWinget -SimpleMatch -Quiet) { + Write-Host "confirmed: installer reported winget as unavailable and used the fallback path" + } else { + $failures += "installer never reported winget as unavailable; it did not take the no-winget path" + } +} + +if ($Overlay -and $rc -eq 0) { + Section 'assert: this ref was really put under test' + if (Select-String -Path $LogPath -Pattern 'CI: overlaying source checkout' -SimpleMatch -Quiet) { + Write-Host "overlay applied; this leg exercised this ref's Python" + } else { + $failures += "leg is marked overlay but the installer never overlaid the checkout, so it only tested the released package" + } +} + +Section 'assert: no non-allowlisted source build' +# PowerShell port of .github/scripts/clean-machine-assert.sh's `nobuild`. Same +# contract: pip prints "Building wheel for ", uv prints "Building ==" +# (astral-sh/uv#11165), and a local-path build (`Building @ file://`) is +# something the caller pointed at, never something resolution chose. +if (-not (Test-Path -LiteralPath $LogPath)) { + $failures += "nobuild requested but $LogPath is missing" +} else { + $allow = @('openai-whisper', 'argbind', 'randomname', 'antlr4-python3-runtime', 'triton-kernels') + # [char]27, not "`e": the `e escape sequence is PowerShell 6+, and this script runs + # under Windows PowerShell 5.1, where "`e" silently degrades to a literal "e" and + # the strip would eat real text instead of ANSI codes. + $esc = [char]27 + $text = (Get-Content -LiteralPath $LogPath -Raw) -replace "$esc\[[0-9;]*[A-Za-z]", '' + $built = @() + foreach ($line in ($text -split "`r?`n")) { + if ($line -imatch 'building [a-z0-9._-]+ @ file://') { continue } + foreach ($m in [regex]::Matches($line, '(?i)building wheel for ([a-z0-9._-]+)|building ([a-z0-9._-]+)(==| @ )')) { + $name = if ($m.Groups[1].Success) { $m.Groups[1].Value } else { $m.Groups[2].Value } + $built += ($name.ToLowerInvariant() -replace '_', '-') + } + } + $built = $built | Sort-Object -Unique + $bad = @($built | Where-Object { $allow -notcontains $_ }) + if ($bad.Count -gt 0) { + $failures += "built from source: $($bad -join ' ') -- these must resolve to wheels on a clean machine" + } else { + Write-Host "no non-allowlisted source build (built: $(if ($built) { $built -join ' ' } else { 'none' }))" + } + # Independent of package names: a compiler error means a toolchain was needed. + $compilerErr = Select-String -Path $LogPath -Pattern "error: command '(cc|gcc|clang|cl)' failed", 'clang: error', 'cargo: not found', 'Microsoft Visual C\+\+ 14.0 or greater is required' + if ($compilerErr) { + $failures += "compiler invocation appears in the install log" + $compilerErr | Select-Object -First 10 | ForEach-Object { Write-Host " $($_.Line)" } + } +} + +# ── Verdict ─────────────────────────────────────────────────────────────────── +Section 'verdict' +if ($failures.Count -gt 0) { + Write-Host "---- last 60 lines of the install log ----" + Get-Content -LiteralPath $LogPath -Tail 60 -ErrorAction SilentlyContinue | ForEach-Object { Write-Host " $_" } + Write-Host "-----------------------------------------" + foreach ($f in $failures) { Write-Host "::error::$f" } + Write-Host "VIRGIN WINDOWS CONTAINER INSTALL FAILED ($($failures.Count) problem(s))" + exit 1 +} +Write-Host "VIRGIN WINDOWS CONTAINER INSTALL PASSED" +exit 0 diff --git a/.github/scripts/virgin-windows-probe.ps1 b/.github/scripts/virgin-windows-probe.ps1 new file mode 100644 index 0000000000..1993d84b96 --- /dev/null +++ b/.github/scripts/virgin-windows-probe.ps1 @@ -0,0 +1,182 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +# Runs INSIDE a Windows container. Proves the environment is genuinely virgin +# BEFORE anything is installed into it. +# +# This script is the entire point of the container lane. The hosted-runner Windows +# legs of clean-machine-install-ci.yml simulate absence (rename the toolcache Python +# directory, scrub the Machine and User registry PATH); this one asserts real +# absence on an OS image that never had a toolchain. If these assertions do not +# run, the lane proves nothing that the masked legs did not already prove. + +$ErrorActionPreference = 'Continue' +$failures = @() + +function Section($t) { Write-Host ""; Write-Host "=== $t ===" } + +# ── The interpreter itself ──────────────────────────────────────────────────── +# install.ps1 must run under Windows PowerShell 5.1, which is what a real Windows +# box ships. pwsh 7 is a runner-image extra no user is promised. nanoserver has +# NEITHER, which is why this lane is on servercore. +Section 'interpreter' +Write-Host "PSVersion : $($PSVersionTable.PSVersion)" +Write-Host "PSEdition : $($PSVersionTable.PSEdition)" +Write-Host "CLRVersion : $($PSVersionTable.CLRVersion)" +Write-Host "Host : $($Host.Name)" +if ($PSVersionTable.PSEdition -ne 'Desktop') { + $failures += "PSEdition is '$($PSVersionTable.PSEdition)', not Desktop -- this is not Windows PowerShell 5.1" +} +if ($PSVersionTable.PSVersion.Major -ne 5) { + $failures += "PSVersion is $($PSVersionTable.PSVersion), not 5.x" +} + +Section 'operating system' +cmd /c ver +$cv = Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion' -ErrorAction SilentlyContinue +if ($cv) { + Write-Host "ProductName : $($cv.ProductName)" + Write-Host "EditionID : $($cv.EditionID)" + Write-Host "InstallationType: $($cv.InstallationType)" + Write-Host "CurrentBuild : $($cv.CurrentBuild).$($cv.UBR)" +} +Write-Host "USERNAME : $env:USERNAME" +Write-Host "USERPROFILE : $env:USERPROFILE" +Write-Host "LOCALAPPDATA : $env:LOCALAPPDATA" +Write-Host "PROCESSOR_ARCH : $env:PROCESSOR_ARCHITECTURE" + +# install.ps1 line 254/258 does Join-Path $env:USERPROFILE ".unsloth\studio" with no +# null guard, so an unset USERPROFILE aborts under ErrorActionPreference=Stop. +# The lane sets UNSLOTH_STUDIO_HOME, but record whether a bare container would have +# survived without it. +if ([string]::IsNullOrWhiteSpace($env:USERPROFILE)) { + Write-Host "::warning::USERPROFILE is unset in this container; install.ps1's default install root would abort" +} + +# ── The assertion the whole lane exists for ─────────────────────────────────── +Section 'virginity: developer toolchain must be ABSENT' +# python/py/git/cmake/cl/winget are the six the task names. uv is added because +# install.ps1 would happily reuse a preinstalled one and skip its own bootstrap. +$mustBeAbsent = @('python', 'python3', 'py', 'git', 'cmake', 'cl', 'winget', 'uv') +foreach ($t in $mustBeAbsent) { + $c = Get-Command $t -ErrorAction SilentlyContinue + $where = if ($c) { $c.Source } else { 'ABSENT' } + Write-Host (" {0,-10} {1}" -f $t, $where) + if ($c) { $failures += "$t is present at $($c.Source) -- this container is NOT virgin" } +} + +Section 'informational: present but not a developer toolchain' +# These are OS components, not a toolchain. curl.exe and tar.exe ship in System32 on +# Server 2022 and are the only transport into a container with no git; naming them +# keeps the premise honest rather than silently relying on them. +foreach ($t in 'cmd', 'powershell', 'curl', 'tar', 'certutil', 'msiexec', 'reg', 'where', 'pwsh', 'node', 'npm', 'msbuild', 'dotnet', 'gcc') { + $c = Get-Command $t -ErrorAction SilentlyContinue + Write-Host (" {0,-10} {1}" -f $t, $(if ($c) { $c.Source } else { 'ABSENT' })) +} + +Section 'virginity: no toolchain on disk either' +# A binary can be absent from PATH and still be found by uv's own interpreter +# discovery or by py.exe's registry view -- that is exactly how the hosted Windows +# leg once reported `python ABSENT` and then installed with the runner's 3.13.14. +# Check the disk and the registry, not just PATH. +$badPaths = @( + 'C:\Python27', 'C:\Python3*', 'C:\Program Files\Python*', 'C:\Program Files (x86)\Python*', + 'C:\Program Files\Git', 'C:\Program Files\CMake', 'C:\Program Files\Microsoft Visual Studio', + 'C:\Program Files (x86)\Microsoft Visual Studio', 'C:\hostedtoolcache', 'C:\ProgramData\chocolatey' +) +foreach ($p in $badPaths) { + # Wildcards can match several directories; take the first so the message names a + # real path instead of stringifying an array. + $hit = @(Get-Item -Path $p -ErrorAction SilentlyContinue) | Select-Object -First 1 + if ($hit) { + Write-Host " PRESENT $($hit.FullName)" + $failures += "toolchain directory exists on disk: $($hit.FullName)" + } else { + Write-Host " absent $p" + } +} + +$pyReg = @('HKLM:\SOFTWARE\Python', 'HKCU:\SOFTWARE\Python') +foreach ($k in $pyReg) { + if (Test-Path $k) { + Write-Host " PRESENT $k" + $failures += "a registered Python install exists at $k" + } else { + Write-Host " absent $k" + } +} + +Section 'PATH as the container sees it' +Write-Host "Process PATH:" +($env:PATH -split ';') | Where-Object { $_ } | ForEach-Object { Write-Host " $_" } +foreach ($scope in 'Machine', 'User') { + Write-Host "$scope PATH: $([System.Environment]::GetEnvironmentVariable('Path', $scope))" +} + +# ── The VC++ runtime question the hosted leg cannot answer ──────────────────── +Section 'VC++ runtime (honest measurement)' +# clean-machine-install-ci.yml carries an explicit HONESTY NOTE that the hosted image +# ships the VC++ 2015-2022 runtime in System32 and it cannot be removed without +# breaking the runner, so `import torch` succeeding there does NOT prove a no-winget +# machine has the runtime. This container is the only environment in CI that can +# answer it, so their absence is asserted, not merely recorded: if a future base image +# starts shipping them the lane silently degrades into another masked leg. +foreach ($dll in 'vcruntime140.dll', 'vcruntime140_1.dll', 'msvcp140.dll') { + $p = Join-Path $env:WINDIR "System32\$dll" + $present = Test-Path $p + Write-Host (" {0,-20} {1}" -f $dll, $(if ($present) { 'PRESENT' } else { 'ABSENT' })) + if ($present) { $failures += "System32\$dll is present -- this image already ships the VC++ runtime, which is the one thing the hosted runner cannot un-ship" } +} +foreach ($k in 'HKLM:\SOFTWARE\Microsoft\VisualStudio\14.0\VC\Runtimes\x64', + 'HKLM:\SOFTWARE\WOW6432Node\Microsoft\VisualStudio\14.0\VC\Runtimes\x64') { + $r = Get-ItemProperty $k -ErrorAction SilentlyContinue + Write-Host (" {0} -> {1}" -f $k, $(if ($r) { "Installed=$($r.Installed) $($r.Major).$($r.Minor)" } else { 'absent' })) +} + +# ── Can the installer's transport work at all here? ─────────────────────────── +Section 'outbound HTTPS and TLS' +# install.ps1 never sets [Net.ServicePointManager]::SecurityProtocol, so it inherits +# the .NET Framework default. Test the DEFAULT first: if that fails and Tls12 works, +# the installer has a real portability bug on hardened images, not a container quirk. +Write-Host "default SecurityProtocol: $([Net.ServicePointManager]::SecurityProtocol)" +$probeUrls = @( + 'https://www.python.org/ftp/python/', + 'https://astral.sh/uv/install.ps1', + 'https://pypi.org/simple/', + 'https://aka.ms/vs/17/release/vc_redist.x64.exe' +) +$defaultOk = @{} +foreach ($u in $probeUrls) { + try { + $null = Invoke-WebRequest -Uri $u -UseBasicParsing -TimeoutSec 60 -Method Head -ErrorAction Stop + Write-Host " OK (default TLS) $u"; $defaultOk[$u] = $true + } catch { + Write-Host " FAIL (default TLS) $u -- $($_.Exception.Message)"; $defaultOk[$u] = $false + } +} +if ($defaultOk.Values -contains $false) { + Write-Host "retrying the failures with an explicit Tls12..." + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + foreach ($u in $probeUrls) { + if ($defaultOk[$u]) { continue } + try { + $null = Invoke-WebRequest -Uri $u -UseBasicParsing -TimeoutSec 60 -Method Head -ErrorAction Stop + Write-Host " OK (Tls12) $u" + Write-Host "::warning::$u needs an explicit Tls12; install.ps1 never sets SecurityProtocol, so this is a real installer portability gap" + } catch { + Write-Host " FAIL (Tls12) $u -- $($_.Exception.Message)" + $failures += "no outbound HTTPS to $u even with Tls12 -- the container cannot reach the installer's download hosts" + } + } +} + +# ── Verdict ─────────────────────────────────────────────────────────────────── +Section 'verdict' +if ($failures.Count -gt 0) { + foreach ($f in $failures) { Write-Host "::error::$f" } + Write-Host "VIRGINITY ASSERTION FAILED ($($failures.Count) problem(s))" + exit 1 +} +Write-Host "VIRGINITY ASSERTION PASSED" +Write-Host "no python, py, git, cmake, cl, winget or uv on PATH, on disk, or in the registry" +exit 0 diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index c7029ba54f..d0ec3391e1 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -62,6 +62,9 @@ on: # preinstalled Python and full developer tooling. - 'studio/backend/requirements/**' - '.github/scripts/clean-machine-*.sh' + # The virgin Windows container lane lives in this workflow too. + - '.github/scripts/virgin-windows-*.ps1' + - '.github/scripts/ensure-docker-daemon.ps1' - '.github/workflows/clean-machine-install-ci.yml' push: branches: [main] @@ -77,6 +80,9 @@ on: - 'studio/install_python_stack.py' - 'studio/backend/requirements/**' - '.github/scripts/clean-machine-*.sh' + # The virgin Windows container lane lives in this workflow too. + - '.github/scripts/virgin-windows-*.ps1' + - '.github/scripts/ensure-docker-daemon.ps1' - '.github/workflows/clean-machine-install-ci.yml' workflow_dispatch: inputs: @@ -733,19 +739,18 @@ jobs: winget: 'masked' experimental: false overlay: true - # Windows on ARM reaches the dependency install and stops on two packages - # that publish no win_arm64 wheel at all: - # pyarrow==25.0.0 (via datasets) -- PyPI has win_amd64 only, so uv falls - # back to the sdist and its CMake configure fails - # hf-transfer==0.1.9 -- a maturin/Rust sdist whose openssl-sys build script - # wants perl, which the image does not have - # The ARM handling itself works: the log shows "windows on arm: skipping - # torchaudio", and torch 2.10.0+cpu and torchvision install from wheels. What - # is left is a real product gap on this platform, not a gap in the simulation, - # so the leg stays experimental and keeps reporting it. + # Windows on ARM gets a native ARM64 CPython, and torchaudio has never + # published a win_arm64 wheel at any version, so the PyTorch step cannot + # resolve and install.ps1 stops at "Failed to install PyTorch". #7549 fixes it + # by preferring an x64 interpreter on an ARM64 host (x64 wheels run fine under + # emulation), and that fix lives in install.ps1 on an unmerged PR, so nothing + # in THIS branch can make the row green. The Install step is therefore + # continue-on-error and the step below pins that exact failure: the job stays + # required, so a DIFFERENT failure is still red, and the pin turns into a hard + # error the moment the installer starts picking an x64 Python. - os: windows-11-arm winget: 'visible' - experimental: true + experimental: false overlay: true steps: @@ -899,6 +904,8 @@ jobs: - name: Install id: install shell: pwsh + # ARM64 only: its failure is pinned below rather than gating (see the matrix). + continue-on-error: ${{ matrix.os == 'windows-11-arm' }} env: # Empty, and therefore ignored by install.ps1, on the non-overlay legs. UNSLOTH_CI_SOURCE_OVERLAY: ${{ matrix.overlay && inputs.installer_source != 'published' && github.workspace || '' }} @@ -930,6 +937,54 @@ jobs: Write-Host "installer exit code: $rc" exit $rc + # KNOWN OUTCOME PIN, retire when #7549 merges. continue-on-error on Install would + # otherwise tolerate a bootstrap outage or an unrelated early exit exactly like + # the intended diagnostic, so every branch here that is not the pinned failure + # exits 1 and fails the (required) job. + - name: Assert the windows-11-arm outcome is a known one + if: always() && matrix.os == 'windows-11-arm' && steps.install.outcome != 'skipped' + shell: pwsh + run: | + # The flip condition. #7549 makes install.ps1 prefer an x64 CPython on an ARM64 + # host, and torchaudio does publish win_amd64, so an AMD64 venv interpreter + # means the fix has landed and this pin is stale. Asked of the interpreter + # (sysconfig), not inferred from PROCESSOR_ARCHITECTURE, which describes the + # shell. The failing PyTorch step runs after the venv exists, so the absence of + # that interpreter is itself a different failure. + $venvPy = Join-Path $env:UNSLOTH_STUDIO_HOME 'unsloth_studio\Scripts\python.exe' + if (-not (Test-Path -LiteralPath $venvPy)) { + Write-Host "::error::no venv interpreter at $venvPy; the run did not even reach the pinned PyTorch failure" + exit 1 + } + $tag = (& $venvPy -c "import sysconfig; print(sysconfig.get_platform())" 2>&1 | Out-String).Trim() + $global:LASTEXITCODE = 0 + Write-Host "venv interpreter platform: $tag" + if ($tag -ne 'win-arm64') { + Write-Host "::error::the installer selected a '$tag' interpreter on this ARM64 host, so #7549 has landed. Delete this pin, drop continue-on-error from the Install step, and let the leg gate normally." + exit 1 + } + if ('${{ steps.install.outcome }}' -eq 'success') { + Write-Host '::error::the ARM64 leg installed successfully; the pinned failure is gone, so delete this pin and drop continue-on-error from the Install step' + exit 1 + } + if (-not (Test-Path logs/install.log)) { + Write-Host '::error::the ARM64 leg produced no install log' + exit 1 + } + $log = Get-Content logs/install.log -Raw + # All three, so a failure anywhere else in the installer is still red: it must + # be the PyTorch step, it must be about torchaudio, and it must be the missing + # win_arm64 platform tag rather than (say) a network error. + $atTorchStep = $log -match 'Failed to install PyTorch \(exit code' + $isTorchaudio = $log -match 'versions of torchaudio are available' + $isNoArmWheel = $log -match 'matching platform tag \(e\.g\., `win_arm64`\)' + Write-Host "PyTorch step: $atTorchStep / torchaudio: $isTorchaudio / no win_arm64 wheel: $isNoArmWheel" + if (-not ($atTorchStep -and $isTorchaudio -and $isNoArmWheel)) { + Write-Host '::error::the ARM64 leg did not fail at the pinned "torchaudio has no win_arm64 wheel" resolution error out of the PyTorch step; this is a new failure' + exit 1 + } + Write-Host '::notice::known outcome: a native ARM64 CPython plus no win_arm64 torchaudio wheel. Fixed by #7549 (prefer an x64 interpreter on ARM64 hosts); nothing in this branch can change it.' + # See the macOS job: proves the leg is testing what its matrix row claims. - name: Assert this ref's Python was really put under test if: matrix.overlay && inputs.installer_source != 'published' && steps.install.outcome == 'success' @@ -977,3 +1032,243 @@ jobs: path: logs/ retention-days: 7 if-no-files-found: warn + + # ── Windows, genuinely virgin: the same install inside a Windows container ──── + # The `win` legs above only SIMULATE absence, and two things they structurally + # cannot test are the VC++ 2015-2022 runtime (it ships in the runner image's + # System32 and cannot be removed without breaking the runner, hence the HONESTY + # NOTE on the torch assert) and a Windows with no Microsoft Store at all rather + # than a winget hidden from PATH. A servercore container answers both, so this + # lane lives here rather than in a sibling file: same premise, same path filters, + # and the reader compares masked against real in one place. + # + # Constraints, all load-bearing: + # * `container:` is Linux-only on the Actions runner (actions/runner#1402), so + # docker is driven from ordinary `run:` steps and the payload goes in by + # `docker cp` -- actions/checkout inside the container would need git. + # * servercore, not nanoserver: install.ps1 needs Windows PowerShell 5.1, which + # nanoserver does not ship at all. + # * windows-2022, not windows-latest: process isolation needs the host and + # container builds to match, and only the 2022 image pre-caches ltsc2022. + # windows-latest is Server 2025 and caches no Windows images. + windows_container_probe: + name: virgin win container / probe + runs-on: windows-2022 + timeout-minutes: 30 + env: + IMAGE: mcr.microsoft.com/windows/servercore:ltsc2022 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 1 + persist-credentials: false + + # Docker is installed on every windows-2022 image but is not always already + # running: one spike leg died in 21s on npipe:////./pipe/docker_engine, and that + # flake misreads as "Windows containers are unavailable". + - name: Ensure the Docker daemon is running + shell: pwsh + run: ./.github/scripts/ensure-docker-daemon.ps1 + + - name: Docker engine facts + shell: pwsh + run: | + New-Item -ItemType Directory -Force -Path logs | Out-Null + docker version + $osType = (docker info --format '{{.OSType}}').Trim() + Write-Host "OSType $osType / Isolation $(docker info --format '{{.Isolation}}')" + if ($osType -ne 'windows') { + Write-Host "::error::docker is serving '$osType' containers, not windows; this lane cannot run here" + exit 1 + } + docker images --format '{{.Repository}}:{{.Tag}} {{.Size}}' + + - name: Start the container + shell: pwsh + run: | + # Deliberately never refresh a cached image: process isolation needs the + # container build <= the host build, and MCR has shipped a patched image + # ahead of the runner host before (actions/runner-images#11582 broke every + # Windows container job for ~2 weeks). The cached one is the one that + # matched at runner-image build time. + if ((docker images --format '{{.Repository}}:{{.Tag}}') -contains $env:IMAGE) { + Write-Host "using the runner's pre-cached $env:IMAGE (no pull)" + } else { + Write-Host "::warning::$env:IMAGE is not pre-cached; pulling (slow, and it may outrun the host build)" + docker pull $env:IMAGE + if ($LASTEXITCODE -ne 0) { Write-Host "::error::could not pull $env:IMAGE"; exit 1 } + } + # A keepalive entrypoint so each assertion can be its own `docker exec`, and + # therefore its own step with its own exit code. + docker run -d --name virgin $env:IMAGE cmd /c "ping -t localhost >nul" + if ($LASTEXITCODE -ne 0) { Write-Host "::error::could not start a container from $env:IMAGE"; exit 1 } + Write-Host "isolation: $(docker inspect virgin --format '{{.HostConfig.Isolation}}')" + docker exec virgin cmd /c "mkdir C:\ci" + docker cp "$env:GITHUB_WORKSPACE\." virgin:C:\ci + docker exec virgin cmd /c "dir C:\ci\install.ps1" + + - name: Assert the container is genuinely virgin + shell: pwsh + run: | + docker exec virgin powershell.exe -NoLogo -NoProfile -NonInteractive ` + -ExecutionPolicy Bypass -File C:\ci\.github\scripts\virgin-windows-probe.ps1 ` + *>&1 | Tee-Object -FilePath logs/virginity.log + exit $LASTEXITCODE + + - name: Tear down + if: always() + shell: pwsh + run: | + docker rm -f virgin 2>&1 | Out-Null + exit 0 + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: virgin-win-probe + path: logs/ + retention-days: 7 + if-no-files-found: warn + + windows_container_install: + name: virgin win container / overlay=${{ matrix.overlay }} + needs: windows_container_probe + runs-on: windows-2022 + timeout-minutes: 90 + env: + IMAGE: mcr.microsoft.com/windows/servercore:ltsc2022 + strategy: + fail-fast: false + matrix: + include: + # The consumer path: install.ps1 from this ref, unsloth from PyPI. + - overlay: false + # This ref's studio/setup.ps1 and install_python_stack.py, via + # UNSLOTH_CI_SOURCE_OVERLAY (install.ps1:2643). Without it a branch changing + # setup.ps1 gets a green run that proves nothing about the change. + - overlay: true + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Ensure the Docker daemon is running + shell: pwsh + run: ./.github/scripts/ensure-docker-daemon.ps1 + + - name: Start the container + shell: pwsh + run: | + New-Item -ItemType Directory -Force -Path logs | Out-Null + # See the probe job: use the pre-cached image, never refresh it. + if (-not ((docker images --format '{{.Repository}}:{{.Tag}}') -contains $env:IMAGE)) { + Write-Host "::warning::$env:IMAGE not pre-cached; pulling" + docker pull $env:IMAGE + } + docker run -d --name virgin $env:IMAGE cmd /c "ping -t localhost >nul" + if ($LASTEXITCODE -ne 0) { Write-Host "::error::could not start a container from $env:IMAGE"; exit 1 } + docker exec virgin cmd /c "mkdir C:\ci" + docker cp "$env:GITHUB_WORKSPACE\." virgin:C:\ci + docker exec virgin cmd /c "mkdir C:\ci-out" + + # Re-run here and not only in `probe`: different runner, and an install leg that + # skipped the check would be reporting on an environment it never verified. + - name: Assert the container is genuinely virgin + shell: pwsh + run: | + docker exec virgin powershell.exe -NoLogo -NoProfile -NonInteractive ` + -ExecutionPolicy Bypass -File C:\ci\.github\scripts\virgin-windows-probe.ps1 ` + *>&1 | Tee-Object -FilePath logs/virginity.log + exit $LASTEXITCODE + + # AFTER the virginity assertion, so that assertion still proves what it says. + # A fresh container ships an almost empty trusted-root store; a real Windows + # desktop fills it via automatic root update, so seeding it makes the container + # MORE representative, not less. Needed because studio/install_node_prebuilt.py + # downloads Node with bare urllib.request.urlopen and so reads the empty Windows + # ROOT store and gets CERTIFICATE_VERIFY_FAILED; uv and pip bundle certifi and + # are unaffected. That product fragility is reported separately, not fixed here. + - name: Seed the container's trusted root CA store + shell: pwsh + run: | + docker exec virgin cmd /c "certutil -generateSSTFromWU C:\roots.sst && certutil -addstore -f Root C:\roots.sst" ` + *>&1 | Select-Object -Last 15 + if ($LASTEXITCODE -ne 0) { + Write-Host '::error::could not seed the container root CA store; Python-side HTTPS will fail' + exit 1 + } + + - name: Install into the virgin container + id: install + shell: pwsh + # KNOWN OUTCOME PIN, retire when #7549 merges. studio/setup.ps1 on this ref and + # in the released wheel both hard-stop on a winget-only git gate and reach for + # winget again for the VC++ runtime, and a Server Core container has no + # Microsoft Store and therefore no winget, ever. #7549 is what relaxes both, and + # it is not in this branch, so no change here can make either row pass. The job + # stays required and the step below decides: only the pinned signature is + # tolerated, and it errors out once the install starts working. + continue-on-error: true + run: | + $overlayArg = if ('${{ matrix.overlay }}' -eq 'true') { 'C:\ci' } else { '' } + docker exec virgin powershell.exe -NoLogo -NoProfile -NonInteractive ` + -ExecutionPolicy Bypass -File C:\ci\.github\scripts\virgin-windows-install.ps1 ` + -Overlay "$overlayArg" *>&1 | Tee-Object -FilePath logs/install-outer.log + exit $LASTEXITCODE + + - name: Assert the container install outcome is a known one + if: always() && steps.install.outcome != 'skipped' + shell: pwsh + run: | + if ('${{ steps.install.outcome }}' -eq 'success') { + Write-Host '::error::a virgin Windows container now installs, so #7549 has landed. Delete this pin and drop continue-on-error from the Install step so this row gates.' + exit 1 + } + if (-not (Test-Path logs/install-outer.log)) { + Write-Host '::error::the container install produced no log' + exit 1 + } + $log = Get-Content logs/install-outer.log -Raw + # The overlay hook is this PR's own feature and gates unconditionally: without + # this the overlay row would be indistinguishable from the released-wheel row, + # since on this ref both stop at the same gate. + if ('${{ matrix.overlay }}' -eq 'true' -and -not ($log -match 'CI: overlaying source checkout')) { + Write-Host '::error::the overlay row never overlaid the checkout, so it only tested the released package' + exit 1 + } + # The two winget-only gates this lane exists to surface. Anything else is a new + # problem and must be red. + $gitGate = $log -match 'Git is required but could not be installed automatically' + $vcGate = $log -match 'torch failed to import' + if (-not ($gitGate -or $vcGate)) { + Write-Host '::error::the container install failed at neither the winget-only git gate nor the missing VC++ runtime; this is a new failure' + exit 1 + } + if ($gitGate) { Write-Host '::notice::known outcome: winget-only git gate (studio/setup.ps1), fixed by #7549' } + if ($vcGate) { Write-Host '::notice::known outcome: no VC++ runtime and Ensure-VCRedist is winget-only, fixed by #7549' } + + - name: Recover the install log from the container + if: always() + shell: pwsh + run: | + docker cp virgin:C:\ci-out\install.log logs/install.log 2>&1 | Out-Null + if (Test-Path logs/install.log) { Write-Host "recovered $((Get-Item logs/install.log).Length) bytes" } + else { Write-Host '::warning::no install log inside the container' } + exit 0 + + - name: Tear down + if: always() + shell: pwsh + run: | + docker rm -f virgin 2>&1 | Out-Null + exit 0 + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: virgin-win-container-overlay-${{ matrix.overlay }} + path: logs/ + retention-days: 7 + if-no-files-found: warn diff --git a/.github/workflows/desktop-app-clean-machine-ci.yml b/.github/workflows/desktop-app-clean-machine-ci.yml index a2e92a271b..7d39269888 100644 --- a/.github/workflows/desktop-app-clean-machine-ci.yml +++ b/.github/workflows/desktop-app-clean-machine-ci.yml @@ -359,10 +359,32 @@ jobs: fi [ -n "$SH" ] && [ -f "$SH" ] || { echo "::error::the bundle ships no install.sh resource"; exit 1; } echo "bundled installer: $SH" + # KNOWN OUTCOME PIN, retire when the desktop release catches up to #7547. + # The bundle carries its own install.sh, and REL_TAG predates #7547, so on a + # stripped runner it still exits 2 at the NEED_SUDO handshake for the optional + # set instead of falling through to prebuilt llama.cpp. No change to this PR + # can move that; only a new release can. _SMART_APT_OPTIONAL is the guard #7547 + # added, so finding it means the release caught up and this pin must go. + if grep -q '_SMART_APT_OPTIONAL' "$SH"; then + echo "::error::the bundled install.sh now carries #7547; delete this pin block and let the venv + torch assertions below run unconditionally" + exit 1 + fi # --tauri rejects a custom studio home (install.sh:102-114), so drop the # workspace-scoped override, and close stdin as install.rs does. + rc=0 env -u UNSLOTH_STUDIO_HOME \ - bash "$SH" --tauri < /dev/null 2>&1 | tee logs/bundled-install.log + bash "$SH" --tauri < /dev/null 2>&1 | tee logs/bundled-install.log || rc=$? + echo "bundled installer exit code: $rc" + # Exit code AND the exact optional set, so a different NEED_SUDO list or any + # other non-zero exit is still a failure. + if [ "$rc" -eq 2 ] && grep -qE '^\[TAURI:NEED_SUDO\] cmake git build-essential libcurl4-openssl-dev[[:space:]]*$' logs/bundled-install.log; then + echo "::notice::known pre-#7547 outcome: the shipped bundle's install.sh asked to elevate for the optional set and exited 2. Not a regression here; the next desktop release retires this pin." + exit 0 + fi + [ "$rc" -eq 0 ] || { + echo "::error::bundled installer exited $rc, which is neither success nor the pinned pre-#7547 outcome (exit 2 plus exactly '[TAURI:NEED_SUDO] cmake git build-essential libcurl4-openssl-dev')" + exit 1 + } PY="$HOME/.unsloth/studio/unsloth_studio/bin/python" [ -x "$PY" ] || { echo "::error::bundled installer left no venv at $PY"; exit 1; } "$PY" -V From 9b0f6d76d4ea915db1e0546190dfd67d86493763 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 02:44:02 +0000 Subject: [PATCH 22/36] Check signatures on Mach-O main executables only The macho check asserted a valid signature for every Mach-O under the studio home, and failed the macos-15 mask/pipe leg on 29 files: lxml, charset_normalizer, cygrpc, _upb, fontTools, caio, brotli and a bundled libportaudio.dylib. Those are MH_BUNDLE and MH_DYLIB images dlopen'd into a process without library validation, they ship unsigned in the wheels, and the same run had already installed and imported them with the installer exiting 0. Key the signature half off the Mach-O filetype and run it only on main executables. Report an absent seal separately from one that fails to verify, and capture codesign output instead of piping it into grep, which returned the unsigned exit status through pipefail and called every unsigned binary broken. The architecture half is unchanged and still a hard failure: it is what closes the Rosetta 2 gap. The zero-Mach-O guard is unchanged. The .venv_t5_* sidecars stay in scope; setup.sh creates them during a normal install and transformers_version.py puts them on sys.path, so they are payload. --- .github/scripts/clean-machine-assert.sh | 66 +++++++++++++++++++++---- 1 file changed, 56 insertions(+), 10 deletions(-) diff --git a/.github/scripts/clean-machine-assert.sh b/.github/scripts/clean-machine-assert.sh index 4fb2b5291f..94563bc3fc 100755 --- a/.github/scripts/clean-machine-assert.sh +++ b/.github/scripts/clean-machine-assert.sh @@ -12,8 +12,9 @@ # "Building ==" from uv. Needs UNSLOTH_VERBOSE=1, or # run_install_cmd (install.sh:193-243) discards uv's output on success # and there is nothing to read. -# macho Every Mach-O under $MACHO_ROOT is the host architecture and is signed. -# Closes the Rosetta 2 gap, the one divergence masking cannot reproduce. +# macho Every Mach-O under $MACHO_ROOT is the host architecture, and every +# Mach-O MAIN EXECUTABLE is signed. Closes the Rosetta 2 gap, the one +# divergence masking cannot reproduce. # # Usage: bash .github/scripts/clean-machine-assert.sh absent notools nobuild macho set -uo pipefail @@ -160,13 +161,18 @@ for check in "$@"; do # architecture rather than hope the runner lacks Rosetta. # `lipo` is an xcrun shim and is gone after masking, so read `file -b`, exactly # as the desktop lane does. Keyed off `uname -m`, since macos-15-intel is x86_64. + # + # SCOPE: all of $MACHO_ROOT, including the .venv_t5_510/_530/_550 sidecars. + # Those are payload, not scratch: setup.sh:579-581 creates them during a + # normal install and transformers_version.py:338-348 puts them on sys.path. + # Any exclusion must be a named path rule, never a narrowed find. root="${MACHO_ROOT:-${UNSLOTH_STUDIO_HOME:-$HOME/.unsloth}}" want="$(uname -m)" [ "$want" = "aarch64" ] && want=arm64 if [ ! -d "$root" ]; then fail "macho requested but $root does not exist" else - n=0 bad_arch="" unsigned="" + n=0 nexe=0 bad_arch="" unsigned="" broken="" while IFS= read -r f; do desc="$(file -b "$f" 2>/dev/null || true)" case "$desc" in *Mach-O*) ;; *) continue ;; esac @@ -177,11 +183,49 @@ for check in "$@"; do *"$want"*) ;; *) bad_arch="$bad_arch $f [$desc]" ;; esac - # arm64 only: AMFI SIGKILLs unsigned code there ("Killed: 9"), while x86_64 - # loads it happily, so an unsigned x86_64 payload is not the same defect. - # Ad-hoc is enough, which is what the linker emits by default. - if [ "$want" = "arm64" ] && ! codesign -v "$f" >/dev/null 2>&1; then - unsigned="$unsigned $f" + + # Signature: MAIN EXECUTABLES ONLY. Asserting it for every Mach-O failed + # the mask/pipe leg on 29 ordinary PyPI extension modules (lxml, + # charset_normalizer, cygrpc, fontTools, ...) plus libportaudio.dylib. + # The premise was wrong: those are MH_BUNDLE/MH_DYLIB images dlopen'd + # into a process without library validation and ship unsigned, and the + # run that flagged them had already imported them with the installer + # exiting 0. Enforcement lands on main executables and gatekept .app + # bundles, so that is all this asserts. + # + # Key off the filetype `file` reports, not the path or extension: a .so + # may be a bundle or a dylib, and an executable may have no extension. + # The library veto is second so a mixed-type fat file counts as a + # library. Substring tests are order-independent: Apple's `file` prints + # `Mach-O 64-bit executable arm64`, GNU's `Mach-O 64-bit arm64 executable`. + _is_exe=0 + case "$desc" in *executable*) _is_exe=1 ;; esac + case "$desc" in *"shared library"*|*bundle*) _is_exe=0 ;; esac + # Named rule so a failure says which path matched; the filetype test + # already covers the MH_EXECUTE at .app/Contents/MacOS/. + case "$f" in *.app/Contents/MacOS/*) _is_exe=1 ;; esac + [ "$_is_exe" = 1 ] && nexe=$((nexe + 1)) + + # arm64 only: the kernel refuses to exec an unsigned arm64 main binary + # ("Killed: 9"), while x86_64 execs it happily, so an unsigned x86_64 + # payload is not the same defect. + if [ "$want" = "arm64" ] && [ "$_is_exe" = 1 ]; then + # Ad-hoc counts as signed: arm64 linkers apply an ad-hoc seal by + # default, so the test is "has a seal that verifies", not "has an + # identity". `spctl`/`--strict` would demand an authority and reject + # ad-hoc, so neither is used. + if ! codesign -v "$f" >/dev/null 2>&1; then + # Nothing to verify and a seal that does not match mean different + # things. Captured, not piped into grep: `codesign -dvv` exits + # non-zero on an unsigned file, and under the `pipefail` above that + # status is what `codesign ... | grep -q` returns even on a match, + # reporting every unsigned binary as a broken signature. + _sig="$(codesign -dvv "$f" 2>&1 || true)" + case "$_sig" in + *"not signed at all"*) unsigned="$unsigned $f" ;; + *) broken="$broken $f" ;; + esac + fi fi done < <(find "$root" -type f \( -perm -u+x -o -name '*.dylib' -o -name '*.so' -o -name '*.node' \) 2>/dev/null) if [ "$n" = "0" ]; then @@ -191,9 +235,11 @@ for check in "$@"; do elif [ -n "$bad_arch" ]; then fail "Mach-O is not $want, so it runs here only under Rosetta 2, which a fresh Mac does not have:$bad_arch" elif [ -n "$unsigned" ]; then - fail "unsigned Mach-O, which AMFI kills on arm64:$unsigned" + fail "unsigned Mach-O main executable, which arm64 macOS refuses to exec:$unsigned" + elif [ -n "$broken" ]; then + fail "Mach-O main executable carries a signature that does not verify:$broken" else - ok "$n Mach-O files under $root are $want$([ "$want" = arm64 ] && echo ' and signed')" + ok "$n Mach-O files under $root are $want$([ "$want" = arm64 ] && echo "; all $nexe main executable(s) signed")" fi fi ;; From 58e5fa5e6e82a3f824b2f24aeb2cda576e2693eb Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 02:52:45 +0000 Subject: [PATCH 23/36] Make the WSL job gate, assert Windows installed no toolchain, strip before the .deb --- .../workflows/clean-machine-install-ci.yml | 51 ++++++++++++++++++- .../desktop-app-clean-machine-ci.yml | 30 +++++++---- 2 files changed, 69 insertions(+), 12 deletions(-) diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index d0ec3391e1..bec5ec5a67 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -593,11 +593,16 @@ jobs: # No third-party action: the official Ubuntu rootfs plus `wsl --import` is # deterministic and checksum-verifiable, adding no supply-chain dependency to a repo # that audits its lockfiles. + # + # Gating, deliberately: this is the only job that runs the real WSL branch, so a + # job-level continue-on-error made the distro import, the installer exit code, the + # `platform wsl` assertion and the CLI check all unable to fail anything. Eight + # consecutive staging runs were green through every step, so there is no flake to + # absorb; if the pinned rootfs ever moves, a red job is the correct signal. wsl: name: wsl ubuntu-24.04 runs-on: windows-latest timeout-minutes: 50 - continue-on-error: true steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -996,6 +1001,50 @@ jobs: } Write-Host "overlay applied; this leg exercised this ref's Python" + - name: Assert the install added no compiler toolchain + if: always() && steps.install.outcome != 'skipped' + shell: pwsh + run: | + # macOS and Linux re-run `absent`/`nobuild` after the install; Windows checked + # nothing afterwards, so setup.ps1 committing to a llama.cpp SOURCE build would + # winget-install CMake (setup.ps1:816-822) and VS Build Tools (845-857) and the + # leg still went green. Git is out of scope on purpose: bootstrapping it through + # winget (setup.ps1:1658-1661) is the consumer path the visible leg exists to + # exercise. The VC++ runtime is a runtime, not a toolchain, and is likewise fine. + $bad = @() + if (-not (Test-Path logs/install.log)) { + Write-Host '::error::no install log, so nothing proves the install stayed toolchain-free' + exit 1 + } + # The announcements inside Ensure-BuildToolsForLlamaSourceBuild, which runs only + # when a source build is committed. Matched instead of the package ids: setup.ps1 + # PRINTS `winget install Microsoft.VisualStudio.2022.BuildTools` as manual advice + # when winget is missing, and advice is not an install. + foreach ($m in 'CMake not found -- installing via winget', + 'Visual Studio Build Tools not found -- installing via winget') { + if (Select-String -Path logs/install.log -Pattern $m -SimpleMatch -Quiet) { + $bad += "install log reports: $m" + } + } + # winget puts what it installs on the MACHINE PATH, which this step's own + # process PATH (scrubbed, from GITHUB_ENV) never sees, so read the registry + # copies back rather than ask Get-Command. The scrub already removed every + # entry matching these, so a match here means the install put one back. + foreach ($scope in 'Machine','User') { + $raw = [System.Environment]::GetEnvironmentVariable('Path', $scope) + if ([string]::IsNullOrWhiteSpace($raw)) { continue } + foreach ($e in ([System.Environment]::ExpandEnvironmentVariables($raw) -split ';')) { + if ($e -match 'CMake|BuildTools|Microsoft Visual Studio|LLVM') { + $bad += "$scope PATH regained $e" + } + } + } + if ($bad) { + Write-Host "::error::the install put a compiler toolchain on this machine: $($bad -join '; ')" + exit 1 + } + Write-Host 'no CMake and no VS Build Tools install; the prebuilt contract held' + - name: Assert torch loads, and record what that does and does not prove if: steps.install.outcome == 'success' shell: pwsh diff --git a/.github/workflows/desktop-app-clean-machine-ci.yml b/.github/workflows/desktop-app-clean-machine-ci.yml index 7d39269888..2f45576483 100644 --- a/.github/workflows/desktop-app-clean-machine-ci.yml +++ b/.github/workflows/desktop-app-clean-machine-ci.yml @@ -306,6 +306,25 @@ jobs: gh release download "$REL_TAG" --repo "$REL_REPO" --pattern "$pat" --dir dl ls -la dl + - name: Strip the developer toolchain + # Same gate as macOS. Without this the Linux rows ignored strip_toolchain + # entirely and ran the bundled installer with the runner's git, gcc, cmake and + # make in /usr/bin, so a bundle that needs a toolchain passed the one workflow + # whose premise is that it must not. + # + # BEFORE the bundle install, as macOS and Windows already do: dpkg runs the + # package's own maintainer scripts, and installing first meant they ran with the + # hosted image's git, compilers and cmake in /usr/bin, so a release whose scripts + # reached for one would pass here and fail on a clean machine. Nothing in that + # install needs a masked tool -- clean-machine-env.sh moves aside only $TOOLS + # (compilers, git, cmake, make, brew, cargo), leaving apt, dpkg and sudo -- and + # the current bundle ships a postrm and no install-time script at all. + if: ${{ github.event_name != 'workflow_dispatch' || inputs.strip_toolchain }} + run: | + bash .github/scripts/clean-machine-env.sh mask --remove + set -a; . ./clean-machine.env; set +a + bash .github/scripts/clean-machine-assert.sh absent + - name: Install with NO dev tooling, only runtime libs run: | # Deliberately not build-essential/cmake/git: a user installing a .deb has @@ -328,17 +347,6 @@ jobs: echo "BIN=$BIN" >> "$GITHUB_ENV" echo "binary: $BIN" - - name: Strip the developer toolchain - # Same gate as macOS. Without this the Linux rows ignored strip_toolchain - # entirely and ran the bundled installer with the runner's git, gcc, cmake and - # make in /usr/bin, so a bundle that needs a toolchain passed the one workflow - # whose premise is that it must not. After apt: the .deb install needs dpkg. - if: ${{ github.event_name != 'workflow_dispatch' || inputs.strip_toolchain }} - run: | - bash .github/scripts/clean-machine-env.sh mask --remove - set -a; . ./clean-machine.env; set +a - bash .github/scripts/clean-machine-assert.sh absent - - name: Run the bundled installer, the path first launch takes run: | set -a; [ -f ./clean-machine.env ] && . ./clean-machine.env; set +a From d1223f7cc002bc26bcabfef06b0f44c432a0068c Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 02:56:15 +0000 Subject: [PATCH 24/36] Assert the root Linux legs did not compile llama.cpp with the apt-installed toolchain --- .../workflows/clean-machine-install-ci.yml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index bec5ec5a67..fa033c4a4b 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -564,6 +564,33 @@ jobs: "$VENV/bin/python" -V [ -x "$VENV/bin/unsloth" ] || { echo "::error::installer exited 0 but left no unsloth CLI at $VENV/bin/unsloth"; exit 1; } + - name: Assert the apt-installed toolchain did not compile llama.cpp + if: steps.install_root.outcome == 'success' + run: | + # HONESTY NOTE: these legs START toolchain-free but do not stay that way. + # As root, _smart_apt_install's first `apt-get install` (install.sh:786-788) + # succeeds before the _SMART_APT_OPTIONAL guard (805-809) can suppress + # anything, so `cmake git build-essential libcurl4-openssl-dev` really are + # installed mid-run. That is product behaviour on any root Linux install, not + # a CI artefact. What must still hold is that nothing USED them: `nobuild` + # reads Python builds only, and llama.cpp is the one thing that silently falls + # back to a source compile once a compiler is around. + for t in cmake git gcc; do + printf '%-6s %s\n' "$t" "$(command -v "$t" 2>/dev/null || echo ABSENT)" + done + # install_llama_prebuilt.py:5629 writes this marker; a source-built tree has + # no such metadata (studio/setup.sh:1517), so its presence is the one + # unambiguous "the prebuilt path won" signal. + META="$UNSLOTH_STUDIO_HOME/llama.cpp/UNSLOTH_PREBUILT_INFO.json" + if [ ! -f "$META" ]; then + echo "::error::llama.cpp carries no prebuilt metadata at $META, so it did not come from the prebuilt bundle; with the compiler installed above, that is the silent source build this workflow exists to rule out" + ls -la "$UNSLOTH_STUDIO_HOME/llama.cpp" 2>/dev/null || true + grep -nE "llama\.cpp|prebuilt" logs/install.log | tail -30 || true + exit 1 + fi + echo "llama.cpp came from the prebuilt bundle:" + head -c 800 "$META"; echo + - name: Assert no source build if: always() run: | From c6a1174e67ca2f42f3f445290c945c2a0f54b769 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 03:19:21 +0000 Subject: [PATCH 25/36] Pin the macOS desktop legs on the same pre-7547 release lag The Linux rows already pin the shipped bundle's own install.sh exiting 2 at the NEED_SUDO handshake. macos-15 and macos-26 fail the same way for the same reason: desktop-v0.1.50-beta predates #7547, so the bundled installer still hard-exits on the Xcode CLT gate that #7547 turned into a warning. Accept exit 1 plus that exact gate line, and nothing else. _check_macos_deps is the function #7547 added, so its presence in the bundle means the release caught up and the block errors out asking for the pin to be deleted. --- .../desktop-app-clean-machine-ci.yml | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/.github/workflows/desktop-app-clean-machine-ci.yml b/.github/workflows/desktop-app-clean-machine-ci.yml index 2f45576483..ee332dfa72 100644 --- a/.github/workflows/desktop-app-clean-machine-ci.yml +++ b/.github/workflows/desktop-app-clean-machine-ci.yml @@ -185,9 +185,30 @@ jobs: # passes without ever running the bundled installer. Invoke it as # src-tauri/src/install.rs does: --tauri, stdin closed, no tty. --tauri # rejects a custom studio home (install.sh:102-114), so drop the override. + # KNOWN OUTCOME PIN, retire when the desktop release catches up to #7547. + # REL_TAG predates #7547, so the bundle's own install.sh still hard-exits on + # the Xcode CLT gate that #7547 replaced with a warning. No change to this PR + # can move that; only a new release can. _check_macos_deps is the function + # #7547 added, so finding it means the release caught up and this pin must go. + SH="$APP/Contents/Resources/install.sh" + if grep -q '_check_macos_deps' "$SH"; then + echo "::error::the bundled install.sh now carries #7547; delete this pin block and let the venv + torch assertions below run unconditionally" + exit 1 + fi + rc=0 env -u UNSLOTH_STUDIO_HOME \ - bash "$APP/Contents/Resources/install.sh" --tauri \ - < /dev/null 2>&1 | tee logs/bundled-install.log + bash "$SH" --tauri \ + < /dev/null 2>&1 | tee logs/bundled-install.log || rc=$? + echo "bundled installer exit code: $rc" + # Exit code AND the exact gate line, so any other non-zero exit still fails. + if [ "$rc" -eq 1 ] && grep -qE '^==> Xcode Command Line Tools are required\.[[:space:]]*$' logs/bundled-install.log; then + echo "::notice::known pre-#7547 outcome: the shipped bundle's install.sh stopped on the Xcode CLT gate and exited 1. Not a regression here; the next desktop release retires this pin." + exit 0 + fi + [ "$rc" -eq 0 ] || { + echo "::error::bundled installer exited $rc, which is neither success nor the pinned pre-#7547 outcome (exit 1 plus '==> Xcode Command Line Tools are required.')" + exit 1 + } PY="$HOME/.unsloth/studio/unsloth_studio/bin/python" [ -x "$PY" ] || { echo "::error::bundled installer left no venv at $PY"; exit 1; } "$PY" -V From 0f760008ce373eacc25365454d382995638f7ff5 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 03:58:54 +0000 Subject: [PATCH 26/36] Pin the WSL pipe truncation and the masked-winget git gate The WSL leg dies at install.sh:2082 with an unterminated quoted string. Nothing is wrong with that line: piping the script into sh is not atomic. dash reads it from the pipe in 8192-byte blocks and runs each command as it parses, and install.sh:2007 calls _maybe_reroute_strixhalo_to_2404, which on WSL alone shells out to Windows interop; interop relays the stdin it inherited and drains the pipe. dash has 11 blocks buffered at that point, ending at byte 90112, which falls inside "$STUDIO_LOCAL_INSTALL" on line 2082. Truncating install.sh at 90112 and parsing it reproduces the message verbatim, and running the whole file under a stdin-draining interop stub reproduces the exit code too. #7548 wraps the body in _unsloth_main so sh parses everything before running anything, and the same reproduction against its head is clean. The eight green staging runs cited when this job's continue-on-error came off were all on trees that already carried #7548, so that evidence never covered this branch. Pin the exact signature instead: exit 2 plus the shell's own unterminated-quoted-string error, with the _unsloth_main marker read back out of the distro as the flip condition. Pin winget=masked the same way. studio/setup.ps1:1655-1669 gates on git unconditionally and can only fetch it through winget, so masking winget leaves no way to satisfy it. #7549 relaxes the gate, and its wording appearing in the tree retires the pin. --- .../workflows/clean-machine-install-ci.yml | 117 ++++++++++++++++-- 1 file changed, 109 insertions(+), 8 deletions(-) diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index fa033c4a4b..99d31dae13 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -623,9 +623,15 @@ jobs: # # Gating, deliberately: this is the only job that runs the real WSL branch, so a # job-level continue-on-error made the distro import, the installer exit code, the - # `platform wsl` assertion and the CLI check all unable to fail anything. Eight - # consecutive staging runs were green through every step, so there is no flake to - # absorb; if the pinned rootfs ever moves, a red job is the correct signal. + # `platform wsl` assertion and the CLI check all unable to fail anything. There is no + # flake to absorb; if the pinned rootfs ever moves, a red job is the correct signal. + # + # The eight green staging runs cited when that continue-on-error came off do not + # transfer to this branch: every one of them ran a tree that already carried #7548 + # (installer-fix-ci and integ-installer-fixes-ci both have `_unsloth_main` in + # install.sh), and #7548 is exactly what makes the script survive being piped. This + # branch does not carry it, so the Install step pins that one difference and + # everything else in this job still gates. wsl: name: wsl ubuntu-24.04 runs-on: windows-latest @@ -657,6 +663,7 @@ jobs: wsl -d unsloth-ci -- sh -c 'for t in curl wget git gcc cmake python3 sudo; do printf "%-8s %s\n" "$t" "$(command -v $t || echo ABSENT)"; done' - name: Install inside WSL, piped exactly as documented + id: install shell: pwsh run: | # Only ca-certificates + curl: the advertised one-liner cannot start without a @@ -685,6 +692,41 @@ jobs: wsl -d unsloth-ci -u root -- sh -c 'cd /root && cat install.sh | sh' 2>&1 | Tee-Object -FilePath logs/wsl-install.log $installRc = $LASTEXITCODE Write-Host "installer exit: $installRc" + + # KNOWN OUTCOME PIN, retire when #7548 merges. Piping a 218KB script into sh is + # not atomic: dash reads it from the pipe in 8192-byte blocks and runs each + # command as soon as it parses, so any command that inherits fd 0 and reads to + # EOF swallows the rest of the script. install.sh:2007 runs + # _maybe_reroute_strixhalo_to_2404, which on WSL alone shells out to Windows + # interop (powershell.exe for the WMI GPU name, then wsl.exe -l -q), and interop + # relays the stdin it inherited. dash has exactly 11 blocks buffered by then, + # ending at byte 90112, which lands inside "$STUDIO_LOCAL_INSTALL" on line 2082 + # -- hence that line number rather than anything wrong with it. Truncating + # install.sh at 90112 and parsing it reproduces the message verbatim. Every + # other platform runs the same file and stays green because nothing in that + # window touches stdin. #7548 wraps the body in _unsloth_main so sh must parse + # the whole script before running any of it, which is why that marker is what + # retires this pin. Read it out of the distro so it tracks the script that + # actually ran, published or not. + wsl -d unsloth-ci -u root -- sh -c 'grep -q "^_unsloth_main()" /root/install.sh' + if ($LASTEXITCODE -eq 0) { + Write-Host '::error::the installer under test now carries #7548; delete this pin block and let the exit-code check below gate unconditionally' + exit 1 + } + if (-not (Test-Path logs/wsl-install.log)) { + Write-Host '::error::the WSL install produced no log' + exit 1 + } + # Exit code AND the shell's own parse error, so any other non-zero exit, and any + # other syntax error, still fails this required job. The line number is not + # matched: it moves with any edit to install.sh while the failure is the same. + $truncated = Select-String -Path logs/wsl-install.log ` + -Pattern 'sh: [0-9]+: Syntax error: Unterminated quoted string' -Quiet + if ($installRc -eq 2 -and $truncated) { + Write-Host '::notice::known outcome: install.sh is not pipe-safe, so WSL interop drained the pipe and sh hit EOF mid-string. Fixed by #7548; nothing in this branch can change it.' + "pinned=true" | Out-File -FilePath $env:GITHUB_OUTPUT -Append + exit 0 + } # Printing the code discarded it, and the next step's CLI check does not # compensate: install.sh links the `unsloth` shim (4174-4182) BEFORE it reports # a failing studio/setup.sh (4219-4230), so a late setup failure leaves a shim @@ -716,6 +758,14 @@ jobs: Write-Host '::error::installer never reported ''platform wsl''; the WSL branch was not exercised' exit 1 } + # Only the CLI half is waived under the pin: the truncation kills the run + # before anything is installed, so there is nothing to find. The `platform wsl` + # assertion above still gates either way, because the installer prints it + # several hundred lines before the point where the pipe runs dry. + if ('${{ steps.install.outputs.pinned }}' -eq 'true') { + Write-Host '::notice::CLI check waived: the Install step recorded the pinned pre-#7548 pipe truncation' + exit 0 + } # No `|| echo`: substituting a message for the missing CLI made the inner # shell, this step and the job all succeed even when the install produced # nothing usable, which is half of what this step asks. @@ -764,9 +814,11 @@ jobs: # "Git is required but could not be installed automatically" gate: no winget # means no way to fetch git. That gate is what #7549 relaxes to the --local and # llama.cpp source paths that actually use git; with it applied the leg is - # green (staging run 30407859691, all 16 legs). So this row stays required: it - # is a merge-order dependency, not a product gap, and the overlay is what lets - # this workflow see the fix land. + # green (staging run 30407859691, all 16 legs). It is a merge-order dependency, + # not a product gap, and the overlay is what lets this workflow see the fix + # land. The Install step is therefore continue-on-error and the step below pins + # that exact failure: the row stays required, so a DIFFERENT failure is still + # red, and the pin turns into a hard error the moment the relaxed gate appears. - os: windows-latest winget: 'masked' experimental: false @@ -936,8 +988,9 @@ jobs: - name: Install id: install shell: pwsh - # ARM64 only: its failure is pinned below rather than gating (see the matrix). - continue-on-error: ${{ matrix.os == 'windows-11-arm' }} + # ARM64 and winget=masked only: both failures are pinned below rather than + # gating (see the matrix). + continue-on-error: ${{ matrix.os == 'windows-11-arm' || matrix.winget == 'masked' }} env: # Empty, and therefore ignored by install.ps1, on the non-overlay legs. UNSLOTH_CI_SOURCE_OVERLAY: ${{ matrix.overlay && inputs.installer_source != 'published' && github.workspace || '' }} @@ -1017,6 +1070,54 @@ jobs: } Write-Host '::notice::known outcome: a native ARM64 CPython plus no win_arm64 torchaudio wheel. Fixed by #7549 (prefer an x64 interpreter on ARM64 hosts); nothing in this branch can change it.' + # KNOWN OUTCOME PIN, retire when #7549 merges. Same reason as the ARM64 pin above: + # continue-on-error on Install would otherwise tolerate a bootstrap outage or an + # unrelated early exit exactly like the intended diagnostic, so every branch here + # that is not the pinned failure exits 1 and fails the (required) job. + - name: Assert the winget=masked outcome is a known one + if: always() && matrix.winget == 'masked' && steps.install.outcome != 'skipped' + shell: pwsh + run: | + # The flip condition. studio/setup.ps1:1655-1669 gates on git unconditionally + # and can only get it from winget, so masking winget leaves no way to satisfy + # it; #7549 relaxes the gate to the --local and llama.cpp source-build paths + # that actually use git, so that wording being present means the fix landed. + # Read from the checkout because this row is overlay: true, which is what makes + # the branch's setup.ps1 the one that runs (install.ps1:2626-2642). A published + # dispatch takes setup.ps1 from the released wheel instead, so there the tree + # says nothing and only the outcome checks below can retire the pin. + if ('${{ inputs.installer_source }}' -ne 'published') { + $relaxed = Select-String -Path studio/setup.ps1 -SimpleMatch -Quiet ` + -Pattern 'Git is required for --local and llama.cpp source-build installs' + if ($relaxed) { + Write-Host '::error::studio/setup.ps1 now carries the relaxed #7549 git gate; delete this pin and drop winget=masked from continue-on-error on the Install step so this row gates.' + exit 1 + } + } + if ('${{ steps.install.outcome }}' -eq 'success') { + Write-Host '::error::the masked leg installed successfully; the pinned failure is gone, so delete this pin and drop winget=masked from continue-on-error on the Install step' + exit 1 + } + if (-not (Test-Path logs/install.log)) { + Write-Host '::error::the masked leg produced no install log' + exit 1 + } + $log = Get-Content logs/install.log -Raw + # All three, so a failure anywhere else is still red: winget really was absent, + # the git gate is what fired, and it is what stopped studio setup rather than + # a warning the install walked past. The winget check is not redundant with the + # matrix -- without it this pin would also absorb a leg whose PATH scrub failed + # and which then died on the same gate for an entirely different reason. + $noWinget = $log -match 'will require Python \+ uv to be already installed' + $gitGate = $log -match 'Git is required but could not be installed automatically' + $setupRc = $log -match 'unsloth studio setup failed \(exit code 1\)' + Write-Host "no winget: $noWinget / git gate: $gitGate / setup exit 1: $setupRc" + if (-not ($noWinget -and $gitGate -and $setupRc)) { + Write-Host '::error::the masked leg did not stop at the winget-only git gate in studio/setup.ps1; this is a new failure' + exit 1 + } + Write-Host '::notice::known outcome: setup.ps1 gates on git unconditionally and can only fetch it through winget, which this row masks. Fixed by #7549 (relax the gate to --local and source-build installs); nothing in this branch can change it.' + # See the macOS job: proves the leg is testing what its matrix row claims. - name: Assert this ref's Python was really put under test if: matrix.overlay && inputs.installer_source != 'published' && steps.install.outcome == 'success' From 04ca461b554b10b4c35779633876e34c79fb6071 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 04:18:57 +0000 Subject: [PATCH 27/36] Retire the WSL pipe pin now that #7548 is in main The pin flipped exactly as designed: it looks for _unsloth_main in the installer it actually ran, and #7548 put it there. Delete the pin and the CLI waiver, and assert the opposite instead. WSL is the only platform whose install shells out to Windows interop mid-script, and interop relays the stdin it inherited, so this job is the one that can catch the pipe being drained again. A truncation here is now a hard failure. --- .../workflows/clean-machine-install-ci.yml | 57 +++++-------------- 1 file changed, 13 insertions(+), 44 deletions(-) diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index 99d31dae13..3ff22f3b23 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -626,12 +626,9 @@ jobs: # `platform wsl` assertion and the CLI check all unable to fail anything. There is no # flake to absorb; if the pinned rootfs ever moves, a red job is the correct signal. # - # The eight green staging runs cited when that continue-on-error came off do not - # transfer to this branch: every one of them ran a tree that already carried #7548 - # (installer-fix-ci and integ-installer-fixes-ci both have `_unsloth_main` in - # install.sh), and #7548 is exactly what makes the script survive being piped. This - # branch does not carry it, so the Install step pins that one difference and - # everything else in this job still gates. + # It is also the only job that can catch a piped install being truncated: WSL is the + # one platform whose install shells out to Windows interop mid-script, and interop + # relays the stdin it inherited. #7548 is in main now, so this gates unconditionally. wsl: name: wsl ubuntu-24.04 runs-on: windows-latest @@ -663,7 +660,6 @@ jobs: wsl -d unsloth-ci -- sh -c 'for t in curl wget git gcc cmake python3 sudo; do printf "%-8s %s\n" "$t" "$(command -v $t || echo ABSENT)"; done' - name: Install inside WSL, piped exactly as documented - id: install shell: pwsh run: | # Only ca-certificates + curl: the advertised one-liner cannot start without a @@ -693,39 +689,20 @@ jobs: $installRc = $LASTEXITCODE Write-Host "installer exit: $installRc" - # KNOWN OUTCOME PIN, retire when #7548 merges. Piping a 218KB script into sh is - # not atomic: dash reads it from the pipe in 8192-byte blocks and runs each - # command as soon as it parses, so any command that inherits fd 0 and reads to - # EOF swallows the rest of the script. install.sh:2007 runs - # _maybe_reroute_strixhalo_to_2404, which on WSL alone shells out to Windows - # interop (powershell.exe for the WMI GPU name, then wsl.exe -l -q), and interop - # relays the stdin it inherited. dash has exactly 11 blocks buffered by then, - # ending at byte 90112, which lands inside "$STUDIO_LOCAL_INSTALL" on line 2082 - # -- hence that line number rather than anything wrong with it. Truncating - # install.sh at 90112 and parsing it reproduces the message verbatim. Every - # other platform runs the same file and stays green because nothing in that - # window touches stdin. #7548 wraps the body in _unsloth_main so sh must parse - # the whole script before running any of it, which is why that marker is what - # retires this pin. Read it out of the distro so it tracks the script that - # actually ran, published or not. - wsl -d unsloth-ci -u root -- sh -c 'grep -q "^_unsloth_main()" /root/install.sh' - if ($LASTEXITCODE -eq 0) { - Write-Host '::error::the installer under test now carries #7548; delete this pin block and let the exit-code check below gate unconditionally' - exit 1 - } if (-not (Test-Path logs/wsl-install.log)) { Write-Host '::error::the WSL install produced no log' exit 1 } - # Exit code AND the shell's own parse error, so any other non-zero exit, and any - # other syntax error, still fails this required job. The line number is not - # matched: it moves with any edit to install.sh while the failure is the same. - $truncated = Select-String -Path logs/wsl-install.log ` - -Pattern 'sh: [0-9]+: Syntax error: Unterminated quoted string' -Quiet - if ($installRc -eq 2 -and $truncated) { - Write-Host '::notice::known outcome: install.sh is not pipe-safe, so WSL interop drained the pipe and sh hit EOF mid-string. Fixed by #7548; nothing in this branch can change it.' - "pinned=true" | Out-File -FilePath $env:GITHUB_OUTPUT -Append - exit 0 + # This job is the one that proves the pipe stays intact. WSL is the only + # platform whose install shells out to Windows interop mid-script + # (_maybe_reroute_strixhalo_to_2404 -> powershell.exe, wsl.exe), and interop + # relays the stdin it inherited, so before #7548 it drank the rest of the + # script and sh died on a half-read line. #7548's _unsloth_main wrapper makes + # sh parse the whole file first; a truncation here means that regressed. + if (Select-String -Path logs/wsl-install.log ` + -Pattern 'Syntax error: Unterminated quoted string' -Quiet) { + Write-Host '::error::the piped install was truncated again; install.sh is no longer parsed in full before it runs' + exit 1 } # Printing the code discarded it, and the next step's CLI check does not # compensate: install.sh links the `unsloth` shim (4174-4182) BEFORE it reports @@ -758,14 +735,6 @@ jobs: Write-Host '::error::installer never reported ''platform wsl''; the WSL branch was not exercised' exit 1 } - # Only the CLI half is waived under the pin: the truncation kills the run - # before anything is installed, so there is nothing to find. The `platform wsl` - # assertion above still gates either way, because the installer prints it - # several hundred lines before the point where the pipe runs dry. - if ('${{ steps.install.outputs.pinned }}' -eq 'true') { - Write-Host '::notice::CLI check waived: the Install step recorded the pinned pre-#7548 pipe truncation' - exit 0 - } # No `|| echo`: substituting a message for the missing CLI made the inner # shell, this step and the job all succeed even when the install produced # nothing usable, which is half of what this step asks. From 22495b3485bfaf13e405c1d8e0af31a104089064 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 04:43:43 +0000 Subject: [PATCH 28/36] Gate the no-elevation Linux install and split off the no-transport case --- .../workflows/clean-machine-install-ci.yml | 211 +++++++++++++++--- 1 file changed, 182 insertions(+), 29 deletions(-) diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index 3ff22f3b23..db7f966bc7 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -41,7 +41,8 @@ # the editable build itself calls `git rev-parse` / # `git archive` through setuptools-scm's file finder, so an # overlay would answer the leg's own question for it. -# linux ubuntu2404-nonroot dies at the elevation gate before a venv exists. +# linux ubuntu2404-nonroot-notransport dies at the elevation gate before a venv +# exists. # wsl only install.sh is copied into the distro; there is no # source tree inside WSL to overlay from. @@ -383,14 +384,35 @@ jobs: runner: ubuntu-24.04-arm experimental: false overlay: true - # No elevation: today this hard-fails at install.sh:856-861. Expected; the - # point is to pin the message and prove it is actionable rather than a bare - # `curl: (56)`. No overlay: it never reaches a venv to overlay into. + # No elevation, but WITH the transport the advertised one-liner needs: not + # root, no sudo anywhere on the image, no toolchain, ca-certificates + curl + # and nothing else. Since #7547 the optional set (cmake, git, + # build-essential, libcurl4-openssl-dev) never escalates, so this must + # install end to end off prebuilt llama.cpp, and until now nothing proved it. + # + # Gating, and overlay: true for the reason fedora41 is: the RELEASED + # install_python_stack.py has no "skip triton kernels when git is missing" + # guard, so without the overlay the run reaches the final step and dies there + # on pure release lag (staging run 30421021166: venv, frontend, torch and + # extras all fine, then `Installing triton kernels (pip) failed`). The + # overlay removes that difference, exactly as it does on fedora. - label: ubuntu2404-nonroot image: ubuntu:24.04 runner: ubuntu-latest - experimental: true + experimental: false + overlay: true + nonroot: true + # No elevation AND no transport. apt is the only way to get curl and reaching + # apt is what needs elevation, so failing is correct; the point is to pin the + # exact message and prove it is actionable rather than a bare `curl: (56)`. + # No overlay: it never reaches a venv to overlay into. + - label: ubuntu2404-nonroot-notransport + image: ubuntu:24.04 + runner: ubuntu-latest + experimental: false overlay: false + nonroot: true + no_transport: true # Non-apt: today this hard-fails at install.sh:2034. Expected; forces the # decision on whether dnf/pacman/zypper get supported. - label: fedora41 @@ -455,22 +477,74 @@ jobs: [ -f ci-source/pyproject.toml ] || { echo "::error::source tarball for ${GITHUB_SHA} unpacked without a pyproject.toml"; ls -la ci-source; exit 1; } echo "overlay source: $(pwd)/ci-source" + # curl is what fetched install.sh above, so the no-transport leg cannot simply + # never install it. Take it away again afterwards: from the installer's point of + # view the machine has no way to download anything, which is the case under test. + - name: Take the transport away again + if: matrix.no_transport + run: | + apt-get remove -y -qq curl >/dev/null + for t in curl wget; do + if command -v "$t" >/dev/null 2>&1; then + echo "::error::$t is still resolvable, so this leg is not the no-transport case" + exit 1 + fi + done + echo "no curl and no wget: the installer has no transport" + - name: Create an unprivileged user - if: matrix.label == 'ubuntu2404-nonroot' + if: matrix.nonroot run: | useradd -m tester # Switching user without a login shell keeps the caller's environment, so the # workflow-wide UNSLOTH_STUDIO_HOME follows tester in, and install.sh - # validates that override in _resolve_studio_destinations (503-559), long - # before the elevation gate (840-861). Without a writable target this leg dies - # on "cannot be created" rather than "cannot elevate". + # validates that override in _resolve_studio_destinations (536-591), long + # before the elevation gate (829-905). Without a writable target these legs + # die on "cannot be created" rather than on anything they are asking about. mkdir -p "$UNSLOTH_STUDIO_HOME" - # No sudo installed and not root -> exercises the "cannot elevate" branch. chown -R tester logs install.sh "$UNSLOTH_STUDIO_HOME" + # The editable overlay writes .egg-info next to the pyproject.toml, so the + # source tree has to belong to tester too or the overlay fails on permissions + # rather than on anything this leg is asking about. + if [ -d ci-source ]; then chown -R tester ci-source; fi + + # Not calling sudo is not the same as not having it: a leg that merely avoided + # the call would pass on an image where elevation was available all along, and + # the whole claim of these two rows is that there is no elevation to be had. + - name: Prove the unprivileged user genuinely cannot elevate + if: matrix.nonroot + run: | + uid="$(su tester -c 'id -u')" + echo "tester uid: $uid" + if [ "$uid" = "0" ]; then + echo "::error::tester resolved to uid 0, so this leg is not unprivileged" + exit 1 + fi + # Absent from disk, not merely off PATH: install.sh probes with `command -v` + # (install.sh:830), so a binary tester could not reach would still be a lie + # about the image. + for p in /usr/bin/sudo /bin/sudo /usr/local/bin/sudo /usr/sbin/sudo /sbin/sudo; do + if [ -e "$p" ]; then + echo "::error::$p exists, so this image is not sudo-free" + exit 1 + fi + done + if su tester -c 'command -v sudo' >/dev/null 2>&1; then + echo "::error::sudo resolves for tester; the no-elevation premise does not hold" + exit 1 + fi + # The capability, not just the tool: the escalation install.sh would attempt + # writes the dpkg database, so an unwritable one is what actually makes + # `apt-get install` impossible for tester. + if su tester -c 'test -w /var/lib/dpkg/status'; then + echo "::error::tester can write the dpkg database, so it is effectively root" + exit 1 + fi + echo "tester is unprivileged, has no sudo on disk, and cannot write dpkg state" - name: Install (root) id: install_root - if: matrix.label != 'ubuntu2404-nonroot' + if: ${{ !matrix.nonroot }} run: | set -o pipefail # Resolved here, not in `env:`, so it tracks the step's real working @@ -487,23 +561,71 @@ jobs: echo "installer exit code: $rc" exit "$rc" - - name: Install (unprivileged, expected to fail cleanly) - if: matrix.label == 'ubuntu2404-nonroot' + # The no-elevation case the whole workflow was missing: everything absent AND no + # way to become root, but the transport the documented one-liner needs is there. + # Gating, and asserted end to end by the same steps the root legs use. + - name: Install (unprivileged, no sudo) + id: install_nonroot + if: ${{ matrix.nonroot && !matrix.no_transport }} + run: | + set -o pipefail + # su without a login shell keeps the environment, so this reaches tester. + if [ -d ci-source ]; then + export UNSLOTH_CI_SOURCE_OVERLAY="$PWD/ci-source" + echo "overlaying this ref's source from $UNSLOTH_CI_SOURCE_OVERLAY" + fi + rc=0 + # Piped, like the root legs: the advertised command, and the shape that turns + # an early exit into curl:(56). + su tester -c 'cat install.sh | sh' 2>&1 | tee logs/install.log || rc=$? + echo "installer exit code: $rc" + exit "$rc" + + - name: Install (unprivileged and no transport, expected to fail cleanly) + id: install_notransport + if: matrix.no_transport + continue-on-error: true run: | set -o pipefail rc=0 - su tester -c 'cat install.sh | sh' > logs/install.log 2>&1 || rc=$? + su tester -c 'cat install.sh | sh' 2>&1 | tee logs/install.log || rc=$? echo "installer exit code: $rc" - tail -40 logs/install.log - # It may legitimately fail; what must NOT happen is an unexplained exit or a - # bare broken-pipe error standing in for a real diagnosis. - if [ "$rc" != "0" ] && ! grep -qiE "sudo is not available|apt-get install|missing:|permission" logs/install.log; then - echo "::error::unprivileged install failed with no actionable message" + exit "$rc" + + # KNOWN OUTCOME PIN. This row is required, and continue-on-error on the step above + # would otherwise tolerate a bootstrap outage or an unrelated early exit exactly + # like the intended diagnostic, so every branch here that is not the pinned + # outcome exits 1. + - name: Assert the no-transport outcome is the elevation gate + if: always() && matrix.no_transport && steps.install_notransport.outcome != 'skipped' + run: | + if [ "${{ steps.install_notransport.outcome }}" = "success" ]; then + echo "::error::the installer completed with no transport and no way to elevate; that outcome is new, so this pin is stale" exit 1 fi + [ -f logs/install.log ] || { echo "::error::the no-transport leg produced no install log"; exit 1; } + tail -40 logs/install.log + # Both, so a failure anywhere else is still red: it must be the transport that + # was missing, and it must be the no-sudo branch of _smart_apt_install + # (install.sh:899-903) that stopped it rather than a prompt, a dpkg lock or a + # network error. + if ! grep -q "missing: curl" logs/install.log; then + echo "::error::the installer never reported the transport as missing; it did not reach the elevation gate" + exit 1 + fi + if ! grep -q "sudo is not available on this system" logs/install.log; then + echo "::error::the installer did not stop at the no-sudo branch of the apt helper; this is a new failure" + exit 1 + fi + # Actionable, not a bare `curl: (56)`: the message has to say what to run. + if ! grep -q "apt-get install -y curl" logs/install.log; then + echo "::error::the elevation gate did not print the command a user should run" + exit 1 + fi + echo "::notice::known outcome: no transport and no way to elevate, refused with an actionable message" - # continue-on-error like the nonroot leg, so without the same check a bootstrap - # outage or an unrelated early exit is tolerated like the intended diagnostic. + # experimental, so without this check a bootstrap outage or an unrelated early + # exit is tolerated like the intended diagnostic. - name: Assert the Fedora outcome is a known one if: always() && matrix.label == 'fedora41' run: | @@ -545,7 +667,7 @@ jobs: # See the macOS job: proves the leg is testing what its matrix row claims. - name: Assert this ref's Python was really put under test - if: matrix.overlay && inputs.installer_source != 'published' && steps.install_root.outcome == 'success' + if: matrix.overlay && inputs.installer_source != 'published' && (steps.install_root.outcome == 'success' || steps.install_nonroot.outcome == 'success') run: | grep -q "CI: overlaying source checkout" logs/install.log || { echo "::error::this leg is marked overlay: true but the installer never overlaid the checkout, so it only tested the released package" @@ -557,24 +679,26 @@ jobs: # satisfies it. Unlike WSL and Windows, these required Linux rows had no check # that the install produced anything runnable. - name: Assert the install is actually usable - if: steps.install_root.outcome == 'success' + if: steps.install_root.outcome == 'success' || steps.install_nonroot.outcome == 'success' run: | VENV="$UNSLOTH_STUDIO_HOME/unsloth_studio" [ -x "$VENV/bin/python" ] || { echo "::error::installer exited 0 but left no managed Python at $VENV/bin/python"; ls -la "$UNSLOTH_STUDIO_HOME" || true; exit 1; } "$VENV/bin/python" -V [ -x "$VENV/bin/unsloth" ] || { echo "::error::installer exited 0 but left no unsloth CLI at $VENV/bin/unsloth"; exit 1; } - - name: Assert the apt-installed toolchain did not compile llama.cpp - if: steps.install_root.outcome == 'success' + - name: Assert llama.cpp came from the prebuilt bundle + if: steps.install_root.outcome == 'success' || steps.install_nonroot.outcome == 'success' run: | - # HONESTY NOTE: these legs START toolchain-free but do not stay that way. - # As root, _smart_apt_install's first `apt-get install` (install.sh:786-788) - # succeeds before the _SMART_APT_OPTIONAL guard (805-809) can suppress + # HONESTY NOTE: the ROOT legs START toolchain-free but do not stay that way. + # As root, _smart_apt_install's first `apt-get install` (install.sh:797-799) + # succeeds before the _SMART_APT_OPTIONAL guard (814-821) can suppress # anything, so `cmake git build-essential libcurl4-openssl-dev` really are # installed mid-run. That is product behaviour on any root Linux install, not # a CI artefact. What must still hold is that nothing USED them: `nobuild` # reads Python builds only, and llama.cpp is the one thing that silently falls - # back to a source compile once a compiler is around. + # back to a source compile once a compiler is around. The unprivileged leg + # never gets that far -- the optional set cannot escalate -- so there the same + # marker proves the prebuilt path won with no compiler on the machine at all. for t in cmake git gcc; do printf '%-6s %s\n' "$t" "$(command -v "$t" 2>/dev/null || echo ABSENT)" done @@ -591,6 +715,35 @@ jobs: echo "llama.cpp came from the prebuilt bundle:" head -c 800 "$META"; echo + # The claim this leg exists to make: a user with no elevation gets a full install + # and the machine is no less clean afterwards. Without it the row would prove only + # that SOME install happened, which the root legs already show. + - name: Assert the unprivileged install elevated nothing + if: steps.install_nonroot.outcome == 'success' + run: | + left="" + for t in sudo cmake git gcc; do + p="$(command -v "$t" 2>/dev/null || echo ABSENT)" + printf '%-6s %s\n' "$t" "$p" + [ "$p" = ABSENT ] || left="$left $t" + done + if [ -n "$left" ]; then + echo "::error::the unprivileged install put system packages on the machine:$left, so something escalated" + exit 1 + fi + # And it took the no-toolchain path knowingly rather than by accident. + if ! grep -q "using prebuilt llama.cpp (missing:" logs/install.log; then + echo "::error::the installer never reported the optional build tools as missing; it did not take the no-toolchain path" + grep -n "deps" logs/install.log | tail -20 || true + exit 1 + fi + # #7547 is what made the optional set stop escalating. If either prompt comes + # back, an unprivileged user is blocked on tools nothing here uses. + if grep -qE "We require sudo elevated permissions|No terminal to confirm on" logs/install.log; then + echo "::error::the installer tried to elevate for the optional build tools; #7547's no-escalation guard has regressed" + exit 1 + fi + - name: Assert no source build if: always() run: | From 404e38baf9feeb0358f9dfc922ed87b27443ed55 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 04:52:27 +0000 Subject: [PATCH 29/36] Assert no source build on the hosted Windows legs and keep winget for the desktop lane --- .github/scripts/assert-nobuild.ps1 | 69 +++++++++++++++++++ .github/scripts/virgin-windows-install.ps1 | 40 ++--------- .../workflows/clean-machine-install-ci.yml | 23 ++++++- .../desktop-app-clean-machine-ci.yml | 37 +++++++++- 4 files changed, 130 insertions(+), 39 deletions(-) create mode 100644 .github/scripts/assert-nobuild.ps1 diff --git a/.github/scripts/assert-nobuild.ps1 b/.github/scripts/assert-nobuild.ps1 new file mode 100644 index 0000000000..64a1ba9813 --- /dev/null +++ b/.github/scripts/assert-nobuild.ps1 @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +# The `nobuild` contract from clean-machine-assert.sh, for Windows. +# +# Why a port and not `shell: bash`: the clean-machine scrub drops every `*\Git\*` +# entry from PATH and from the Machine/User registry copies, and the bash version +# needs sed/grep/tr/sort out of C:\Program Files\Git\usr\bin. This also runs inside +# the servercore container, which has no bash at all. Both Windows lanes call this +# one file so the sdist allowlist cannot drift between them. +# +# Usage: assert-nobuild.ps1 -LogPath logs/install.log (exit 1 = a source build) +[CmdletBinding()] +param([Parameter(Mandatory = $true)][string] $LogPath) + +if (-not (Test-Path -LiteralPath $LogPath)) { + Write-Host "::error::nobuild requested but $LogPath is missing" + exit 1 +} + +# "Built an sdist" is NOT "needed a compiler". Every name here was checked against +# its own sdist: setuptools.build_meta backend, no ext_modules, no .c/.cpp/.pyx/.rs +# file, so its PEP 517 build is a pure-Python copy step. UNSLOTH_ALLOW_SDIST extends +# the list. Kept identical to clean-machine-assert.sh's `_allow`. +$allow = @('openai-whisper', 'argbind', 'randomname', 'antlr4-python3-runtime', 'triton-kernels') +if ($env:UNSLOTH_ALLOW_SDIST) { + $allow += ($env:UNSLOTH_ALLOW_SDIST -split '\s+' | Where-Object { $_ }) +} +# Lowercased and underscore-folded on both sides: a distribution name and the name uv +# prints can disagree on the separator (triton_kernels vs triton-kernels). +$allow = @($allow | ForEach-Object { $_.ToLowerInvariant() -replace '_', '-' }) + +# [char]27, not "`e": the `e escape is PowerShell 6+, and this runs under Windows +# PowerShell 5.1 too, where "`e" degrades to a literal "e" and the strip would eat +# real text instead of ANSI codes. +$esc = [char]27 +$text = (Get-Content -LiteralPath $LogPath -Raw) -replace "$esc\[[0-9;]*[A-Za-z]", '' +$built = @() +foreach ($line in ($text -split "`r?`n")) { + # A local-path build is something the caller pointed at (the CI source overlay), + # never something dependency resolution chose. Index dependencies always print + # `==`, so no signal is lost. + if ($line -imatch 'building [a-z0-9._-]+ @ file://') { continue } + # pip prints `Building wheel for `, uv prints `Building ==` + # (astral-sh/uv#11165). Requiring `==` or ` @ ` after the name keeps this off the + # installer's own lowercase "building frontend..." progress text. + foreach ($m in [regex]::Matches($line, '(?i)building wheel for ([a-z0-9._-]+)|building ([a-z0-9._-]+)(==| @ )')) { + $name = if ($m.Groups[1].Success) { $m.Groups[1].Value } else { $m.Groups[2].Value } + $built += ($name.ToLowerInvariant() -replace '_', '-') + } +} +$built = @($built | Sort-Object -Unique) +$bad = @($built | Where-Object { $allow -notcontains $_ }) + +$rc = 0 +if ($bad.Count -gt 0) { + Write-Host "::error::built from source: $($bad -join ' ') -- these must resolve to wheels on a clean machine" + $rc = 1 +} else { + Write-Host "[assert] OK no non-allowlisted source build (built: $(if ($built) { $built -join ' ' } else { 'none' }))" +} +# Independent of package names: a compiler error means a toolchain was needed. +$compilerErr = Select-String -Path $LogPath -Pattern "error: command '(cc|gcc|clang|cl)' failed", 'clang: error', 'cargo: not found', 'Microsoft Visual C\+\+ 14.0 or greater is required' +if ($compilerErr) { + Write-Host '::error::compiler invocation appears in the install log' + $compilerErr | Select-Object -First 10 | ForEach-Object { Write-Host " $($_.Line)" } + $rc = 1 +} +exit $rc diff --git a/.github/scripts/virgin-windows-install.ps1 b/.github/scripts/virgin-windows-install.ps1 index 2f89759c6c..2d7b7a6289 100644 --- a/.github/scripts/virgin-windows-install.ps1 +++ b/.github/scripts/virgin-windows-install.ps1 @@ -132,40 +132,14 @@ if ($Overlay -and $rc -eq 0) { } Section 'assert: no non-allowlisted source build' -# PowerShell port of .github/scripts/clean-machine-assert.sh's `nobuild`. Same -# contract: pip prints "Building wheel for ", uv prints "Building ==" -# (astral-sh/uv#11165), and a local-path build (`Building @ file://`) is -# something the caller pointed at, never something resolution chose. -if (-not (Test-Path -LiteralPath $LogPath)) { - $failures += "nobuild requested but $LogPath is missing" +# Shared with the hosted Windows legs so the sdist allowlist lives in one place; the +# script prints its own diagnosis, so only the verdict is folded in here. +$nobuild = Join-Path $PSScriptRoot 'assert-nobuild.ps1' +if (-not (Test-Path -LiteralPath $nobuild)) { + $failures += "assert-nobuild.ps1 is missing next to this script, so the no-build contract went unchecked" } else { - $allow = @('openai-whisper', 'argbind', 'randomname', 'antlr4-python3-runtime', 'triton-kernels') - # [char]27, not "`e": the `e escape sequence is PowerShell 6+, and this script runs - # under Windows PowerShell 5.1, where "`e" silently degrades to a literal "e" and - # the strip would eat real text instead of ANSI codes. - $esc = [char]27 - $text = (Get-Content -LiteralPath $LogPath -Raw) -replace "$esc\[[0-9;]*[A-Za-z]", '' - $built = @() - foreach ($line in ($text -split "`r?`n")) { - if ($line -imatch 'building [a-z0-9._-]+ @ file://') { continue } - foreach ($m in [regex]::Matches($line, '(?i)building wheel for ([a-z0-9._-]+)|building ([a-z0-9._-]+)(==| @ )')) { - $name = if ($m.Groups[1].Success) { $m.Groups[1].Value } else { $m.Groups[2].Value } - $built += ($name.ToLowerInvariant() -replace '_', '-') - } - } - $built = $built | Sort-Object -Unique - $bad = @($built | Where-Object { $allow -notcontains $_ }) - if ($bad.Count -gt 0) { - $failures += "built from source: $($bad -join ' ') -- these must resolve to wheels on a clean machine" - } else { - Write-Host "no non-allowlisted source build (built: $(if ($built) { $built -join ' ' } else { 'none' }))" - } - # Independent of package names: a compiler error means a toolchain was needed. - $compilerErr = Select-String -Path $LogPath -Pattern "error: command '(cc|gcc|clang|cl)' failed", 'clang: error', 'cargo: not found', 'Microsoft Visual C\+\+ 14.0 or greater is required' - if ($compilerErr) { - $failures += "compiler invocation appears in the install log" - $compilerErr | Select-Object -First 10 | ForEach-Object { Write-Host " $($_.Line)" } - } + & $nobuild -LogPath $LogPath + if ($LASTEXITCODE -ne 0) { $failures += "a non-allowlisted source build appears in the install log" } } # ── Verdict ─────────────────────────────────────────────────────────────────── diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index db7f966bc7..6cc469c74c 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -66,6 +66,7 @@ on: # The virgin Windows container lane lives in this workflow too. - '.github/scripts/virgin-windows-*.ps1' - '.github/scripts/ensure-docker-daemon.ps1' + - '.github/scripts/assert-nobuild.ps1' - '.github/workflows/clean-machine-install-ci.yml' push: branches: [main] @@ -84,6 +85,7 @@ on: # The virgin Windows container lane lives in this workflow too. - '.github/scripts/virgin-windows-*.ps1' - '.github/scripts/ensure-docker-daemon.ps1' + - '.github/scripts/assert-nobuild.ps1' - '.github/workflows/clean-machine-install-ci.yml' workflow_dispatch: inputs: @@ -1021,10 +1023,11 @@ jobs: $newPath = ($kept -join ';') "PATH=$newPath" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 # install.ps1's Refresh-SessionPath (318-337, called at 1246/1278/1295/1360/ - # 1369/2797) rebuilds $env:Path from the Machine and User registry values, so a - # process-only scrub lasts until the first bootstrap refresh, after which + # 1369/2797) merges the Machine and User registry PATHs back into $env:Path, so + # a process-only scrub lasts until the first bootstrap refresh, after which # Git/CMake/VS/LLVM are back and the rest of the install is not clean. The - # runner is ephemeral, so rewrite the registry copies too. Expand first: + # runner is ephemeral, so rewrite the registry copies too. It is a merge, not a + # replace, so the shim above keeps resolving. Expand first: # SetEnvironmentVariable rewrites REG_EXPAND_SZ as REG_SZ # (dotnet/runtime#1442). foreach ($scope in 'Machine','User') { @@ -1295,6 +1298,20 @@ jobs: } Write-Host 'no CMake and no VS Build Tools install; the prebuilt contract held' + - name: Assert no source build + if: always() && steps.install.outcome != 'skipped' + shell: pwsh + run: | + # The step above only catches a NEW CMake or VS Build Tools install. The + # image's Visual Studio survives a PATH scrub: setup.ps1's Find-VsBuildTools + # (763-800) reaches it through vswhere and a Program Files scan, and the + # visible leg logs `vs Visual Studio 18 2026 (vswhere)` on the same machine + # whose pre-flight printed `cl ABSENT`. So a dependency that lost its Windows + # wheel would compile against that MSVC and the leg would stay green, while + # macOS and Linux caught it. uv really does build sdists here (openai-whisper, + # antlr4-python3-runtime, randomname, argbind), so this is the live path. + & "$env:GITHUB_WORKSPACE/.github/scripts/assert-nobuild.ps1" -LogPath logs/install.log + - name: Assert torch loads, and record what that does and does not prove if: steps.install.outcome == 'success' shell: pwsh diff --git a/.github/workflows/desktop-app-clean-machine-ci.yml b/.github/workflows/desktop-app-clean-machine-ci.yml index ee332dfa72..acb695f9bf 100644 --- a/.github/workflows/desktop-app-clean-machine-ci.yml +++ b/.github/workflows/desktop-app-clean-machine-ci.yml @@ -534,11 +534,29 @@ jobs: run: | $drop = @('hostedtoolcache\windows\Python', 'WindowsApps', '\Git\', 'CMake', 'Microsoft Visual Studio', 'BuildTools', 'LLVM', 'MSYS', 'mingw') + # winget is an app-execution alias under ...\Local\Microsoft\WindowsApps, so + # the WindowsApps fragment -- there to take the Store's python.exe alias away + # -- drops the OS package manager with it. winget is not developer tooling; + # every consumer Windows machine this bundle ships to has it, and the bundled + # install.ps1 reaches for it for the git that studio/setup.ps1:1657-1669 still + # gates on unconditionally. Without it this lane only re-runs the no-winget + # fallback that clean-machine-install-ci.yml already covers and pins on its + # winget=masked row, and it does so as a hard failure. Resolve winget before + # the scrub and hand it back through a shim, exactly as that workflow does. + $wingetCmd = Get-Command winget -ErrorAction SilentlyContinue + if (-not $wingetCmd) { + Write-Host '::error::winget was not on PATH before the strip; this image ships it and the bundled installer needs it' + exit 1 + } + $shim = Join-Path $env:RUNNER_TEMP 'winget-shim' + New-Item -ItemType Directory -Force -Path $shim | Out-Null + Set-Content -LiteralPath (Join-Path $shim 'winget.cmd') -Encoding ascii ` + -Value "@`"$($wingetCmd.Source)`" %*" $scrub = { param($entries) ,@($entries | Where-Object { $p = $_; $p -and -not ($drop | Where-Object { $p -like "*$_*" }) }) } - "PATH=$((& $scrub ($env:PATH -split ';')) -join ';')" | + "PATH=$shim;$((& $scrub ($env:PATH -split ';')) -join ';')" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 # Take the toolcache Python off disk, not just off PATH: py.exe lives in # C:\Windows (which must stay) and uv does its own interpreter discovery, @@ -551,9 +569,11 @@ jobs: } } # The bundled install.ps1 this job runs calls Refresh-SessionPath (318-337), - # which rebuilds $env:Path from the Machine and User registry values, so a + # which merges the Machine and User registry PATHs back into $env:Path, so a # process-only scrub lasts until the first refresh and Git/CMake/VS/LLVM come - # back. The runner is ephemeral, so rewrite the registry copies too. Expand + # back from the registry. The runner is ephemeral, so rewrite the registry + # copies too. (A merge keeps what the process already had, which is why the + # winget shim above survives.) Expand # first: SetEnvironmentVariable rewrites REG_EXPAND_SZ as REG_SZ # (dotnet/runtime#1442). foreach ($scope in 'Machine','User') { @@ -604,6 +624,17 @@ jobs: # machine that is in fact clean. $global:LASTEXITCODE = 0 } + # The shim is the only reason winget resolves after the WindowsApps drop. It + # survives the installer's own refreshes because Refresh-SessionPath + # (install.ps1:318-337) and setup.ps1's Refresh-Environment MERGE the current + # $env:Path back in rather than replace it -- but assert it, or this lane + # silently degrades into the no-winget leg the installer workflow already pins. + $winget = Get-Command winget -ErrorAction SilentlyContinue + Write-Host ("winget {0}" -f $(if ($winget) { $winget.Source } else { 'ABSENT' })) + if (-not $winget) { + Write-Host '::error::winget did not survive the strip; the bundled installer would take the no-winget fallback instead of the consumer path' + exit 1 + } if ($leaked) { Write-Host "::error::developer tooling survived the strip: $($leaked -join '; ')" exit 1 From c6168a31fab714929193aa2e0d9197b1fd07d01e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 04:54:12 +0000 Subject: [PATCH 30/36] Retry the container root CA seeding instead of failing on one Windows Update timeout --- .github/workflows/clean-machine-install-ci.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index 6cc469c74c..08af140b58 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -1509,10 +1509,20 @@ jobs: - name: Seed the container's trusted root CA store shell: pwsh run: | - docker exec virgin cmd /c "certutil -generateSSTFromWU C:\roots.sst && certutil -addstore -f Root C:\roots.sst" ` - *>&1 | Select-Object -Last 15 + # -generateSSTFromWU pulls each root from ctldl.windowsupdate.com, and that + # host times out often enough to be the leg's main flake (staging run + # 30423072537 died on WinHttp 12002 while the sibling row seeded fine). + # Retry, but never tolerate a total failure: without the roots, Node's + # urllib download later fails with CERTIFICATE_VERIFY_FAILED. + for ($i = 1; $i -le 3; $i++) { + docker exec virgin cmd /c "certutil -generateSSTFromWU C:\roots.sst && certutil -addstore -f Root C:\roots.sst" ` + *>&1 | Select-Object -Last 15 + if ($LASTEXITCODE -eq 0) { break } + Write-Host "::warning::root CA seeding attempt $i failed; retrying" + Start-Sleep -Seconds 15 + } if ($LASTEXITCODE -ne 0) { - Write-Host '::error::could not seed the container root CA store; Python-side HTTPS will fail' + Write-Host '::error::could not seed the container root CA store after 3 attempts; Python-side HTTPS will fail' exit 1 } From 31eb3aed45e702c9c0da2f613881b52ff79af78f Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 04:54:52 +0000 Subject: [PATCH 31/36] Run the clean-machine workflow for the prebuilt installer helpers it overlays --- .github/workflows/clean-machine-install-ci.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index 08af140b58..735fc4e20b 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -56,6 +56,13 @@ on: - 'studio/setup.sh' - 'studio/setup.ps1' - 'studio/install_python_stack.py' + # setup.sh (727) and setup.ps1 (2343, 3630, 3916) call these directly, and the + # overlay makes them THIS ref's code, so they decide whether a clean machine gets + # a native prebuilt or falls back to a toolchain-dependent path. Left off the + # list, a change to one of them skipped the only workflow that can see it. + - 'studio/install_*_prebuilt.py' + - 'studio/prebuilt_core.py' + - 'studio/node_prebuilt_pins.json' # The overlay exists so a constraints or requirements change is actually # exercised here (see the header). Without these paths the one workflow that # resolves them with no compiler and no cached wheels never runs for the PR that @@ -63,6 +70,7 @@ on: # preinstalled Python and full developer tooling. - 'studio/backend/requirements/**' - '.github/scripts/clean-machine-*.sh' + - '.github/scripts/assert-llama-loads.sh' # The virgin Windows container lane lives in this workflow too. - '.github/scripts/virgin-windows-*.ps1' - '.github/scripts/ensure-docker-daemon.ps1' @@ -80,8 +88,16 @@ on: - 'studio/setup.sh' - 'studio/setup.ps1' - 'studio/install_python_stack.py' + # setup.sh (727) and setup.ps1 (2343, 3630, 3916) call these directly, and the + # overlay makes them THIS ref's code, so they decide whether a clean machine gets + # a native prebuilt or falls back to a toolchain-dependent path. Left off the + # list, a change to one of them skipped the only workflow that can see it. + - 'studio/install_*_prebuilt.py' + - 'studio/prebuilt_core.py' + - 'studio/node_prebuilt_pins.json' - 'studio/backend/requirements/**' - '.github/scripts/clean-machine-*.sh' + - '.github/scripts/assert-llama-loads.sh' # The virgin Windows container lane lives in this workflow too. - '.github/scripts/virgin-windows-*.ps1' - '.github/scripts/ensure-docker-daemon.ps1' From b992e72750baff0195d0dce0a33830f379574121 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 04:58:37 +0000 Subject: [PATCH 32/36] Narrow the container pin to its own gates and scan uv and the venv interpreter for arch --- .github/scripts/clean-machine-assert.sh | 30 +++++++++++++++-- .../workflows/clean-machine-install-ci.yml | 32 +++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/.github/scripts/clean-machine-assert.sh b/.github/scripts/clean-machine-assert.sh index 94563bc3fc..4bf1f5957f 100755 --- a/.github/scripts/clean-machine-assert.sh +++ b/.github/scripts/clean-machine-assert.sh @@ -172,11 +172,30 @@ for check in "$@"; do if [ ! -d "$root" ]; then fail "macho requested but $root does not exist" else - n=0 nexe=0 bad_arch="" unsigned="" broken="" + # SCOPE, part 2: the two payloads the install RUNS ON live outside $root. + # `uv venv` links /bin/python at its base interpreter rather than + # copying it, and the find below has no -L, so the interpreter that executed + # every install step is invisible to it; the uv that fetched it lands in + # $HOME/.local/bin. Both are exactly what Rosetta 2 hides -- an x86_64 uv or + # managed CPython runs green here and dies on the factory-fresh Mac this job + # stands in for. + _macho_targets() { + find "$root" -type f \( -perm -u+x -o -name '*.dylib' -o -name '*.so' -o -name '*.node' \) 2>/dev/null + # -L follows the interpreter symlink; -maxdepth keeps this a bin/ lookup and + # not a second walk of site-packages through the venv's lib64 link. Depth 4 + # covers /unsloth_studio, the .venv_t5_* sidecars and the tauri + # layout's /studio/unsloth_studio. + find -L "$root" -maxdepth 4 -type f -path '*/bin/python' 2>/dev/null + for _uv in "$HOME/.local/bin/uv" "$(command -v uv 2>/dev/null || true)"; do + [ -n "$_uv" ] && [ -f "$_uv" ] && printf '%s\n' "$_uv" + done + } + n=0 nexe=0 nout=0 bad_arch="" unsigned="" broken="" while IFS= read -r f; do desc="$(file -b "$f" 2>/dev/null || true)" case "$desc" in *Mach-O*) ;; *) continue ;; esac n=$((n + 1)) + case "$f" in "$root"/*) ;; *) nout=$((nout + 1)) ;; esac # Substring, not equality: a universal binary lists every slice it carries, # and one that includes the host arch is fine. case "$desc" in @@ -227,11 +246,16 @@ for check in "$@"; do esac fi fi - done < <(find "$root" -type f \( -perm -u+x -o -name '*.dylib' -o -name '*.so' -o -name '*.node' \) 2>/dev/null) + done < <(_macho_targets | sort -u) if [ "$n" = "0" ]; then # An empty scan reads exactly like a clean one, so the check would pass on a # wrong root and prove nothing. fail "no Mach-O found under $root; the arch/signature assertion proved nothing" + elif [ "$nout" = "0" ]; then + # Same rule for the roots added above: install.sh always bootstraps uv into + # $HOME/.local/bin, so zero hits outside $root means the extra scan matched + # nothing and uv's architecture went unproven. + fail "no Mach-O outside $root was scanned, so uv and the venv's base interpreter escaped the check" elif [ -n "$bad_arch" ]; then fail "Mach-O is not $want, so it runs here only under Rosetta 2, which a fresh Mac does not have:$bad_arch" elif [ -n "$unsigned" ]; then @@ -239,7 +263,7 @@ for check in "$@"; do elif [ -n "$broken" ]; then fail "Mach-O main executable carries a signature that does not verify:$broken" else - ok "$n Mach-O files under $root are $want$([ "$want" = arm64 ] && echo "; all $nexe main executable(s) signed")" + ok "$n Mach-O files under $root, plus uv and the venv's base interpreter, are $want$([ "$want" = arm64 ] && echo "; all $nexe main executable(s) signed")" fi fi ;; diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index 735fc4e20b..448b19e244 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -1588,6 +1588,38 @@ jobs: Write-Host '::error::the container install failed at neither the winget-only git gate nor the missing VC++ runtime; this is a new failure' exit 1 } + # `-or` on its own is too generous. virgin-windows-install.ps1:97 runs the + # torch assertion whenever the venv interpreter exists, whatever the installer + # did, and this image has no VC++ runtime, so ANY failure after venv creation + # -- a Node download, a setup step, a bad prebuilt -- arrives here carrying the + # $vcGate text and was accepted as the pinned outcome. Enumerate what the + # harness actually recorded instead: it prints one `::error::` per + # entry of its $failures list (that script:151), and every one has to be a + # pinned gate. Anchored, because it also dumps the install log tail indented + # two spaces and those copies must not count. + $recorded = @(Get-Content logs/install-outer.log | + ForEach-Object { if ($_ -match '^::error::(.+)$') { $Matches[1].Trim() } }) + Write-Host "recorded failures: $($recorded.Count)" + $recorded | ForEach-Object { Write-Host " $_" } + if ($recorded.Count -eq 0) { + Write-Host '::error::the container install failed but recorded no ::error:: line, so nothing identifies which gate stopped it' + exit 1 + } + # The git gate makes install.ps1 exit non-zero; the missing runtime makes the + # torch assert fail. Nothing else is pinned. + $pinned = @('^installer exited \d+$', '^torch failed to import from the managed Python') + $unexpected = @($recorded | Where-Object { $r = $_; -not ($pinned | Where-Object { $r -match $_ }) }) + if ($unexpected.Count -gt 0) { + Write-Host "::error::the container install recorded a failure outside the pinned gates: $($unexpected -join '; '); this is a new failure" + exit 1 + } + # And a non-zero exit has to BE the git gate: without this a post-venv failure + # that also exits 1 is indistinguishable from the pinned one. + if (($recorded | Where-Object { $_ -like 'installer exited*' }) -and + -not ($gitGate -and ($log -match 'unsloth studio setup failed \(exit code 1\)'))) { + Write-Host '::error::the installer exited non-zero somewhere other than the winget-only git gate in studio/setup.ps1; this is a new failure' + exit 1 + } if ($gitGate) { Write-Host '::notice::known outcome: winget-only git gate (studio/setup.ps1), fixed by #7549' } if ($vcGate) { Write-Host '::notice::known outcome: no VC++ runtime and Ensure-VCRedist is winget-only, fixed by #7549' } From 906ef42eada4d858881b99be60c2c474126ff713 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 05:04:38 +0000 Subject: [PATCH 33/36] Tighten the clean-machine comments --- .github/scripts/assert-nobuild.ps1 | 28 +- .github/scripts/clean-machine-assert.sh | 85 +++--- .../workflows/clean-machine-install-ci.yml | 257 ++++++++---------- .../desktop-app-clean-machine-ci.yml | 71 +++-- 4 files changed, 187 insertions(+), 254 deletions(-) diff --git a/.github/scripts/assert-nobuild.ps1 b/.github/scripts/assert-nobuild.ps1 index 64a1ba9813..c9937bfd95 100644 --- a/.github/scripts/assert-nobuild.ps1 +++ b/.github/scripts/assert-nobuild.ps1 @@ -3,11 +3,10 @@ # The `nobuild` contract from clean-machine-assert.sh, for Windows. # -# Why a port and not `shell: bash`: the clean-machine scrub drops every `*\Git\*` -# entry from PATH and from the Machine/User registry copies, and the bash version -# needs sed/grep/tr/sort out of C:\Program Files\Git\usr\bin. This also runs inside -# the servercore container, which has no bash at all. Both Windows lanes call this -# one file so the sdist allowlist cannot drift between them. +# A port and not `shell: bash`: the clean-machine scrub drops every `*\Git\*` PATH +# entry, and the bash version needs sed/grep/tr/sort out of Git's usr/bin. It also runs +# inside the servercore container, which has no bash at all. Both Windows lanes call +# this one file so the sdist allowlist cannot drift. # # Usage: assert-nobuild.ps1 -LogPath logs/install.log (exit 1 = a source build) [CmdletBinding()] @@ -18,10 +17,9 @@ if (-not (Test-Path -LiteralPath $LogPath)) { exit 1 } -# "Built an sdist" is NOT "needed a compiler". Every name here was checked against -# its own sdist: setuptools.build_meta backend, no ext_modules, no .c/.cpp/.pyx/.rs -# file, so its PEP 517 build is a pure-Python copy step. UNSLOTH_ALLOW_SDIST extends -# the list. Kept identical to clean-machine-assert.sh's `_allow`. +# "Built an sdist" is NOT "needed a compiler": every name here has a +# setuptools.build_meta backend, no ext_modules and no .c/.cpp/.pyx/.rs file, so its +# PEP 517 build is a pure-Python copy step. Identical to clean-machine-assert.sh. $allow = @('openai-whisper', 'argbind', 'randomname', 'antlr4-python3-runtime', 'triton-kernels') if ($env:UNSLOTH_ALLOW_SDIST) { $allow += ($env:UNSLOTH_ALLOW_SDIST -split '\s+' | Where-Object { $_ }) @@ -30,20 +28,18 @@ if ($env:UNSLOTH_ALLOW_SDIST) { # prints can disagree on the separator (triton_kernels vs triton-kernels). $allow = @($allow | ForEach-Object { $_.ToLowerInvariant() -replace '_', '-' }) -# [char]27, not "`e": the `e escape is PowerShell 6+, and this runs under Windows -# PowerShell 5.1 too, where "`e" degrades to a literal "e" and the strip would eat -# real text instead of ANSI codes. +# [char]27, not "`e": that escape is PowerShell 6+, and under Windows PowerShell 5.1 +# it degrades to a literal "e" and the strip eats real text instead of ANSI codes. $esc = [char]27 $text = (Get-Content -LiteralPath $LogPath -Raw) -replace "$esc\[[0-9;]*[A-Za-z]", '' $built = @() foreach ($line in ($text -split "`r?`n")) { # A local-path build is something the caller pointed at (the CI source overlay), - # never something dependency resolution chose. Index dependencies always print - # `==`, so no signal is lost. + # never something resolution chose; index dependencies always print `==`. if ($line -imatch 'building [a-z0-9._-]+ @ file://') { continue } # pip prints `Building wheel for `, uv prints `Building ==` - # (astral-sh/uv#11165). Requiring `==` or ` @ ` after the name keeps this off the - # installer's own lowercase "building frontend..." progress text. + # (astral-sh/uv#11165); the `==` or ` @ ` requirement keeps this off the + # installer's own lowercase "building frontend..." text. foreach ($m in [regex]::Matches($line, '(?i)building wheel for ([a-z0-9._-]+)|building ([a-z0-9._-]+)(==| @ )')) { $name = if ($m.Groups[1].Success) { $m.Groups[1].Value } else { $m.Groups[2].Value } $built += ($name.ToLowerInvariant() -replace '_', '-') diff --git a/.github/scripts/clean-machine-assert.sh b/.github/scripts/clean-machine-assert.sh index 4bf1f5957f..37b9604f7c 100755 --- a/.github/scripts/clean-machine-assert.sh +++ b/.github/scripts/clean-machine-assert.sh @@ -30,9 +30,8 @@ for check in "$@"; do case "$check" in absent) - # Deliberately NOT `command -v`: on a virgin Mac /usr/bin/{git,cc} EXIST as CLT - # stubs, so `command -v` succeeds and only RUNNING them fails ("invalid active - # developer path"). The honest invariant is: must not WORK. + # NOT `command -v`: on a virgin Mac /usr/bin/{git,cc} EXIST as CLT stubs, so it + # succeeds and only RUNNING them fails. The invariant is: must not WORK. if xcode-select -p >/dev/null 2>&1; then fail "xcode-select -p still resolves to $(xcode-select -p 2>/dev/null); not a clean Mac" else @@ -74,10 +73,9 @@ for check in "$@"; do while IFS=$'\t' read -r tool rest; do [ -n "$tool" ] || continue case " $allow " in *" $tool "*) continue ;; esac - # `xcode-select -p` only ASKS whether a toolchain is selected; the installer - # has to ask, and the fix is that it carries on without one. Counting the - # question as USE would fail the very leg proving the toolchain went - # untouched. `--install`, which pops the CLT installer, stays a hit. + # `xcode-select -p` only ASKS whether a toolchain is selected, and the fix + # is that the installer carries on without one, so the question is not USE. + # `--install`, which pops the CLT installer, stays a hit. if [ "$tool" = "xcode-select" ]; then case "$rest" in -p|--print-path|-v|--version|"") continue ;; @@ -95,9 +93,8 @@ for check in "$@"; do ;; nobuild) - # "Built an sdist" is NOT "needed a compiler", so the contract is "nothing - # needing a COMPILER was built". Every name below was checked against its - # actual sdist: setuptools.build_meta backend, no ext_modules, not one + # "Built an sdist" is NOT "needed a compiler". Every name below was checked + # against its own sdist: setuptools.build_meta backend, no ext_modules, no # .c/.cpp/.pyx/.rs file, so its PEP 517 build is a pure-Python copy step. # openai-whisper, argbind, randomname -- no version ever ships a wheel # antlr4-python3-runtime==4.9.3 -- pinned below the 4.13.2 wheel @@ -110,23 +107,19 @@ for check in "$@"; do # UNSLOTH_ALLOW_SDIST extends the allowlist. # # Lowercased and underscore-folded on both sides: a distribution name and the - # name uv prints can disagree on the separator (requirement triton_kernels vs - # build line triton-kernels), and a one-spelling allowlist silently misses. + # name uv prints can disagree on the separator (triton_kernels vs triton-kernels). _allow="$(printf '%s' "openai-whisper argbind randomname antlr4-python3-runtime triton-kernels ${UNSLOTH_ALLOW_SDIST:-}" | tr 'A-Z_' 'a-z-')" if [ ! -f "$LOG" ]; then fail "nobuild requested but $LOG is missing" else - # uv does NOT use pip's phrasing: it prints `Building ==` to - # stderr (astral-sh/uv#11165), so the pip-only pattern left _built empty on - # every uv source build. Match both. Requiring `==` or ` @ ` after the name - # keeps this off the installer's own lowercase "building frontend..." - # progress text. Strip ANSI first so a coloured run (FORCE_COLOR) parses. - # - # `Building @ file://...` is dropped first: a local-path build is - # something the caller pointed at (install.sh --local, or the - # UNSLOTH_CI_SOURCE_OVERLAY editable overlay), never a dependency resolution - # chose. Index dependencies always print `==`, so no signal is - # lost: a genuine sdist from PyPI is still caught, including one named unsloth. + # uv prints `Building ==`, pip prints `Building wheel for + # ` (astral-sh/uv#11165), so match both; the `==` or ` @ ` requirement + # keeps this off the installer's own lowercase "building frontend..." text, and + # ANSI is stripped first so a coloured run (FORCE_COLOR) parses. + # `Building @ file://...` is dropped: a local-path build is something the + # caller pointed at (--local, or the editable overlay), never something + # resolution chose. Index dependencies always print `==`, so a + # genuine PyPI sdist is still caught, including one named unsloth. _esc=$(printf '\033') _built="$(sed -E "s/${_esc}\[[0-9;]*[A-Za-z]//g" "$LOG" 2>/dev/null \ | grep -viE "building [a-z0-9._-]+ @ file://" \ @@ -157,10 +150,9 @@ for check in "$@"; do macho) # The one thing masking cannot reproduce: Rosetta 2 is preinstalled on hosted # runners and absent from a factory-fresh Mac, so an x86_64-only payload runs - # green here and dies with "bad CPU type in executable" for the user. Assert the - # architecture rather than hope the runner lacks Rosetta. - # `lipo` is an xcrun shim and is gone after masking, so read `file -b`, exactly - # as the desktop lane does. Keyed off `uname -m`, since macos-15-intel is x86_64. + # green here and dies with "bad CPU type in executable" for the user. `lipo` is an + # xcrun shim and gone after masking, so read `file -b`, keyed off `uname -m` + # (macos-15-intel is x86_64). # # SCOPE: all of $MACHO_ROOT, including the .venv_t5_510/_530/_550 sidecars. # Those are payload, not scratch: setup.sh:579-581 creates them during a @@ -203,20 +195,17 @@ for check in "$@"; do *) bad_arch="$bad_arch $f [$desc]" ;; esac - # Signature: MAIN EXECUTABLES ONLY. Asserting it for every Mach-O failed - # the mask/pipe leg on 29 ordinary PyPI extension modules (lxml, - # charset_normalizer, cygrpc, fontTools, ...) plus libportaudio.dylib. - # The premise was wrong: those are MH_BUNDLE/MH_DYLIB images dlopen'd - # into a process without library validation and ship unsigned, and the - # run that flagged them had already imported them with the installer - # exiting 0. Enforcement lands on main executables and gatekept .app - # bundles, so that is all this asserts. + # Signature: MAIN EXECUTABLES ONLY. Asserting it for every Mach-O failed the + # mask/pipe leg on 29 ordinary PyPI extension modules plus libportaudio.dylib: + # those are MH_BUNDLE/MH_DYLIB images dlopen'd without library validation and + # ship unsigned, and that run had already imported them with the installer + # exiting 0. macOS enforces on main executables and gatekept .app bundles. # - # Key off the filetype `file` reports, not the path or extension: a .so - # may be a bundle or a dylib, and an executable may have no extension. - # The library veto is second so a mixed-type fat file counts as a - # library. Substring tests are order-independent: Apple's `file` prints - # `Mach-O 64-bit executable arm64`, GNU's `Mach-O 64-bit arm64 executable`. + # Key off the filetype `file` reports, not the path: a .so may be a bundle or a + # dylib, and an executable may have no extension. The library veto is second so + # a mixed-type fat file counts as a library. Substring tests are + # order-independent (Apple prints `... executable arm64`, GNU `... arm64 + # executable`). _is_exe=0 case "$desc" in *executable*) _is_exe=1 ;; esac case "$desc" in *"shared library"*|*bundle*) _is_exe=0 ;; esac @@ -229,16 +218,14 @@ for check in "$@"; do # ("Killed: 9"), while x86_64 execs it happily, so an unsigned x86_64 # payload is not the same defect. if [ "$want" = "arm64" ] && [ "$_is_exe" = 1 ]; then - # Ad-hoc counts as signed: arm64 linkers apply an ad-hoc seal by - # default, so the test is "has a seal that verifies", not "has an - # identity". `spctl`/`--strict` would demand an authority and reject - # ad-hoc, so neither is used. + # Ad-hoc counts as signed: arm64 linkers seal ad-hoc by default, so the + # test is "has a seal that verifies", not "has an identity". `spctl` and + # `--strict` would demand an authority and reject ad-hoc. if ! codesign -v "$f" >/dev/null 2>&1; then - # Nothing to verify and a seal that does not match mean different - # things. Captured, not piped into grep: `codesign -dvv` exits - # non-zero on an unsigned file, and under the `pipefail` above that - # status is what `codesign ... | grep -q` returns even on a match, - # reporting every unsigned binary as a broken signature. + # Nothing to verify and a seal that does not match mean different things. + # Captured, not piped into grep: `codesign -dvv` exits non-zero on an + # unsigned file, and under `pipefail` that status is what the pipeline + # returns even on a match. _sig="$(codesign -dvv "$f" 2>&1 || true)" case "$_sig" in *"not signed at all"*) unsigned="$unsigned $f" ;; diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index 448b19e244..5476e1dacc 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -15,24 +15,18 @@ # Linux is the exception: containers are genuinely clean. # # ── What each leg actually puts under test ──────────────────────────────────── -# install.sh / install.ps1 come from this ref, but they install unsloth FROM PyPI, -# the consumer path, which has to stay that way. Everything Python-side is therefore -# read out of the RELEASED wheel: studio/setup.sh, setup.ps1, -# install_python_stack.py, and every requirements and constraints file those reach -# through Path(__file__). Left alone this workflow validates only the two shell -# installers, and a branch changing constraints.txt or setup.ps1 gets a green run -# that proves nothing about the change; the `Assert the Fedora outcome is a known -# one` step below was already working around exactly that. +# install.sh / install.ps1 come from this ref, but they install unsloth FROM PyPI -- +# the consumer path, which has to stay that way -- so everything Python-side would +# come out of the RELEASED wheel (setup.sh, setup.ps1, install_python_stack.py and +# every requirements/constraints file they reach via Path(__file__)), and a branch +# changing any of them would get a green run proving nothing about the change. # -# So `overlay: true` legs re-point the venv at this ref before studio setup runs, via -# UNSLOTH_CI_SOURCE_OVERLAY (install.sh / install.ps1, just above their "Run studio -# setup" section): a `--no-deps` editable install of the checkout. `import studio` -# then resolves to the working tree, so the existing setup-script lookup finds this -# ref's setup.sh / setup.ps1 and install_python_stack reads this ref's constraints. -# Deliberately NOT `install.sh --local`: that also installs -# `unsloth-zoo @ git+https://...`, which genuinely needs git, and git absence is the -# whole point of the masked legs. The overlay resolves nothing and clones nothing, so -# it survives git, cmake and the compilers all being gone. +# `overlay: true` legs therefore re-point the venv at this ref before studio setup +# runs, via UNSLOTH_CI_SOURCE_OVERLAY: a `--no-deps` editable install of the +# checkout, so `import studio` resolves to the working tree and the setup-script +# lookup finds this ref's setup.sh / setup.ps1. Deliberately NOT `install.sh +# --local`, which also pulls `unsloth-zoo @ git+https://...` and so needs the git +# these legs remove; an editable overlay resolves and clones nothing. # # Legs left on `overlay: false`, and why: # mac */mask/pipe the `curl | sh` shape a user runs. Kept end-to-end on the @@ -56,17 +50,14 @@ on: - 'studio/setup.sh' - 'studio/setup.ps1' - 'studio/install_python_stack.py' - # setup.sh (727) and setup.ps1 (2343, 3630, 3916) call these directly, and the + # setup.sh (727) and setup.ps1 (2343, 3630, 3916) call these directly and the # overlay makes them THIS ref's code, so they decide whether a clean machine gets - # a native prebuilt or falls back to a toolchain-dependent path. Left off the - # list, a change to one of them skipped the only workflow that can see it. + # a native prebuilt or a toolchain-dependent fallback. - 'studio/install_*_prebuilt.py' - 'studio/prebuilt_core.py' - 'studio/node_prebuilt_pins.json' - # The overlay exists so a constraints or requirements change is actually - # exercised here (see the header). Without these paths the one workflow that - # resolves them with no compiler and no cached wheels never runs for the PR that - # changes them, and the update-smoke jobs cannot stand in: they start from a + # The overlay exists so a constraints or requirements change is exercised here + # (see the header). The update-smoke jobs cannot stand in: they start from a # preinstalled Python and full developer tooling. - 'studio/backend/requirements/**' - '.github/scripts/clean-machine-*.sh' @@ -78,10 +69,8 @@ on: - '.github/workflows/clean-machine-install-ci.yml' push: branches: [main] - # Same list as the PR filter. A direct push to main touching setup.sh, setup.ps1, - # install_python_stack.py, a requirements file or a clean-machine helper skipped - # this workflow entirely, so the post-merge backstop for exactly those files never - # happened. + # Same list as the PR filter: without it a direct push to main touching any of + # these skipped the workflow, so the post-merge backstop never happened. paths: - 'install.sh' - 'install.ps1' @@ -237,19 +226,19 @@ jobs: set -o pipefail rc=0 FLAGS="${{ matrix.flags }}" - # A consumer has no CI=true, no GITHUB_*, no RUNNER_*: an installer branching - # on any of them is a hidden dependency nobody outside CI exercises. Scoped to - # the installer's own process, so $GITHUB_OUTPUT below still resolves. `case` + # A consumer has no CI=true, no GITHUB_*, no RUNNER_*: branching on any of + # them is a hidden dependency nobody outside CI exercises. Scoped to the + # installer's own process, so $GITHUB_OUTPUT below still resolves. `case` # rather than `sed`, whose BRE has no \| alternation on macOS. CLEAN_ENV="" for v in $(env | cut -d= -f1); do case "$v" in CI|GITHUB_*|RUNNER_*) CLEAN_ENV="$CLEAN_ENV -u $v" ;; esac done echo "unset for the installer:$CLEAN_ENV" - # A `published` dispatch asks whether unsloth.ai's script works. Only `pipe` - # honoured it, so six of the eight macOS rows ran the checked-out script and - # were still reported as published coverage. Resolve it once, here, for every - # delivery. Empty on pull_request/push, so automatic runs stay on this ref. + # A `published` dispatch asks whether unsloth.ai's script works, and only + # `pipe` honoured it, so six of eight macOS rows ran the checked-out script + # under the published label. Resolved once here for every delivery. Empty on + # pull_request/push, so automatic runs stay on this ref. SCRIPT=install.sh if [ "${{ inputs.installer_source }}" = "published" ]; then curl -fsSL https://unsloth.ai/install.sh -o published-install.sh @@ -303,9 +292,8 @@ jobs: fi exit "$rc" - # Without this the gap returns silently: install.sh ignores an unset - # UNSLOTH_CI_SOURCE_OVERLAY, so a typo in the matrix or the expression puts every - # leg back on the released wheel with nothing in the run saying so. + # install.sh ignores an unset UNSLOTH_CI_SOURCE_OVERLAY, so a typo in the matrix + # or the expression silently puts every leg back on the released wheel. - name: Assert this ref's Python was really put under test if: matrix.overlay && inputs.installer_source != 'published' && steps.install.outcome == 'success' run: | @@ -321,9 +309,7 @@ jobs: set -a; . ./clean-machine.env; set +a checks="nobuild" # `absent` ran only BEFORE the install, so an installer that quietly selected - # the CLT or installed a compiler left the leg green while every later source - # build could succeed, the exact behaviour `absent` claims to guard the whole - # run against. Re-run it afterwards. + # the CLT or installed a compiler left the leg green. Re-run it afterwards. [ "${{ matrix.mode }}" = "mask" ] && checks="$checks absent" [ "${{ matrix.mode }}" = "trace" ] && checks="$checks notools" UNSLOTH_CLEAN_ALLOW_WORKING='${{ matrix.allow_working }}' \ @@ -785,21 +771,14 @@ jobs: # ── WSL ─────────────────────────────────────────────────────────────────── # install.sh carries ~126 lines of WSL-specific logic (the `linux|wsl` dependency # branch, UNSLOTH_WSL_REROUTED, the Strix Halo reroute to 24.04) that had never run - # in CI: tests/sh/test_strixhalo_wsl_reroute.sh extracts functions and mocks the - # environment, which cannot catch anything about a real WSL. + # in CI: tests/sh/test_strixhalo_wsl_reroute.sh mocks the environment, which cannot + # catch anything about a real WSL. No third-party action either -- the official + # Ubuntu rootfs plus `wsl --import` is deterministic and checksum-verifiable. # - # No third-party action: the official Ubuntu rootfs plus `wsl --import` is - # deterministic and checksum-verifiable, adding no supply-chain dependency to a repo - # that audits its lockfiles. - # - # Gating, deliberately: this is the only job that runs the real WSL branch, so a - # job-level continue-on-error made the distro import, the installer exit code, the - # `platform wsl` assertion and the CLI check all unable to fail anything. There is no - # flake to absorb; if the pinned rootfs ever moves, a red job is the correct signal. - # - # It is also the only job that can catch a piped install being truncated: WSL is the - # one platform whose install shells out to Windows interop mid-script, and interop - # relays the stdin it inherited. #7548 is in main now, so this gates unconditionally. + # Gating, deliberately: it is the only job that runs the real WSL branch and the only + # one that can catch a piped install being truncated (WSL shells out to Windows + # interop mid-script, and interop relays the stdin it inherited). There is no flake to + # absorb, and #7548 is in main, so this gates unconditionally. wsl: name: wsl ubuntu-24.04 runs-on: windows-latest @@ -864,12 +843,11 @@ jobs: Write-Host '::error::the WSL install produced no log' exit 1 } - # This job is the one that proves the pipe stays intact. WSL is the only - # platform whose install shells out to Windows interop mid-script - # (_maybe_reroute_strixhalo_to_2404 -> powershell.exe, wsl.exe), and interop - # relays the stdin it inherited, so before #7548 it drank the rest of the - # script and sh died on a half-read line. #7548's _unsloth_main wrapper makes - # sh parse the whole file first; a truncation here means that regressed. + # The pipe-integrity check. Interop (_maybe_reroute_strixhalo_to_2404 -> + # powershell.exe, wsl.exe) relays the stdin it inherited, so before #7548 it + # drank the rest of the script and sh died on a half-read line. #7548's + # _unsloth_main wrapper makes sh parse the file first; a truncation here means + # that regressed. if (Select-String -Path logs/wsl-install.log ` -Pattern 'Syntax error: Unterminated quoted string' -Quiet) { Write-Host '::error::the piped install was truncated again; install.sh is no longer parsed in full before it runs' @@ -878,7 +856,7 @@ jobs: # Printing the code discarded it, and the next step's CLI check does not # compensate: install.sh links the `unsloth` shim (4174-4182) BEFORE it reports # a failing studio/setup.sh (4219-4230), so a late setup failure leaves a shim - # whose --version succeeds and the job looked green. + # whose --version succeeds. if ($installRc -ne 0) { Write-Host "::error::WSL installer exited $installRc" exit $installRc @@ -891,9 +869,8 @@ jobs: # The platform line proves the wsl branch was taken rather than plain linux. Select-String -Path logs/wsl-install.log -Pattern 'platform|\[TAURI:DIAG\]|wsl' -ErrorAction SilentlyContinue | Select-Object -First 10 - # Printing could not fail, and that alternation also matches `platform linux`: - # if detection regresses, every WSL branch is skipped and this job still passes - # as a plain-Linux install, the one thing no other job covers. `step` writes + # Printing could not fail, and that alternation also matches `platform linux`, + # so a detection regression would pass as a plain-Linux install. `step` writes # the label in reverse video, so strip ANSI or an anchored match never hits. $esc = [char]27 $platformLines = @( @@ -907,8 +884,7 @@ jobs: exit 1 } # No `|| echo`: substituting a message for the missing CLI made the inner - # shell, this step and the job all succeed even when the install produced - # nothing usable, which is half of what this step asks. + # shell, this step and the job all succeed on an install that produced nothing. $verify = wsl -d unsloth-ci -u root -- sh -c 'set -e; test -x "$HOME/.unsloth/studio/unsloth_studio/bin/unsloth"; "$HOME/.unsloth/studio/unsloth_studio/bin/unsloth" --version' 2>&1 $verifyRc = $LASTEXITCODE $verify | Tee-Object -FilePath logs/wsl-verify.log @@ -995,11 +971,11 @@ jobs: $drop = @('hostedtoolcache\windows\Python', 'WindowsApps', '\Git\', 'CMake', 'Microsoft Visual Studio', 'BuildTools', 'LLVM', 'MSYS', 'mingw', 'Strawberry') - # winget is an app-execution alias in ...\Local\Microsoft\WindowsApps, which - # the blanket drop removes on EVERY leg, so winget=visible was silently running - # the same no-winget fallback as winget=masked. Resolve it before the scrub and - # hand it back through a shim: the visible leg gets winget without the Store's - # python.exe alias returning. windows-11-arm has no winget on the hosted image + # winget is an app-execution alias in ...\Local\Microsoft\WindowsApps, so the + # blanket drop removed it on EVERY leg and winget=visible silently ran the same + # fallback as winget=masked. Resolve it before the scrub and hand it back + # through a shim, so the visible leg gets winget without the Store's python.exe + # alias. windows-11-arm has no winget on the hosted image # (actions/runner-images#14083), so only windows-latest can carry it. $wantWinget = ('${{ matrix.winget }}' -ne 'masked') -and ('${{ matrix.os }}' -eq 'windows-latest') $wingetCmd = Get-Command winget -ErrorAction SilentlyContinue @@ -1074,14 +1050,13 @@ jobs: if ($f -and $t -ne 'py') { $leaked += "$t -> $($f.Source)" } } # The launcher binary may stay, but an interpreter it can still START is a - # leak: Find-CompatiblePython (install.ps1:1130-1153) probes `py` first, so any - # version registered outside the two renamed toolcache directories gets reused - # and Python bootstrap never runs. Exempting `py` left that unchecked. + # leak: Find-CompatiblePython (install.ps1:1130-1153) probes `py` first, so a + # version registered outside the renamed toolcache dirs gets reused and the + # Python bootstrap never runs. if (Get-Command py -ErrorAction SilentlyContinue) { - # -0p prints the launcher's REGISTRY view. The mask renames the toolcache - # directory on disk but cannot rewrite those entries, so -0p keeps naming - # paths that no longer exist: context for a failure, never evidence of one. - # Only a probe that actually STARTS counts. + # -0p is the launcher's REGISTRY view, and the mask renames directories + # without rewriting it, so -0p keeps naming paths that no longer exist: + # context for a failure, never evidence of one. Only a probe that STARTS counts. Write-Host "py -0p (stale registry entries; masked paths no longer exist on disk):" & py -0p 2>&1 | ForEach-Object { Write-Host " $_" } foreach ($v in '-3.11', '-3.12', '-3.13') { @@ -1092,16 +1067,13 @@ jobs: if ($rc -eq 0) { $leaked += "py $v -> $out" } } # A FAILING probe is the outcome we want, but it leaves $LASTEXITCODE - # non-zero and cmdlets never reset it. The runner appends - # if ((Test-Path -LiteralPath variable:\LASTEXITCODE)) { exit $LASTEXITCODE } - # to every pwsh step (actions/runner#351), so all three Windows legs exited 1 - # with no ::error:: printed, on machines that were in fact clean, and never - # reached the Install step. + # non-zero and cmdlets never reset it, and the runner appends + # `exit $LASTEXITCODE` to every pwsh step (actions/runner#351) -- so all + # three legs exited 1, silently, on machines that were in fact clean. $global:LASTEXITCODE = 0 } - # Printing alone could not fail, and the leg was green while not clean: run - # 30365014702 logged `python ABSENT` then `Python 3.13 already installed` / - # `Using CPython ... C:\hostedtoolcache\windows\Python\...`. + # Printing alone could not fail: run 30365014702 logged `python ABSENT` then + # `Python 3.13 already installed` / `Using CPython ... C:\hostedtoolcache\...`. if ($leaked) { Write-Host "::error::developer tooling survived the scrub: $($leaked -join '; ')" exit 1 @@ -1114,8 +1086,7 @@ jobs: exit 1 } } elseif ('${{ matrix.os }}' -eq 'windows-latest' -and -not $winget) { - # Without this the visible leg quietly degrades into a second masked leg and - # nothing in this workflow exercises the normal winget bootstrap. + # Otherwise the visible leg quietly degrades into a second masked leg. Write-Host '::error::winget is not resolvable on the visible leg; the winget bootstrap is not under test' exit 1 } @@ -1150,13 +1121,11 @@ jobs: } else { Write-Host "installer: this ref ($env:GITHUB_SHA)" } - # No -SkipTorch: install.ps1 has no param block and its parser matches - # `--no-torch` only (112-142), so the token was silently dropped and every - # Windows leg installed torch anyway. The assert below needs torch, so get it - # on purpose rather than by accident. - # Under powershell.exe, not this pwsh 7 step: a clean Windows box ships - # Windows PowerShell 5.1 only, and the desktop launches it the same way - # (install.rs:325-339). pwsh 7 is a runner-image extra no user is promised. + # No -SkipTorch: install.ps1's parser matches `--no-torch` only (112-142), so + # the token was silently dropped and every leg installed torch anyway. The + # assert below needs torch, so get it on purpose. Run under powershell.exe, not + # this pwsh 7 step: a clean Windows box ships Windows PowerShell 5.1 only, and + # the desktop launches it the same way (install.rs:325-339). & powershell.exe -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass ` -File $script *>&1 | Tee-Object -FilePath logs/install.log $rc = $LASTEXITCODE @@ -1171,12 +1140,11 @@ jobs: if: always() && matrix.os == 'windows-11-arm' && steps.install.outcome != 'skipped' shell: pwsh run: | - # The flip condition. #7549 makes install.ps1 prefer an x64 CPython on an ARM64 - # host, and torchaudio does publish win_amd64, so an AMD64 venv interpreter - # means the fix has landed and this pin is stale. Asked of the interpreter - # (sysconfig), not inferred from PROCESSOR_ARCHITECTURE, which describes the - # shell. The failing PyTorch step runs after the venv exists, so the absence of - # that interpreter is itself a different failure. + # The flip condition. #7549 makes install.ps1 prefer an x64 CPython on ARM64 + # hosts and torchaudio does publish win_amd64, so an AMD64 venv interpreter + # means the fix landed. Asked of the interpreter (sysconfig), not inferred from + # PROCESSOR_ARCHITECTURE, which describes the shell. The pinned PyTorch failure + # comes after the venv exists, so a missing interpreter is a different failure. $venvPy = Join-Path $env:UNSLOTH_STUDIO_HOME 'unsloth_studio\Scripts\python.exe' if (-not (Test-Path -LiteralPath $venvPy)) { Write-Host "::error::no venv interpreter at $venvPy; the run did not even reach the pinned PyTorch failure" @@ -1221,12 +1189,11 @@ jobs: run: | # The flip condition. studio/setup.ps1:1655-1669 gates on git unconditionally # and can only get it from winget, so masking winget leaves no way to satisfy - # it; #7549 relaxes the gate to the --local and llama.cpp source-build paths - # that actually use git, so that wording being present means the fix landed. - # Read from the checkout because this row is overlay: true, which is what makes - # the branch's setup.ps1 the one that runs (install.ps1:2626-2642). A published - # dispatch takes setup.ps1 from the released wheel instead, so there the tree - # says nothing and only the outcome checks below can retire the pin. + # it; #7549 relaxes the gate to the --local and source-build paths that really + # use git, so that wording appearing means the fix landed. Read from the + # checkout because overlay: true makes the branch's setup.ps1 the one that runs + # (install.ps1:2626-2642); a published dispatch takes it from the released + # wheel instead, where only the outcome checks below can retire the pin. if ('${{ inputs.installer_source }}' -ne 'published') { $relaxed = Select-String -Path studio/setup.ps1 -SimpleMatch -Quiet ` -Pattern 'Git is required for --local and llama.cpp source-build installs' @@ -1245,10 +1212,10 @@ jobs: } $log = Get-Content logs/install.log -Raw # All three, so a failure anywhere else is still red: winget really was absent, - # the git gate is what fired, and it is what stopped studio setup rather than - # a warning the install walked past. The winget check is not redundant with the - # matrix -- without it this pin would also absorb a leg whose PATH scrub failed - # and which then died on the same gate for an entirely different reason. + # the git gate fired, and it is what stopped studio setup rather than a warning + # the install walked past. The winget check is not redundant with the matrix -- + # without it the pin would absorb a leg whose PATH scrub failed and which then + # died on the same gate for a different reason. $noWinget = $log -match 'will require Python \+ uv to be already installed' $gitGate = $log -match 'Git is required but could not be installed automatically' $setupRc = $log -match 'unsloth studio setup failed \(exit code 1\)' @@ -1274,21 +1241,20 @@ jobs: if: always() && steps.install.outcome != 'skipped' shell: pwsh run: | - # macOS and Linux re-run `absent`/`nobuild` after the install; Windows checked - # nothing afterwards, so setup.ps1 committing to a llama.cpp SOURCE build would - # winget-install CMake (setup.ps1:816-822) and VS Build Tools (845-857) and the - # leg still went green. Git is out of scope on purpose: bootstrapping it through - # winget (setup.ps1:1658-1661) is the consumer path the visible leg exists to - # exercise. The VC++ runtime is a runtime, not a toolchain, and is likewise fine. + # Windows checked nothing after the install, so setup.ps1 committing to a + # llama.cpp SOURCE build would winget-install CMake (setup.ps1:816-822) and VS + # Build Tools (845-857) and the leg still went green. Git is out of scope on + # purpose: bootstrapping it through winget (setup.ps1:1658-1661) is the consumer + # path the visible leg exercises. The VC++ runtime is a runtime, not a toolchain. $bad = @() if (-not (Test-Path logs/install.log)) { Write-Host '::error::no install log, so nothing proves the install stayed toolchain-free' exit 1 } # The announcements inside Ensure-BuildToolsForLlamaSourceBuild, which runs only - # when a source build is committed. Matched instead of the package ids: setup.ps1 - # PRINTS `winget install Microsoft.VisualStudio.2022.BuildTools` as manual advice - # when winget is missing, and advice is not an install. + # for a committed source build. Matched instead of the package ids, because + # setup.ps1 PRINTS `winget install ...BuildTools` as manual advice when winget is + # missing, and advice is not an install. foreach ($m in 'CMake not found -- installing via winget', 'Visual Studio Build Tools not found -- installing via winget') { if (Select-String -Path logs/install.log -Pattern $m -SimpleMatch -Quiet) { @@ -1332,13 +1298,11 @@ jobs: if: steps.install.outcome == 'success' shell: pwsh run: | - # HONESTY NOTE: the hosted image ships the VC++ 2015-2022 runtime in System32 - # and it cannot be removed without breaking the runner, so `import torch` - # succeeding here does NOT prove a genuinely clean no-winget machine has the - # runtime: Test-VCRedistInstalled (setup.ps1:875) finds the preinstalled DLL - # and Ensure-VCRedist (891) short-circuits before it needs winget. Record that, - # then assert what CAN fail: torch imports, and the masked leg really did take - # the no-winget path. + # HONESTY NOTE: the image ships the VC++ 2015-2022 runtime in System32 and it + # cannot be removed without breaking the runner, so `import torch` succeeding + # does NOT prove a clean no-winget machine has it -- Test-VCRedistInstalled + # (setup.ps1:875) finds the preinstalled DLL and Ensure-VCRedist (891) + # short-circuits. Record that, then assert what CAN fail. $sys32 = Join-Path $env:WINDIR 'System32\vcruntime140_1.dll' Write-Host "preinstalled System32 vcruntime140_1.dll: $(Test-Path $sys32)" $py = Join-Path $env:UNSLOTH_STUDIO_HOME 'unsloth_studio\Scripts\python.exe' @@ -1367,12 +1331,10 @@ jobs: # ── Windows, genuinely virgin: the same install inside a Windows container ──── # The `win` legs above only SIMULATE absence, and two things they structurally - # cannot test are the VC++ 2015-2022 runtime (it ships in the runner image's - # System32 and cannot be removed without breaking the runner, hence the HONESTY - # NOTE on the torch assert) and a Windows with no Microsoft Store at all rather - # than a winget hidden from PATH. A servercore container answers both, so this - # lane lives here rather than in a sibling file: same premise, same path filters, - # and the reader compares masked against real in one place. + # cannot test are the VC++ 2015-2022 runtime (it ships in the runner image's System32 + # and cannot be removed without breaking the runner) and a Windows with no Microsoft + # Store at all rather than a winget hidden from PATH. A servercore container answers + # both, so this lane lives here: same premise, same path filters, masked next to real. # # Constraints, all load-bearing: # * `container:` is Linux-only on the Actions runner (actions/runner#1402), so @@ -1395,9 +1357,9 @@ jobs: fetch-depth: 1 persist-credentials: false - # Docker is installed on every windows-2022 image but is not always already - # running: one spike leg died in 21s on npipe:////./pipe/docker_engine, and that - # flake misreads as "Windows containers are unavailable". + # Docker is on every windows-2022 image but is not always already running: one + # spike leg died in 21s on npipe:////./pipe/docker_engine, which misreads as + # "Windows containers are unavailable". - name: Ensure the Docker daemon is running shell: pwsh run: ./.github/scripts/ensure-docker-daemon.ps1 @@ -1418,11 +1380,9 @@ jobs: - name: Start the container shell: pwsh run: | - # Deliberately never refresh a cached image: process isolation needs the - # container build <= the host build, and MCR has shipped a patched image - # ahead of the runner host before (actions/runner-images#11582 broke every - # Windows container job for ~2 weeks). The cached one is the one that - # matched at runner-image build time. + # Never refresh a cached image: process isolation needs the container build + # <= the host build, and MCR has shipped a patched image ahead of the host + # before (actions/runner-images#11582 broke Windows containers for ~2 weeks). if ((docker images --format '{{.Repository}}:{{.Tag}}') -contains $env:IMAGE) { Write-Host "using the runner's pre-cached $env:IMAGE (no pull)" } else { @@ -1515,13 +1475,12 @@ jobs: *>&1 | Tee-Object -FilePath logs/virginity.log exit $LASTEXITCODE - # AFTER the virginity assertion, so that assertion still proves what it says. - # A fresh container ships an almost empty trusted-root store; a real Windows - # desktop fills it via automatic root update, so seeding it makes the container - # MORE representative, not less. Needed because studio/install_node_prebuilt.py - # downloads Node with bare urllib.request.urlopen and so reads the empty Windows - # ROOT store and gets CERTIFICATE_VERIFY_FAILED; uv and pip bundle certifi and - # are unaffected. That product fragility is reported separately, not fixed here. + # AFTER the virginity assertion, so that assertion still proves what it says. A + # fresh container ships an almost empty trusted-root store while a real desktop + # fills it via automatic root update, so seeding makes this MORE representative. + # Needed because studio/install_node_prebuilt.py downloads Node with bare + # urllib.request.urlopen, reads the empty Windows ROOT store and gets + # CERTIFICATE_VERIFY_FAILED; uv and pip bundle certifi. Reported separately. - name: Seed the container's trusted root CA store shell: pwsh run: | diff --git a/.github/workflows/desktop-app-clean-machine-ci.yml b/.github/workflows/desktop-app-clean-machine-ci.yml index acb695f9bf..88dd51303a 100644 --- a/.github/workflows/desktop-app-clean-machine-ci.yml +++ b/.github/workflows/desktop-app-clean-machine-ci.yml @@ -10,12 +10,11 @@ # running its bundled Contents/Resources/install.sh, which no CI job exercised. # # Hosted runners have no interactive desktop session, so "runs" means: the bundle -# installs / mounts / extracts, the binary is present, of the right architecture, and -# clears the gatekeeper checks a user would hit (macOS quarantine + codesign, Windows -# installer exit); the process STAYS UP past its preflight (where an unhappy app dies); -# and it writes tauri.log with a preflight disposition -- the field that read -# `ManagedReady` over an unbootable venv in the bug report. Linux gets the strongest -# check: a real webview under Xvfb. +# installs, the binary is present, of the right architecture, and clears the gatekeeper +# checks a user hits (macOS quarantine + codesign, Windows installer exit); the process +# STAYS UP past its preflight, where an unhappy app dies; and it writes tauri.log with a +# preflight disposition, the field that read `ManagedReady` over an unbootable venv in +# the bug report. Linux gets the strongest check: a real webview under Xvfb. name: Desktop app clean machine @@ -61,12 +60,11 @@ env: # one frozen release, so the schedule was re-testing the same fixture forever. REL_REPO: ${{ inputs.release_repo || github.repository }} # Empty unless dispatched: a pinned tag is an immutable fixture, so a nightly against - # it could never catch a newly published broken bundle. Each download step resolves - # the newest desktop-v* release when this is empty -- drafts included, because that is - # how every desktop-v* release here is cut (desktop-v0.1.50-beta, desktop-v0.1.471-beta - # are both drafts), so --exclude-drafts matched nothing and every leg died resolving. - # A draft has no tag ref and releases/tags/ 404s for one, but gh looks drafts up - # over GraphQL, so `gh release download ` still fetches their assets. + # it could never catch a newly published broken bundle. Each download step then + # resolves the newest desktop-v* release, drafts included -- every desktop-v* release + # here is cut as a draft, so --exclude-drafts matched nothing and every leg died + # resolving. releases/tags/ 404s for a draft, but gh looks drafts up over GraphQL, + # so `gh release download ` still fetches their assets. REL_TAG: ${{ inputs.release_tag || '' }} UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home UNSLOTH_STUDIO_DISABLE_PUBLIC_CHECK: '1' @@ -180,11 +178,11 @@ jobs: set -o pipefail APP="$(ls -d /Applications/*Unsloth*.app | head -1)" # A headless runner never clicks Install: preflight sets `not_installed` and - # returns (use-tauri-backend.ts:252-254) while startup-screen.tsx:388-389 - # waits for the button, so launching alone sits on that screen for 90s and - # passes without ever running the bundled installer. Invoke it as - # src-tauri/src/install.rs does: --tauri, stdin closed, no tty. --tauri - # rejects a custom studio home (install.sh:102-114), so drop the override. + # returns (use-tauri-backend.ts:252-254) while startup-screen.tsx:388-389 waits + # for the button, so launching alone sits there for 90s without ever running + # the bundled installer. Invoke it as src-tauri/src/install.rs does: --tauri, + # stdin closed, no tty. --tauri rejects a custom studio home + # (install.sh:102-114), so drop the override. # KNOWN OUTCOME PIN, retire when the desktop release catches up to #7547. # REL_TAG predates #7547, so the bundle's own install.sh still hard-exits on # the Xcode CLT gate that #7547 replaced with a warning. No change to this PR @@ -328,18 +326,14 @@ jobs: ls -la dl - name: Strip the developer toolchain - # Same gate as macOS. Without this the Linux rows ignored strip_toolchain - # entirely and ran the bundled installer with the runner's git, gcc, cmake and - # make in /usr/bin, so a bundle that needs a toolchain passed the one workflow - # whose premise is that it must not. + # Same gate as macOS: without it the Linux rows ignored strip_toolchain and ran + # the bundled installer with the runner's git, gcc, cmake and make in /usr/bin. # # BEFORE the bundle install, as macOS and Windows already do: dpkg runs the - # package's own maintainer scripts, and installing first meant they ran with the - # hosted image's git, compilers and cmake in /usr/bin, so a release whose scripts - # reached for one would pass here and fail on a clean machine. Nothing in that - # install needs a masked tool -- clean-machine-env.sh moves aside only $TOOLS - # (compilers, git, cmake, make, brew, cargo), leaving apt, dpkg and sudo -- and - # the current bundle ships a postrm and no install-time script at all. + # package's own maintainer scripts, so installing first let them see the hosted + # image's toolchain. Nothing in that install needs a masked tool -- + # clean-machine-env.sh moves aside only $TOOLS, leaving the package manager + # itself -- and the current bundle ships a postrm and no install-time script. if: ${{ github.event_name != 'workflow_dispatch' || inputs.strip_toolchain }} run: | bash .github/scripts/clean-machine-env.sh mask --remove @@ -660,13 +654,11 @@ jobs: - name: Run the bundled installer, the path first launch takes shell: pwsh run: | - # The launch step below only proves the process stayed alive. On a fresh - # profile preflight reports not_installed and the app waits for a click on - # Install (use-tauri-backend.ts:252-254, startup-screen.tsx:388-389), so this - # job passed on a bundle whose embedded install.ps1 was missing or broken -- - # the packaged-app failure the workflow exists to catch. - # tauri.conf.json:56-59 ships it as a bundle resource, so find it where NSIS - # put it and invoke it as install.rs:326-341 does. + # The launch step below only proves the process stayed alive: on a fresh + # profile the app waits for a click on Install (use-tauri-backend.ts:252-254, + # startup-screen.tsx:388-389), so this job passed on a bundle whose embedded + # install.ps1 was missing or broken. tauri.conf.json:56-59 ships it as a bundle + # resource, so find it where NSIS put it and invoke it as install.rs:326-341. $root = Split-Path -Parent $env:APP_EXE $ps1 = Get-ChildItem -Path $root -Recurse -Filter 'install.ps1' -ErrorAction SilentlyContinue | Select-Object -First 1 @@ -734,12 +726,11 @@ jobs: -SimpleMatch -Quiet) { $disposition = $true } } } - # Same acceptance criterion macOS and Linux already enforce. Test-Path, - # Get-Content and Select-String cannot fail, so without these two lines the - # step was decoration and the 90s liveness check was the whole bar. - # setup_logging (src-tauri/src/main.rs:50-67) opens tauri.log unconditionally - # at process start, so no log means the binary never got that far, and an app - # that hangs before preflight completes would otherwise pass. + # Same acceptance criterion macOS and Linux enforce. Test-Path, Get-Content and + # Select-String cannot fail, so without these two lines the step was decoration. + # setup_logging (src-tauri/src/main.rs:50-67) opens tauri.log unconditionally at + # process start, so no log means the binary never got that far, and an app that + # hangs before preflight would otherwise pass. if (-not $found) { Write-Host '::error::the app wrote no tauri.log; it never reached setup_logging' exit 1 From 467d02b1dd873509fe5c08218018fb6ee992e73f Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 05:06:42 +0000 Subject: [PATCH 34/36] Re-assert toolchain absence after the desktop .deb pulls its dependencies --- .github/workflows/desktop-app-clean-machine-ci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/desktop-app-clean-machine-ci.yml b/.github/workflows/desktop-app-clean-machine-ci.yml index 88dd51303a..e5746a2343 100644 --- a/.github/workflows/desktop-app-clean-machine-ci.yml +++ b/.github/workflows/desktop-app-clean-machine-ci.yml @@ -362,6 +362,18 @@ jobs: echo "BIN=$BIN" >> "$GITHUB_ENV" echo "binary: $BIN" + # The strip runs before this, but `apt-get install ./dl/*.deb` then pulls the + # bundle's DECLARED dependencies, so a release that adds git, cmake or a compiler + # to that list puts one back in /usr/bin and both required Linux rows still pass. + # `absent` ran only beforehand, so re-run it here, before the bundled installer. + # The current dependency closure is 65 packages of runtime libs and no toolchain, + # so this is green today and only a new dependency can turn it red. + - name: Re-assert the toolchain is still absent after the package install + if: ${{ github.event_name != 'workflow_dispatch' || inputs.strip_toolchain }} + run: | + set -a; . ./clean-machine.env; set +a + bash .github/scripts/clean-machine-assert.sh absent + - name: Run the bundled installer, the path first launch takes run: | set -a; [ -f ./clean-machine.env ] && . ./clean-machine.env; set +a From d6d2408ce601a4c4184b575f6ff8336c1b827d8a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 05:45:10 +0000 Subject: [PATCH 35/36] Retire the #7549 pins and add a wget-only Linux leg #7549 is in main, so the three known-outcome pins that were waiting on it are stale and would now hard-error by design. Each is replaced by the assertion it was standing in for rather than deleted: win windows-11-arm now gates. The x64-on-ARM64 resolver is asserted as an outcome: the venv interpreter reports win-amd64 from its own sysconfig, and torchaudio (no win_arm64 wheel at any version) is installed. Measured on the integration branch before #7549 merged: "only a native ARM64 Python 3.13 was found" -> "installing x64 Python" -> torchaudio 2.10.0+cpu, install green. win windows-latest / winget=masked now gates. The relaxed git gate is asserted from both sides: the old unconditional message must be absent, the no-git branch must have been reached (so the row cannot pass because git leaked back onto PATH), and setup.ps1 must report git as absent-but-not-required. Both Windows rows, and the visible one, gained the usability check the Linux legs have had and Windows never did: a managed interpreter, an unsloth CLI on disk, and that CLI actually running. nobuild and the toolchain check only read the log, so an installer that exited 0 having produced nothing satisfied them. The torch assert also loses its fallback to whatever `python` resolves to. The virgin container overlay row gates, and asserts what only that lane can: it is the one environment whose System32 does not already ship the VC++ 2015-2022 runtime, so it is the only place Ensure-VCRedist's direct aka.ms download can be proved to run rather than be short-circuited. The overlay=false row keeps a pin, with a new reason: it installs unsloth from PyPI on purpose, and setup.ps1 inside 2026.7.5 (uploaded the 23rd) predates #7549, so it still stops at the old gate. That is release lag, it flips on the next release, and the pinned signature is now the old wording rather than "#7549 has not landed". Also adds linux ubuntu2404-nonroot-wget. install.sh's download() takes curl or wget and _transport_missing is true only when both are gone, so a wget-only box is supported on paper, but the gating nonroot leg provisions ca-certificates AND curl, so curl won every probe and the wget branch had never run. Same image, same no-sudo user, same asserts, wget instead of curl, and curl proved absent on disk for root and for tester before AND after the install, so the claim is that every download went through wget rather than that curl happened to be unused. --- .../workflows/clean-machine-install-ci.yml | 417 ++++++++++++------ 1 file changed, 280 insertions(+), 137 deletions(-) diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index 5476e1dacc..166092aa27 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -406,6 +406,24 @@ jobs: experimental: false overlay: true nonroot: true + # The same premise with the OTHER transport. install.sh's download() prefers + # curl and falls back to wget (install.sh:729-738), _http_get does the same + # for the connectivity probe (1019-1027) and the Radeon listing repeats it + # (3064-3067), and _check_linux_deps only calls the transport missing when + # BOTH are gone (2077-2079). So a box with wget and no curl -- a Debian + # netinst default, and every image where curl was deliberately removed -- is + # supported on paper and had never been run: the nonroot row above provisions + # ca-certificates AND curl, so curl won every probe and the wget branch was + # only ever reasoned from the code. Everything the row asserts is what the + # nonroot row asserts, plus curl proved absent for the whole run rather than + # merely unused. + - label: ubuntu2404-nonroot-wget + image: ubuntu:24.04 + runner: ubuntu-latest + experimental: false + overlay: true + nonroot: true + wget_only: true # No elevation AND no transport. apt is the only way to get curl and reaching # apt is what needs elevation, so failing is correct; the point is to pin the # exact message and prove it is actionable rather than a bare `curl: (56)`. @@ -441,7 +459,14 @@ jobs: # (it needs git) the only way in for this ref's source is an archive over the # same transport. Neither is a compiler, git or cmake, so the premise holds. # Both are usually in the base image already; naming them makes it certain. - pkgs="ca-certificates curl" + # + # One transport, never both: the wget-only row is testing install.sh's wget + # branch, and that branch is unreachable while curl is on the box. + if [ "${{ matrix.wget_only }}" = "true" ]; then + pkgs="ca-certificates wget" + else + pkgs="ca-certificates curl" + fi if [ "${{ matrix.overlay }}" = "true" ]; then pkgs="$pkgs tar gzip"; fi if command -v apt-get >/dev/null 2>&1; then apt-get update -qq && apt-get install -y -qq --no-install-recommends $pkgs @@ -457,13 +482,20 @@ jobs: run: | mkdir -p logs .github/scripts raw="https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/${GITHUB_SHA}" - curl -fsSL "$raw/.github/scripts/clean-machine-assert.sh" -o .github/scripts/clean-machine-assert.sh + # Whichever transport this row provisioned: the wget-only leg has no curl, on + # purpose, and this is the one download in the job that cannot go through + # install.sh's own helper. Same preference order as that helper. + dl() { + if command -v curl >/dev/null 2>&1; then curl -fsSL "$1" -o "$2" + else wget -q -O "$2" "$1"; fi + } + dl "$raw/.github/scripts/clean-machine-assert.sh" .github/scripts/clean-machine-assert.sh # Empty on pull_request/push, so only an explicit dispatch tests unsloth.ai. if [ "${{ inputs.installer_source }}" = "published" ]; then - curl -fsSL https://unsloth.ai/install.sh -o install.sh + dl https://unsloth.ai/install.sh install.sh echo "installer: published (unsloth.ai)" else - curl -fsSL "$raw/install.sh" -o install.sh + dl "$raw/install.sh" install.sh echo "installer: this ref (${GITHUB_SHA})" fi wc -l install.sh @@ -476,8 +508,13 @@ jobs: run: | set -e mkdir -p ci-source - curl -fsSL "https://codeload.github.com/${GITHUB_REPOSITORY}/tar.gz/${GITHUB_SHA}" \ - | tar -xz -C ci-source --strip-components=1 + src="https://codeload.github.com/${GITHUB_REPOSITORY}/tar.gz/${GITHUB_SHA}" + # See the step above: the wget-only leg has to fetch with wget. + if command -v curl >/dev/null 2>&1; then + curl -fsSL "$src" | tar -xz -C ci-source --strip-components=1 + else + wget -q -O - "$src" | tar -xz -C ci-source --strip-components=1 + fi [ -f ci-source/pyproject.toml ] || { echo "::error::source tarball for ${GITHUB_SHA} unpacked without a pyproject.toml"; ls -la ci-source; exit 1; } echo "overlay source: $(pwd)/ci-source" @@ -546,6 +583,43 @@ jobs: fi echo "tester is unprivileged, has no sudo on disk, and cannot write dpkg state" + # The mirror of the sudo proof above, for the other premise this row makes. Not + # calling curl is not the same as not having it: every transport site in + # install.sh probes with `command -v curl` and prefers it (731, 1022, 2078, 3064), + # so a leg that merely avoided the call would go on testing the curl branch and + # report the wget one green. + - name: Prove wget is the only transport + if: matrix.wget_only + run: | + # Absent from disk, not merely off PATH, for the same reason the sudo check + # looks on disk: `command -v` is what install.sh asks, and a binary tester + # could not reach would still be a lie about the image. + for p in /usr/bin/curl /bin/curl /usr/local/bin/curl /usr/sbin/curl /sbin/curl /snap/bin/curl; do + if [ -e "$p" ]; then + echo "::error::$p exists, so this leg is not wget-only" + exit 1 + fi + done + for u in root tester; do + if su "$u" -c 'command -v curl' >/dev/null 2>&1; then + echo "::error::curl resolves for $u; the wget-only premise does not hold" + exit 1 + fi + done + # The package too, so a dependency that quietly pulled the binary back in is + # caught here rather than silently reinstating the curl branch. + if dpkg-query -W -f='${Status}' curl 2>/dev/null | grep -q 'install ok installed'; then + echo "::error::the curl package is installed; the wget-only premise does not hold" + exit 1 + fi + # And tester really does have the other one, or the row is the no-transport + # case wearing a different label. + su tester -c 'command -v wget' >/dev/null 2>&1 || { + echo "::error::tester cannot reach wget, so this leg has no transport at all" + exit 1 + } + echo "wget only: $(su tester -c 'wget --version' | head -1)" + - name: Install (root) id: install_root if: ${{ !matrix.nonroot }} @@ -748,6 +822,33 @@ jobs: exit 1 fi + # Absent at the start is not absent throughout, and only the whole-run claim makes + # the leg mean anything: every download the installer just did -- the uv bootstrap + # included (install.sh:2232) -- had to go through wget, and it did only if curl was + # never there to be preferred. + - name: Re-prove curl never appeared, and that wget carried the install + if: matrix.wget_only && steps.install_nonroot.outcome == 'success' + run: | + for p in /usr/bin/curl /bin/curl /usr/local/bin/curl /usr/sbin/curl /sbin/curl /snap/bin/curl; do + if [ -e "$p" ]; then + echo "::error::$p appeared during the install, so the run did not stay wget-only" + exit 1 + fi + done + if command -v curl >/dev/null 2>&1; then + echo "::error::curl resolves after the install, so the run did not stay wget-only" + exit 1 + fi + # _check_linux_deps (install.sh:2077-2079) calls the transport missing only + # when curl AND wget are both gone, and the elevation gate below it is what the + # notransport row pins. Reaching it here would mean wget was not recognised as + # a transport at all. + if grep -q "missing: curl" logs/install.log; then + echo "::error::install.sh reported the transport as missing on a box that has wget, so it does not accept wget as one" + exit 1 + fi + echo "curl absent before and after; every download went through wget" + - name: Assert no source build if: always() run: | @@ -926,28 +1027,25 @@ jobs: # Ensure-VCRedist silently does not run, leaving torch unable to load, hence # the explicit `import torch` assert below. # - # On this ref alone it stops at studio/setup.ps1:1655-1669, the unconditional - # "Git is required but could not be installed automatically" gate: no winget - # means no way to fetch git. That gate is what #7549 relaxes to the --local and - # llama.cpp source paths that actually use git; with it applied the leg is - # green (staging run 30407859691, all 16 legs). It is a merge-order dependency, - # not a product gap, and the overlay is what lets this workflow see the fix - # land. The Install step is therefore continue-on-error and the step below pins - # that exact failure: the row stays required, so a DIFFERENT failure is still - # red, and the pin turns into a hard error the moment the relaxed gate appears. + # It used to stop at studio/setup.ps1's unconditional "Git is required but + # could not be installed automatically" gate -- no winget meant no way to fetch + # git -- and was carried as a pinned known failure until #7549 landed. #7549 + # relaxed that gate to the --local and llama.cpp source paths that actually use + # git (setup.ps1:1750-1759), so the row installs end to end and gates like any + # other; the assert below is what proves it took the relaxed branch rather than + # passing because git leaked back onto PATH. - os: windows-latest winget: 'masked' experimental: false overlay: true # Windows on ARM gets a native ARM64 CPython, and torchaudio has never - # published a win_arm64 wheel at any version, so the PyTorch step cannot - # resolve and install.ps1 stops at "Failed to install PyTorch". #7549 fixes it - # by preferring an x64 interpreter on an ARM64 host (x64 wheels run fine under - # emulation), and that fix lives in install.ps1 on an unmerged PR, so nothing - # in THIS branch can make the row green. The Install step is therefore - # continue-on-error and the step below pins that exact failure: the job stays - # required, so a DIFFERENT failure is still red, and the pin turns into a hard - # error the moment the installer starts picking an x64 Python. + # published a win_arm64 wheel at any version (nor have pyarrow and hf-transfer, + # which datasets pulls in), so the PyTorch step could not resolve and + # install.ps1 stopped at "Failed to install PyTorch". Pinned until #7549, which + # makes the installer prefer an x64 interpreter on an ARM64 host and bootstrap + # one when only ARM64 is installed (install.ps1:1160-1253, 1335-1353); x64 + # wheels run fine emulated. The row now gates, and the assert below checks the + # outcome that fix has to produce rather than the log line announcing it. - os: windows-11-arm winget: 'visible' experimental: false @@ -1100,9 +1198,6 @@ jobs: - name: Install id: install shell: pwsh - # ARM64 and winget=masked only: both failures are pinned below rather than - # gating (see the matrix). - continue-on-error: ${{ matrix.os == 'windows-11-arm' || matrix.winget == 'masked' }} env: # Empty, and therefore ignored by install.ps1, on the non-overlay legs. UNSLOTH_CI_SOURCE_OVERLAY: ${{ matrix.overlay && inputs.installer_source != 'published' && github.workspace || '' }} @@ -1132,99 +1227,89 @@ jobs: Write-Host "installer exit code: $rc" exit $rc - # KNOWN OUTCOME PIN, retire when #7549 merges. continue-on-error on Install would - # otherwise tolerate a bootstrap outage or an unrelated early exit exactly like - # the intended diagnostic, so every branch here that is not the pinned failure - # exits 1 and fails the (required) job. - - name: Assert the windows-11-arm outcome is a known one - if: always() && matrix.os == 'windows-11-arm' && steps.install.outcome != 'skipped' + # Windows asserted nothing about the install ITSELF: nobuild and the toolchain + # check only read the log, so an installer that exited 0 having produced nothing + # satisfied both. The Linux legs have had this since they stopped being pinned; + # these rows needed it more, because two of them are only just off a pin. + - name: Assert the install is actually usable shell: pwsh run: | - # The flip condition. #7549 makes install.ps1 prefer an x64 CPython on ARM64 - # hosts and torchaudio does publish win_amd64, so an AMD64 venv interpreter - # means the fix landed. Asked of the interpreter (sysconfig), not inferred from - # PROCESSOR_ARCHITECTURE, which describes the shell. The pinned PyTorch failure - # comes after the venv exists, so a missing interpreter is a different failure. - $venvPy = Join-Path $env:UNSLOTH_STUDIO_HOME 'unsloth_studio\Scripts\python.exe' - if (-not (Test-Path -LiteralPath $venvPy)) { - Write-Host "::error::no venv interpreter at $venvPy; the run did not even reach the pinned PyTorch failure" + $venv = Join-Path $env:UNSLOTH_STUDIO_HOME 'unsloth_studio' + $py = Join-Path $venv 'Scripts\python.exe' + if (-not (Test-Path -LiteralPath $py)) { + Write-Host "::error::installer exited 0 but left no managed Python at $py" + Get-ChildItem -LiteralPath $env:UNSLOTH_STUDIO_HOME -ErrorAction SilentlyContinue | + Format-Table | Out-String | Write-Host exit 1 } + & $py -V + $cli = Join-Path $venv 'Scripts\unsloth.exe' + if (-not (Test-Path -LiteralPath $cli)) { + Write-Host "::error::installer exited 0 but left no unsloth CLI at $cli" + exit 1 + } + # Present is not the same as runnable: the console script imports the whole + # command tree, so a missing dependency or an unimportable extension surfaces + # here and nowhere else. --version is the one subcommand-free path. + & $cli --version + if ($LASTEXITCODE -ne 0) { + Write-Host '::error::the unsloth CLI is on disk but does not run' + exit 1 + } + + # What #7549 has to produce on this host, checked as an outcome rather than as the + # log line announcing it. torchaudio, pyarrow and hf-transfer publish no win_arm64 + # wheel at any version, so a native ARM64 interpreter cannot resolve the stack; + # the installer's answer is to prefer, and if necessary bootstrap, an x64 CPython + # and let it run emulated. Asked of the interpreter through sysconfig, not inferred + # from PROCESSOR_ARCHITECTURE, which describes the shell rather than the venv. + - name: Assert the ARM64 host installed against an x64 interpreter + if: matrix.os == 'windows-11-arm' + shell: pwsh + run: | + $venvPy = Join-Path $env:UNSLOTH_STUDIO_HOME 'unsloth_studio\Scripts\python.exe' $tag = (& $venvPy -c "import sysconfig; print(sysconfig.get_platform())" 2>&1 | Out-String).Trim() $global:LASTEXITCODE = 0 Write-Host "venv interpreter platform: $tag" - if ($tag -ne 'win-arm64') { - Write-Host "::error::the installer selected a '$tag' interpreter on this ARM64 host, so #7549 has landed. Delete this pin, drop continue-on-error from the Install step, and let the leg gate normally." + if ($tag -ne 'win-amd64') { + Write-Host "::error::the venv was built from a '$tag' interpreter, so the x64 preference on ARM64 hosts has regressed and the missing win_arm64 wheels are back" exit 1 } - if ('${{ steps.install.outcome }}' -eq 'success') { - Write-Host '::error::the ARM64 leg installed successfully; the pinned failure is gone, so delete this pin and drop continue-on-error from the Install step' + # The package that has never shipped a win_arm64 wheel, so its presence is what + # proves the emulated x64 stack really resolved rather than being skipped. + # Metadata, not an import: this asserts resolution, and the import is the job of + # the torch assert below. + & $venvPy -c "from importlib.metadata import version; print('torchaudio', version('torchaudio'))" + if ($LASTEXITCODE -ne 0) { + Write-Host '::error::torchaudio is not installed, so the x64 interpreter did not buy the wheels it was chosen for' exit 1 } - if (-not (Test-Path logs/install.log)) { - Write-Host '::error::the ARM64 leg produced no install log' - exit 1 - } - $log = Get-Content logs/install.log -Raw - # All three, so a failure anywhere else in the installer is still red: it must - # be the PyTorch step, it must be about torchaudio, and it must be the missing - # win_arm64 platform tag rather than (say) a network error. - $atTorchStep = $log -match 'Failed to install PyTorch \(exit code' - $isTorchaudio = $log -match 'versions of torchaudio are available' - $isNoArmWheel = $log -match 'matching platform tag \(e\.g\., `win_arm64`\)' - Write-Host "PyTorch step: $atTorchStep / torchaudio: $isTorchaudio / no win_arm64 wheel: $isNoArmWheel" - if (-not ($atTorchStep -and $isTorchaudio -and $isNoArmWheel)) { - Write-Host '::error::the ARM64 leg did not fail at the pinned "torchaudio has no win_arm64 wheel" resolution error out of the PyTorch step; this is a new failure' - exit 1 - } - Write-Host '::notice::known outcome: a native ARM64 CPython plus no win_arm64 torchaudio wheel. Fixed by #7549 (prefer an x64 interpreter on ARM64 hosts); nothing in this branch can change it.' - # KNOWN OUTCOME PIN, retire when #7549 merges. Same reason as the ARM64 pin above: - # continue-on-error on Install would otherwise tolerate a bootstrap outage or an - # unrelated early exit exactly like the intended diagnostic, so every branch here - # that is not the pinned failure exits 1 and fails the (required) job. - - name: Assert the winget=masked outcome is a known one - if: always() && matrix.winget == 'masked' && steps.install.outcome != 'skipped' + # The other half of what #7549 has to produce. This row is the only place the + # relaxed git gate matters: winget is masked, so there is no way to fetch git at + # all, and setup.ps1 used to refuse to continue without it. Assert the relaxed + # branch was taken, so the row cannot go green because git leaked back onto PATH + # and the gate was never reached. + - name: Assert the no-winget path installed without git + if: matrix.winget == 'masked' shell: pwsh run: | - # The flip condition. studio/setup.ps1:1655-1669 gates on git unconditionally - # and can only get it from winget, so masking winget leaves no way to satisfy - # it; #7549 relaxes the gate to the --local and source-build paths that really - # use git, so that wording appearing means the fix landed. Read from the - # checkout because overlay: true makes the branch's setup.ps1 the one that runs - # (install.ps1:2626-2642); a published dispatch takes it from the released - # wheel instead, where only the outcome checks below can retire the pin. - if ('${{ inputs.installer_source }}' -ne 'published') { - $relaxed = Select-String -Path studio/setup.ps1 -SimpleMatch -Quiet ` - -Pattern 'Git is required for --local and llama.cpp source-build installs' - if ($relaxed) { - Write-Host '::error::studio/setup.ps1 now carries the relaxed #7549 git gate; delete this pin and drop winget=masked from continue-on-error on the Install step so this row gates.' - exit 1 - } - } - if ('${{ steps.install.outcome }}' -eq 'success') { - Write-Host '::error::the masked leg installed successfully; the pinned failure is gone, so delete this pin and drop winget=masked from continue-on-error on the Install step' - exit 1 - } - if (-not (Test-Path logs/install.log)) { - Write-Host '::error::the masked leg produced no install log' - exit 1 - } $log = Get-Content logs/install.log -Raw - # All three, so a failure anywhere else is still red: winget really was absent, - # the git gate fired, and it is what stopped studio setup rather than a warning - # the install walked past. The winget check is not redundant with the matrix -- - # without it the pin would absorb a leg whose PATH scrub failed and which then - # died on the same gate for a different reason. - $noWinget = $log -match 'will require Python \+ uv to be already installed' - $gitGate = $log -match 'Git is required but could not be installed automatically' - $setupRc = $log -match 'unsloth studio setup failed \(exit code 1\)' - Write-Host "no winget: $noWinget / git gate: $gitGate / setup exit 1: $setupRc" - if (-not ($noWinget -and $gitGate -and $setupRc)) { - Write-Host '::error::the masked leg did not stop at the winget-only git gate in studio/setup.ps1; this is a new failure' + if ($log -match 'Git is required but could not be installed automatically') { + Write-Host '::error::setup.ps1 stopped at the unconditional git gate; #7549 relaxed it to --local and source-build installs, so that has regressed' exit 1 } - Write-Host '::notice::known outcome: setup.ps1 gates on git unconditionally and can only fetch it through winget, which this row masks. Fixed by #7549 (relax the gate to --local and source-build installs); nothing in this branch can change it.' + if (-not ($log -match 'Git not found -- attempting install via winget')) { + Write-Host '::error::setup.ps1 found git on this machine, so the scrub leaked it back and this row never exercised the no-git path' + exit 1 + } + # setup.ps1:1757-1758, the non-fatal branch: git absent, nothing on the consumer + # path needs it, install continues. + if (-not ($log -match 'so git is not needed')) { + Write-Host '::error::setup.ps1 never reported git as absent-but-not-required; the relaxed gate did not run' + exit 1 + } + Write-Host 'no winget, no git, and the install completed anyway' # See the macOS job: proves the leg is testing what its matrix row claims. - name: Assert this ref's Python was really put under test @@ -1305,9 +1390,11 @@ jobs: # short-circuits. Record that, then assert what CAN fail. $sys32 = Join-Path $env:WINDIR 'System32\vcruntime140_1.dll' Write-Host "preinstalled System32 vcruntime140_1.dll: $(Test-Path $sys32)" + # The managed interpreter, with no fallback to whatever `python` resolves to: + # the usability assert above already hard-fails when it is missing, and a + # fallback would answer this question with an interpreter the install did not + # create. $py = Join-Path $env:UNSLOTH_STUDIO_HOME 'unsloth_studio\Scripts\python.exe' - if (-not (Test-Path $py)) { $py = (Get-Command python -ErrorAction SilentlyContinue).Source } - if (-not $py) { Write-Host '::error::no python from the install'; exit 1 } & $py -c "import ctypes.util, sys; print('VCRUNTIME140:', ctypes.util.find_library('vcruntime140'))" & $py -c "import torch; print('torch', torch.__version__)" if ($LASTEXITCODE -ne 0) { Write-Host '::error::torch failed to import (VC++ runtime missing?)'; exit 1 } @@ -1433,7 +1520,10 @@ jobs: fail-fast: false matrix: include: - # The consumer path: install.ps1 from this ref, unsloth from PyPI. + # The consumer path: install.ps1 from this ref, unsloth from PyPI. So + # studio/setup.ps1 comes out of the RELEASED wheel, which is why this row and + # the overlay one below do not currently reach the same place: see the pin on + # the Install step. - overlay: false # This ref's studio/setup.ps1 and install_python_stack.py, via # UNSLOTH_CI_SOURCE_OVERLAY (install.ps1:2643). Without it a branch changing @@ -1504,14 +1594,18 @@ jobs: - name: Install into the virgin container id: install shell: pwsh - # KNOWN OUTCOME PIN, retire when #7549 merges. studio/setup.ps1 on this ref and - # in the released wheel both hard-stop on a winget-only git gate and reach for - # winget again for the VC++ runtime, and a Server Core container has no - # Microsoft Store and therefore no winget, ever. #7549 is what relaxes both, and - # it is not in this branch, so no change here can make either row pass. The job - # stays required and the step below decides: only the pinned signature is - # tolerated, and it errors out once the install starts working. - continue-on-error: true + # RELEASE-LAG PIN, overlay=false only. A Server Core container has no Microsoft + # Store and therefore no winget, ever, and studio/setup.ps1 used to hard-stop on + # a winget-only git gate and reach for winget again for the VC++ runtime. #7549 + # relaxed both, and this branch has it -- but the released wheel does not: the + # setup.ps1 inside unsloth 2026.7.5 (uploaded 2026-07-23, and #7549 landed on the + # 28th) still carries the old gate, so the row that deliberately installs from + # PyPI still cannot get past it. That is release lag, not a product gap, and + # nothing in this branch can change it; the overlay row runs the same install + # against this ref's setup.ps1 and gates unconditionally. The step below accepts + # only that exact signature and hard-errors the moment the released wheel catches + # up. + continue-on-error: ${{ !matrix.overlay }} run: | $overlayArg = if ('${{ matrix.overlay }}' -eq 'true') { 'C:\ci' } else { '' } docker exec virgin powershell.exe -NoLogo -NoProfile -NonInteractive ` @@ -1519,12 +1613,68 @@ jobs: -Overlay "$overlayArg" *>&1 | Tee-Object -FilePath logs/install-outer.log exit $LASTEXITCODE - - name: Assert the container install outcome is a known one - if: always() && steps.install.outcome != 'skipped' + # The overlay row runs this ref's studio/setup.ps1, so it carries #7549 and has + # to install end to end. The in-container harness already asserts the venv + # interpreter, the unsloth CLI, `import torch`, the no-winget path, the overlay + # marker and nobuild, and exits 1 listing every failure -- so the Install step + # gating is most of the assertion. What is added here is the part this lane alone + # can prove. + - name: Assert the virgin container install proved what this lane exists for + if: matrix.overlay + shell: pwsh + run: | + $log = Get-Content logs/install-outer.log -Raw + # A `docker exec` that lost its container also exits 0, so read the harness's + # own verdict rather than trusting the exit code alone. + if (-not ($log -match 'VIRGIN WINDOWS CONTAINER INSTALL PASSED')) { + Write-Host '::error::the install step exited 0 but the in-container harness never printed its passing verdict' + exit 1 + } + # The overlay hook is this PR's own feature and gates unconditionally: without + # it this row would be indistinguishable from the released-wheel one. + if (-not ($log -match 'CI: overlaying source checkout')) { + Write-Host '::error::the overlay row never overlaid the checkout, so it only tested the released package' + exit 1 + } + # Git: no Store, no winget, no git, and nothing on the consumer path needs it. + # The relaxed gate is the only reason this row gets past setup.ps1 at all. + if ($log -match 'Git is required but could not be installed automatically') { + Write-Host '::error::studio/setup.ps1 stopped at the unconditional git gate; the relax to --local and llama.cpp source-build installs has regressed' + exit 1 + } + if (-not ($log -match 'so git is not needed')) { + Write-Host '::error::setup.ps1 never reported git as absent-but-not-required, so this container was not gitless and the relaxed gate went untested' + exit 1 + } + # VC++: this container is the ONLY environment in the workflow whose System32 + # does not already ship the 2015-2022 runtime (the hosted Windows legs cannot + # remove it without breaking the runner), so it is the only place the direct + # aka.ms download can be proved to run rather than be short-circuited by + # Test-VCRedistInstalled. Both halves: the fallback was taken, and it worked. + if (-not ($log -match 'downloading the runtime directly')) { + Write-Host '::error::Ensure-VCRedist never took the direct-download fallback, so a container with no VC++ runtime and no winget did not exercise it' + exit 1 + } + if ($log -match 'Could not install the VC\+\+ Redistributable automatically') { + Write-Host '::error::the direct VC++ runtime download ran but left the runtime uninstalled' + exit 1 + } + # The harness already ran `import torch` against the managed interpreter, which + # is what actually needs VCRUNTIME140_1.dll; this is the announcement that the + # DLL got there rather than having been there all along. + Write-Host '::notice::no Store, no winget, no git and no preinstalled VC++ runtime, and the install completed anyway' + + # RELEASE-LAG PIN (overlay=false). See the Install step: this row installs unsloth + # from PyPI on purpose, and the released setup.ps1 predates #7549. continue-on-error + # would otherwise tolerate a bootstrap outage or an unrelated early exit exactly + # like the intended diagnostic, so every branch here that is not the pinned failure + # exits 1 and fails the (required) job. + - name: Assert the released-wheel row failed only on release lag + if: always() && !matrix.overlay && steps.install.outcome != 'skipped' shell: pwsh run: | if ('${{ steps.install.outcome }}' -eq 'success') { - Write-Host '::error::a virgin Windows container now installs, so #7549 has landed. Delete this pin and drop continue-on-error from the Install step so this row gates.' + Write-Host '::error::the released wheel now installs in a virgin container, so it carries the relaxed #7549 gates. Delete this pin and drop continue-on-error from the Install step so this row gates.' exit 1 } if (-not (Test-Path logs/install-outer.log)) { @@ -1532,30 +1682,24 @@ jobs: exit 1 } $log = Get-Content logs/install-outer.log -Raw - # The overlay hook is this PR's own feature and gates unconditionally: without - # this the overlay row would be indistinguishable from the released-wheel row, - # since on this ref both stop at the same gate. - if ('${{ matrix.overlay }}' -eq 'true' -and -not ($log -match 'CI: overlaying source checkout')) { - Write-Host '::error::the overlay row never overlaid the checkout, so it only tested the released package' - exit 1 - } - # The two winget-only gates this lane exists to surface. Anything else is a new - # problem and must be red. + # The pinned signature is the OLD gate wording, which #7549 deleted. Its + # disappearance from a released wheel is the flip condition, and until then a + # failure anywhere else has to be red. $gitGate = $log -match 'Git is required but could not be installed automatically' $vcGate = $log -match 'torch failed to import' if (-not ($gitGate -or $vcGate)) { - Write-Host '::error::the container install failed at neither the winget-only git gate nor the missing VC++ runtime; this is a new failure' + Write-Host '::error::the container install failed at neither the released winget-only git gate nor the missing VC++ runtime; this is a new failure' exit 1 } - # `-or` on its own is too generous. virgin-windows-install.ps1:97 runs the - # torch assertion whenever the venv interpreter exists, whatever the installer - # did, and this image has no VC++ runtime, so ANY failure after venv creation - # -- a Node download, a setup step, a bad prebuilt -- arrives here carrying the + # `-or` on its own is too generous. virgin-windows-install.ps1:97 runs the torch + # assertion whenever the venv interpreter exists, whatever the installer did, + # and this image has no VC++ runtime, so ANY failure after venv creation -- a + # Node download, a setup step, a bad prebuilt -- arrives here carrying the # $vcGate text and was accepted as the pinned outcome. Enumerate what the - # harness actually recorded instead: it prints one `::error::` per - # entry of its $failures list (that script:151), and every one has to be a - # pinned gate. Anchored, because it also dumps the install log tail indented - # two spaces and those copies must not count. + # harness actually recorded instead: it prints one `::error::` per entry + # of its $failures list (that script:151), and every one has to be a pinned + # gate. Anchored, because it also dumps the install log tail indented two spaces + # and those copies must not count. $recorded = @(Get-Content logs/install-outer.log | ForEach-Object { if ($_ -match '^::error::(.+)$') { $Matches[1].Trim() } }) Write-Host "recorded failures: $($recorded.Count)" @@ -1576,11 +1720,10 @@ jobs: # that also exits 1 is indistinguishable from the pinned one. if (($recorded | Where-Object { $_ -like 'installer exited*' }) -and -not ($gitGate -and ($log -match 'unsloth studio setup failed \(exit code 1\)'))) { - Write-Host '::error::the installer exited non-zero somewhere other than the winget-only git gate in studio/setup.ps1; this is a new failure' + Write-Host '::error::the installer exited non-zero somewhere other than the winget-only git gate in the released studio/setup.ps1; this is a new failure' exit 1 } - if ($gitGate) { Write-Host '::notice::known outcome: winget-only git gate (studio/setup.ps1), fixed by #7549' } - if ($vcGate) { Write-Host '::notice::known outcome: no VC++ runtime and Ensure-VCRedist is winget-only, fixed by #7549' } + Write-Host '::notice::known outcome: the released wheel still carries the winget-only git gate and the winget-only Ensure-VCRedist that #7549 replaced. Retire this pin with the next release.' - name: Recover the install log from the container if: always() From f09a1e71a961ab008bcc721b42f1f9ea84620965 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Wed, 29 Jul 2026 06:33:16 +0000 Subject: [PATCH 36/36] Tighten the clean-machine CI comments Comments only, no assertion logic, pins or leg definitions touched. Reflowed every rationale block to denser wording and removed the duplication that had built up across repeated steps: the desktop workflow repeated the fork-PR skip, the desktop-v* tag resolution and the restore-runner note once per platform, and the installer workflow repeated its path-filter rationale in both the pull_request and push blocks. Those now point at the first copy. Every WHY is kept: why the masked legs avoid install.sh --local, what UNSLOTH_CI_SOURCE_OVERLAY is for, why `absent` tests "must not work" rather than command -v, why the .venv_t5_* sidecars are in the macho scan scope, why the signature check is main-executables-only, why each nobuild allowlist entry is a pure-Python sdist, why the WSL job gates and what the pipe truncation was, and why the virgin container's overlay=false row is still pinned. Proved comments-only three ways: both workflow revisions parsed with yaml.safe_load_all and every leaf walked (only `run:` scalars differ); every changed bash body and .sh compared byte-for-byte after `bash --pretty-print -n`; every changed pwsh body and .ps1 compared as a token stream with Comment and NewLine tokens dropped. A negative control injecting one non-comment line into each layer makes all of them fail. --- .github/scripts/assert-nobuild.ps1 | 22 +- .github/scripts/clean-machine-assert.sh | 81 +- .github/scripts/clean-machine-env.sh | 22 +- .github/scripts/ensure-docker-daemon.ps1 | 18 +- .github/scripts/virgin-windows-install.ps1 | 42 +- .github/scripts/virgin-windows-probe.ps1 | 56 +- .../workflows/clean-machine-install-ci.yml | 720 +++++++++--------- .../desktop-app-clean-machine-ci.yml | 273 +++---- install.ps1 | 27 +- install.sh | 23 +- studio/install_python_stack.py | 16 +- 11 files changed, 620 insertions(+), 680 deletions(-) diff --git a/.github/scripts/assert-nobuild.ps1 b/.github/scripts/assert-nobuild.ps1 index c9937bfd95..7ac86cb5d2 100644 --- a/.github/scripts/assert-nobuild.ps1 +++ b/.github/scripts/assert-nobuild.ps1 @@ -1,12 +1,11 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -# The `nobuild` contract from clean-machine-assert.sh, for Windows. -# -# A port and not `shell: bash`: the clean-machine scrub drops every `*\Git\*` PATH -# entry, and the bash version needs sed/grep/tr/sort out of Git's usr/bin. It also runs -# inside the servercore container, which has no bash at all. Both Windows lanes call -# this one file so the sdist allowlist cannot drift. +# The `nobuild` contract from clean-machine-assert.sh, for Windows. A port and not +# `shell: bash`: the scrub drops every `*\Git\*` PATH entry and the bash version needs +# sed/grep/tr/sort out of Git's usr/bin, and it also runs inside the servercore +# container, which has no bash. Both Windows lanes call this one file so the sdist +# allowlist cannot drift. # # Usage: assert-nobuild.ps1 -LogPath logs/install.log (exit 1 = a source build) [CmdletBinding()] @@ -17,9 +16,10 @@ if (-not (Test-Path -LiteralPath $LogPath)) { exit 1 } -# "Built an sdist" is NOT "needed a compiler": every name here has a -# setuptools.build_meta backend, no ext_modules and no .c/.cpp/.pyx/.rs file, so its -# PEP 517 build is a pure-Python copy step. Identical to clean-machine-assert.sh. +# "Built an sdist" is NOT "needed a compiler": every name here was verified against its +# own sdist -- setuptools.build_meta backend, no ext_modules, no .c/.cpp/.pyx/.rs file +# -- so its PEP 517 build is a pure-Python copy step. Identical to +# clean-machine-assert.sh, which carries the per-name rationale. $allow = @('openai-whisper', 'argbind', 'randomname', 'antlr4-python3-runtime', 'triton-kernels') if ($env:UNSLOTH_ALLOW_SDIST) { $allow += ($env:UNSLOTH_ALLOW_SDIST -split '\s+' | Where-Object { $_ }) @@ -34,8 +34,8 @@ $esc = [char]27 $text = (Get-Content -LiteralPath $LogPath -Raw) -replace "$esc\[[0-9;]*[A-Za-z]", '' $built = @() foreach ($line in ($text -split "`r?`n")) { - # A local-path build is something the caller pointed at (the CI source overlay), - # never something resolution chose; index dependencies always print `==`. + # A local-path build is one the caller pointed at (the CI source overlay), never + # one resolution chose; index dependencies always print `==`. if ($line -imatch 'building [a-z0-9._-]+ @ file://') { continue } # pip prints `Building wheel for `, uv prints `Building ==` # (astral-sh/uv#11165); the `==` or ` @ ` requirement keeps this off the diff --git a/.github/scripts/clean-machine-assert.sh b/.github/scripts/clean-machine-assert.sh index 37b9604f7c..dc528e480d 100755 --- a/.github/scripts/clean-machine-assert.sh +++ b/.github/scripts/clean-machine-assert.sh @@ -5,13 +5,12 @@ # Assert the clean-machine contract after an install attempt. # # absent The toolchain really was absent for the whole run. Catches a leg that -# "passed" only because masking silently failed, or because the installer +# "passed" because masking silently failed, or because the installer # quietly installed Xcode CLT behind our back. # notools The trace recorded no compiler/git/brew invocation (trace mode). -# nobuild The wheels-only contract: no "Building wheel" from pip, no -# "Building ==" from uv. Needs UNSLOTH_VERBOSE=1, or -# run_install_cmd (install.sh:193-243) discards uv's output on success -# and there is nothing to read. +# nobuild Wheels-only: no "Building wheel" from pip, no "Building ==" +# from uv. Needs UNSLOTH_VERBOSE=1, or run_install_cmd +# (install.sh:193-243) discards uv's output on success. # macho Every Mach-O under $MACHO_ROOT is the host architecture, and every # Mach-O MAIN EXECUTABLE is signed. Closes the Rosetta 2 gap, the one # divergence masking cannot reproduce. @@ -40,9 +39,9 @@ for check in "$@"; do for tool in git cc clang cmake; do command -v "$tool" >/dev/null 2>&1 || { ok "$tool not on PATH"; continue; } if "$tool" --version >/dev/null 2>&1; then - # On Intel runners /usr/bin/git is not CLT-provided and survives their - # removal, so no masking can take it away. cc and clang do become stubs and - # the macOS consumer path needs no git, so report rather than fail. + # On Intel runners /usr/bin/git is not CLT-provided, so no masking can take + # it away. cc and clang do become stubs and the macOS consumer path needs + # no git, so report rather than fail. case " ${UNSLOTH_CLEAN_ALLOW_WORKING:-} " in *" $tool "*) echo "[assert] NOTE $tool still works ($(command -v "$tool")); allowed on this runner" @@ -74,7 +73,7 @@ for check in "$@"; do [ -n "$tool" ] || continue case " $allow " in *" $tool "*) continue ;; esac # `xcode-select -p` only ASKS whether a toolchain is selected, and the fix - # is that the installer carries on without one, so the question is not USE. + # is that the installer carries on without one, so it is not USE. # `--install`, which pops the CLT installer, stays a hit. if [ "$tool" = "xcode-select" ]; then case "$rest" in @@ -93,17 +92,16 @@ for check in "$@"; do ;; nobuild) - # "Built an sdist" is NOT "needed a compiler". Every name below was checked + # "Built an sdist" is NOT "needed a compiler". Every name below was verified # against its own sdist: setuptools.build_meta backend, no ext_modules, no # .c/.cpp/.pyx/.rs file, so its PEP 517 build is a pure-Python copy step. # openai-whisper, argbind, randomname -- no version ever ships a wheel # antlr4-python3-runtime==4.9.3 -- pinned below the 4.13.2 wheel - # triton-kernels -- requirements/triton-kernels.txt pins a git URL under - # the triton repo's python/triton_kernels subdirectory: 75 Python files, - # a four-line pyproject.toml, no setup.py, kernels compiled at runtime. - # A direct URL the installer names itself, not something resolution - # chose, and only the Linux legs reach it (install_python_stack.py skips - # the step on Windows and macOS). + # triton-kernels -- requirements/triton-kernels.txt pins a git URL under the + # triton repo's python/triton_kernels subdir: 75 Python files, a four-line + # pyproject.toml, no setup.py, kernels compiled at runtime. A direct URL the + # installer names itself, not something resolution chose, and only the Linux + # legs reach it (install_python_stack.py skips it on Windows and macOS). # UNSLOTH_ALLOW_SDIST extends the allowlist. # # Lowercased and underscore-folded on both sides: a distribution name and the @@ -116,10 +114,10 @@ for check in "$@"; do # ` (astral-sh/uv#11165), so match both; the `==` or ` @ ` requirement # keeps this off the installer's own lowercase "building frontend..." text, and # ANSI is stripped first so a coloured run (FORCE_COLOR) parses. - # `Building @ file://...` is dropped: a local-path build is something the - # caller pointed at (--local, or the editable overlay), never something - # resolution chose. Index dependencies always print `==`, so a - # genuine PyPI sdist is still caught, including one named unsloth. + # `Building @ file://...` is dropped: a local-path build is one the + # caller pointed at (--local, or the editable overlay), never one resolution + # chose. Index dependencies always print `==`, so a genuine + # PyPI sdist is still caught, including one named unsloth. _esc=$(printf '\033') _built="$(sed -E "s/${_esc}\[[0-9;]*[A-Za-z]//g" "$LOG" 2>/dev/null \ | grep -viE "building [a-z0-9._-]+ @ file://" \ @@ -154,10 +152,10 @@ for check in "$@"; do # xcrun shim and gone after masking, so read `file -b`, keyed off `uname -m` # (macos-15-intel is x86_64). # - # SCOPE: all of $MACHO_ROOT, including the .venv_t5_510/_530/_550 sidecars. - # Those are payload, not scratch: setup.sh:579-581 creates them during a - # normal install and transformers_version.py:338-348 puts them on sys.path. - # Any exclusion must be a named path rule, never a narrowed find. + # SCOPE: all of $MACHO_ROOT, .venv_t5_510/_530/_550 sidecars included. Those are + # payload, not scratch: setup.sh:579-581 creates them during a normal install and + # transformers_version.py:338-348 puts them on sys.path. Any exclusion must be a + # named path rule, never a narrowed find. root="${MACHO_ROOT:-${UNSLOTH_STUDIO_HOME:-$HOME/.unsloth}}" want="$(uname -m)" [ "$want" = "aarch64" ] && want=arm64 @@ -165,18 +163,17 @@ for check in "$@"; do fail "macho requested but $root does not exist" else # SCOPE, part 2: the two payloads the install RUNS ON live outside $root. - # `uv venv` links /bin/python at its base interpreter rather than - # copying it, and the find below has no -L, so the interpreter that executed - # every install step is invisible to it; the uv that fetched it lands in - # $HOME/.local/bin. Both are exactly what Rosetta 2 hides -- an x86_64 uv or - # managed CPython runs green here and dies on the factory-fresh Mac this job - # stands in for. + # `uv venv` links /bin/python at its base interpreter rather than copying + # it, and the find below has no -L, so the interpreter that ran every install + # step is invisible to it; the uv that fetched it lands in $HOME/.local/bin. + # Both are exactly what Rosetta 2 hides: an x86_64 uv or managed CPython runs + # green here and dies on the factory-fresh Mac this job stands in for. _macho_targets() { find "$root" -type f \( -perm -u+x -o -name '*.dylib' -o -name '*.so' -o -name '*.node' \) 2>/dev/null # -L follows the interpreter symlink; -maxdepth keeps this a bin/ lookup and # not a second walk of site-packages through the venv's lib64 link. Depth 4 - # covers /unsloth_studio, the .venv_t5_* sidecars and the tauri - # layout's /studio/unsloth_studio. + # covers /unsloth_studio, the .venv_t5_* sidecars and the tauri layout's + # /studio/unsloth_studio. find -L "$root" -maxdepth 4 -type f -path '*/bin/python' 2>/dev/null for _uv in "$HOME/.local/bin/uv" "$(command -v uv 2>/dev/null || true)"; do [ -n "$_uv" ] && [ -f "$_uv" ] && printf '%s\n' "$_uv" @@ -195,11 +192,11 @@ for check in "$@"; do *) bad_arch="$bad_arch $f [$desc]" ;; esac - # Signature: MAIN EXECUTABLES ONLY. Asserting it for every Mach-O failed the + # Signature: MAIN EXECUTABLES ONLY. Asserting it on every Mach-O failed the # mask/pipe leg on 29 ordinary PyPI extension modules plus libportaudio.dylib: - # those are MH_BUNDLE/MH_DYLIB images dlopen'd without library validation and - # ship unsigned, and that run had already imported them with the installer - # exiting 0. macOS enforces on main executables and gatekept .app bundles. + # MH_BUNDLE/MH_DYLIB images dlopen'd without library validation, shipped + # unsigned, and that run had already imported them with the installer exiting + # 0. macOS enforces on main executables and gatekept .app bundles. # # Key off the filetype `file` reports, not the path: a .so may be a bundle or a # dylib, and an executable may have no extension. The library veto is second so @@ -218,14 +215,14 @@ for check in "$@"; do # ("Killed: 9"), while x86_64 execs it happily, so an unsigned x86_64 # payload is not the same defect. if [ "$want" = "arm64" ] && [ "$_is_exe" = 1 ]; then - # Ad-hoc counts as signed: arm64 linkers seal ad-hoc by default, so the - # test is "has a seal that verifies", not "has an identity". `spctl` and + # Ad-hoc counts as signed: arm64 linkers seal ad-hoc by default, so the test + # is "has a seal that verifies", not "has an identity". `spctl` and # `--strict` would demand an authority and reject ad-hoc. if ! codesign -v "$f" >/dev/null 2>&1; then # Nothing to verify and a seal that does not match mean different things. # Captured, not piped into grep: `codesign -dvv` exits non-zero on an - # unsigned file, and under `pipefail` that status is what the pipeline - # returns even on a match. + # unsigned file, and under `pipefail` that is the pipeline's status even + # on a match. _sig="$(codesign -dvv "$f" 2>&1 || true)" case "$_sig" in *"not signed at all"*) unsigned="$unsigned $f" ;; @@ -235,8 +232,8 @@ for check in "$@"; do fi done < <(_macho_targets | sort -u) if [ "$n" = "0" ]; then - # An empty scan reads exactly like a clean one, so the check would pass on a - # wrong root and prove nothing. + # An empty scan reads exactly like a clean one, so a wrong root would pass + # and prove nothing. fail "no Mach-O found under $root; the arch/signature assertion proved nothing" elif [ "$nout" = "0" ]; then # Same rule for the roots added above: install.sh always bootstraps uv into diff --git a/.github/scripts/clean-machine-env.sh b/.github/scripts/clean-machine-env.sh index 435c7f26a9..6890f2f31e 100755 --- a/.github/scripts/clean-machine-env.sh +++ b/.github/scripts/clean-machine-env.sh @@ -98,8 +98,8 @@ if [ "$MODE" = "mask" ]; then if [ "$REMOVE" = "1" ] && [ "$OS" = "Darwin" ]; then # Best effort, each step independent and recorded in restore.sh so an - # `if: always()` step can put the runner back. xcode_select_link is what - # `xcode-select -p` reads, so removing it reproduces a virgin Mac's gate; + # `if: always()` step can put the runner back. `xcode-select -p` reads + # xcode_select_link, so removing it reproduces a virgin Mac's gate; # `xcode-select --reset` is NOT enough, it can reselect a full Xcode.app. if [ -e /var/db/xcode_select_link ]; then if sudo rm -f /var/db/xcode_select_link 2>/dev/null; then @@ -120,9 +120,9 @@ if [ "$MODE" = "mask" ]; then fi fi # Xcode.app must go too: with the link removed AND CommandLineTools moved, - # `xcode-select -p` still does not fail, it falls through to the image's Xcode - # bundle (observed: /Applications/Xcode_16.4.app/Contents/Developer), which - # re-arms /usr/bin/{git,cc}. A rename is instant whatever the bundle size. + # `xcode-select -p` still succeeds, falling through to the image's Xcode bundle + # (observed: /Applications/Xcode_16.4.app/Contents/Developer), which re-arms + # /usr/bin/{git,cc}. A rename is instant whatever the bundle size. for app in /Applications/Xcode*.app; do [ -d "$app" ] || continue if sudo mv "$app" "${app}.masked" 2>/dev/null; then @@ -133,9 +133,9 @@ if [ "$MODE" = "mask" ]; then fi done # /usr/local EXISTS on a factory-fresh Mac: a SIP-exempt firmlink, and empty. What - # is absent is its CONTENTS, /usr/local/bin included. So empty it rather than - # remove it. Runs before the Homebrew block below so /usr/local/Homebrew is stashed - # once, with one restore line, in the right order. + # is absent is its CONTENTS, /usr/local/bin included, so empty it rather than remove + # it. Before the Homebrew block below, so /usr/local/Homebrew is stashed once, with + # one restore line, in the right order. if [ -d /usr/local ]; then STASH="$WORK/usr-local" mkdir -p "$STASH" @@ -175,9 +175,9 @@ if [ "$MODE" = "mask" ]; then if [ "$REMOVE" = "1" ] && [ "$OS" = "Linux" ]; then # A hosted Linux runner keeps git, gcc, cmake and make in /usr/bin, which the PATH - # scrub has to keep, so absence must be made real: move the resolved binaries - # aside (recorded in restore.sh). Versioned siblings like gcc-11 survive, but a - # consumer install invokes the unsuffixed names, which is what `absent` checks. + # scrub has to keep, so absence must be made real: move the resolved binaries aside + # (recorded in restore.sh). Versioned siblings like gcc-11 survive, but a consumer + # install invokes the unsuffixed names, which is what `absent` checks. for tool in $TOOLS; do # Repeat per tool: a runner can carry the same name in /usr/bin and # /usr/local/bin, and moving only the first leaves the second on PATH. diff --git a/.github/scripts/ensure-docker-daemon.ps1 b/.github/scripts/ensure-docker-daemon.ps1 index 8120f3f11f..50a2b5cc56 100644 --- a/.github/scripts/ensure-docker-daemon.ps1 +++ b/.github/scripts/ensure-docker-daemon.ps1 @@ -4,14 +4,12 @@ # Waits for the Windows Docker daemon on a hosted runner, starting the service if # it is installed but not running. # -# Docker is installed on every windows-2022 runner image (runner-images installs it -# via Microsoft's install-docker-ce.ps1, without -HyperV, so the daemon serves -# WINDOWS containers) but it is not always already RUNNING when a job starts. A -# spike run died 21 seconds in with +# Docker is installed on every windows-2022 image (runner-images uses Microsoft's +# install-docker-ce.ps1 without -HyperV, so the daemon serves WINDOWS containers) but +# is not always RUNNING when a job starts: a spike run died 21s in with # failed to connect to the docker API at npipe:////./pipe/docker_engine -# while a sibling job on a different runner was fine. Without this wait that flake -# reads as "Windows containers are not available on hosted runners", which is the -# wrong conclusion entirely. +# while a sibling job was fine. Without this wait that flake reads as "Windows +# containers are not available on hosted runners", the wrong conclusion entirely. [CmdletBinding()] param([int] $TimeoutMinutes = 5) @@ -36,8 +34,8 @@ while ($true) { Start-Sleep -Seconds 5 } -# The failing `docker info` probes leave $LASTEXITCODE non-zero, and the runner -# appends `exit $LASTEXITCODE` to every pwsh step (actions/runner#351), so without -# this reset a successful wait still fails the step. +# The failing `docker info` probes leave $LASTEXITCODE non-zero and the runner appends +# `exit $LASTEXITCODE` to every pwsh step (actions/runner#351), so without this reset a +# successful wait still fails the step. $global:LASTEXITCODE = 0 exit 0 diff --git a/.github/scripts/virgin-windows-install.ps1 b/.github/scripts/virgin-windows-install.ps1 index 2d7b7a6289..2c0821ca62 100644 --- a/.github/scripts/virgin-windows-install.ps1 +++ b/.github/scripts/virgin-windows-install.ps1 @@ -1,9 +1,9 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -# Runs INSIDE a Windows container, after virgin-windows-probe.ps1 has proved the -# environment has no toolchain. Runs install.ps1 the way a real user on a bare -# Windows box would, then asserts the same things the hosted Windows leg asserts. +# Runs INSIDE a Windows container, after virgin-windows-probe.ps1 has proved there is no +# toolchain: install.ps1 the way a real user on a bare Windows box runs it, then the same +# assertions the hosted Windows leg makes. [CmdletBinding()] param( @@ -20,16 +20,16 @@ function Section($t) { Write-Host ""; Write-Host "=== $t ===" } # ── Environment the installer needs to be non-interactive ───────────────────── Section 'install environment' # install.ps1:2885-2888 prompts `Start Unsloth Studio now? [Y/n]` when -# [Environment]::UserInteractive is true and stdin is not redirected. Both hold in a -# `docker exec` session, so without this the installer BLOCKS FOREVER on Read-Host -# and the job dies on timeout with no diagnosis. +# [Environment]::UserInteractive is true and stdin is not redirected -- both hold under +# `docker exec` -- so without this the installer BLOCKS FOREVER on Read-Host and the job +# dies on timeout with no diagnosis. $env:UNSLOTH_SKIP_AUTOSTART = '1' -# install.ps1:254/258 joins $env:USERPROFILE with no null guard. Setting the install -# root explicitly also keeps the container's state entirely under one directory. +# install.ps1:254/258 joins $env:USERPROFILE with no null guard. An explicit root also +# keeps the container's state under one directory. $env:UNSLOTH_STUDIO_HOME = 'C:\studio-home' $env:UNSLOTH_STUDIO_DISABLE_PUBLIC_CHECK = '1' -# Without this, uv's output is discarded on success and the nobuild check below can -# only ever report "built: none". +# Without this uv's output is discarded on success and the nobuild check below can only +# ever report "built: none". $env:UNSLOTH_VERBOSE = '1' if ($Overlay) { $env:UNSLOTH_CI_SOURCE_OVERLAY = $Overlay @@ -73,8 +73,7 @@ Section 'assert: the install produced something usable' if ($rc -ne 0) { $failures += "installer exited $rc" } else { - # Mirrors the Linux leg's "Assert the install is actually usable": an installer - # that exits 0 having done nothing must not pass. + # As the Linux leg: an installer that exits 0 having done nothing must not pass. if (-not (Test-Path -LiteralPath $venvPy)) { $failures += "installer exited 0 but left no managed Python at $venvPy" Get-ChildItem -Path $env:UNSLOTH_STUDIO_HOME -ErrorAction SilentlyContinue | Format-Table | Out-String | Write-Host @@ -89,11 +88,10 @@ if ($rc -ne 0) { } Section 'assert: torch imports' -# On the hosted runner this proves less than it looks like: the runner image ships -# the VC++ 2015-2022 runtime in System32, so Test-VCRedistInstalled (setup.ps1:875) -# short-circuits before it needs winget. THIS container is the first environment in -# which that is not true, so a failure here is a genuine finding about bare Windows, -# not a CI artefact. +# On the hosted runner this proves less than it looks: the image ships the VC++ +# 2015-2022 runtime in System32, so Test-VCRedistInstalled (setup.ps1:875) +# short-circuits before it needs winget. THIS container is the first environment where +# that is not true, so a failure here is a genuine finding about bare Windows. if (Test-Path -LiteralPath $venvPy) { foreach ($dll in 'vcruntime140.dll', 'vcruntime140_1.dll', 'msvcp140.dll') { $p = Join-Path $env:WINDIR "System32\$dll" @@ -111,9 +109,9 @@ if (Test-Path -LiteralPath $venvPy) { Section "assert: the installer took the no-winget path" if (Test-Path -LiteralPath $LogPath) { - # install.ps1:1098, the no-winget branch. A container has no Microsoft Store and - # therefore no App Installer, so this is the fallback path (python.org + astral.sh) - # under test -- the whole reason a container is a good harness. + # install.ps1:1098, the no-winget branch. A container has no Microsoft Store and so + # no App Installer, which puts the fallback path (python.org + astral.sh) under + # test -- the whole reason a container is a good harness. $noWinget = 'will require Python + uv to be already installed' if (Select-String -Path $LogPath -Pattern $noWinget -SimpleMatch -Quiet) { Write-Host "confirmed: installer reported winget as unavailable and used the fallback path" @@ -132,8 +130,8 @@ if ($Overlay -and $rc -eq 0) { } Section 'assert: no non-allowlisted source build' -# Shared with the hosted Windows legs so the sdist allowlist lives in one place; the -# script prints its own diagnosis, so only the verdict is folded in here. +# Shared with the hosted Windows legs so the sdist allowlist lives in one place; it +# prints its own diagnosis, so only the verdict is folded in here. $nobuild = Join-Path $PSScriptRoot 'assert-nobuild.ps1' if (-not (Test-Path -LiteralPath $nobuild)) { $failures += "assert-nobuild.ps1 is missing next to this script, so the no-build contract went unchecked" diff --git a/.github/scripts/virgin-windows-probe.ps1 b/.github/scripts/virgin-windows-probe.ps1 index 1993d84b96..a305d2dcae 100644 --- a/.github/scripts/virgin-windows-probe.ps1 +++ b/.github/scripts/virgin-windows-probe.ps1 @@ -1,14 +1,12 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. -# Runs INSIDE a Windows container. Proves the environment is genuinely virgin -# BEFORE anything is installed into it. -# -# This script is the entire point of the container lane. The hosted-runner Windows -# legs of clean-machine-install-ci.yml simulate absence (rename the toolcache Python -# directory, scrub the Machine and User registry PATH); this one asserts real -# absence on an OS image that never had a toolchain. If these assertions do not -# run, the lane proves nothing that the masked legs did not already prove. +# Runs INSIDE a Windows container, proving the environment is genuinely virgin BEFORE +# anything is installed into it. This is the entire point of the container lane: the +# hosted-runner Windows legs of clean-machine-install-ci.yml only simulate absence +# (rename the toolcache Python dir, scrub the Machine and User registry PATH), while +# this asserts real absence on an image that never had a toolchain. Without it the lane +# proves nothing the masked legs did not already prove. $ErrorActionPreference = 'Continue' $failures = @() @@ -45,10 +43,9 @@ Write-Host "USERPROFILE : $env:USERPROFILE" Write-Host "LOCALAPPDATA : $env:LOCALAPPDATA" Write-Host "PROCESSOR_ARCH : $env:PROCESSOR_ARCHITECTURE" -# install.ps1 line 254/258 does Join-Path $env:USERPROFILE ".unsloth\studio" with no -# null guard, so an unset USERPROFILE aborts under ErrorActionPreference=Stop. -# The lane sets UNSLOTH_STUDIO_HOME, but record whether a bare container would have -# survived without it. +# install.ps1:254/258 does Join-Path $env:USERPROFILE ".unsloth\studio" with no null +# guard, so an unset USERPROFILE aborts under ErrorActionPreference=Stop. The lane sets +# UNSLOTH_STUDIO_HOME, but record whether a bare container would have survived without. if ([string]::IsNullOrWhiteSpace($env:USERPROFILE)) { Write-Host "::warning::USERPROFILE is unset in this container; install.ps1's default install root would abort" } @@ -66,27 +63,26 @@ foreach ($t in $mustBeAbsent) { } Section 'informational: present but not a developer toolchain' -# These are OS components, not a toolchain. curl.exe and tar.exe ship in System32 on -# Server 2022 and are the only transport into a container with no git; naming them -# keeps the premise honest rather than silently relying on them. +# OS components, not a toolchain. curl.exe and tar.exe ship in System32 on Server 2022 +# and are the only transport into a container with no git; naming them keeps the +# premise honest rather than silently relying on them. foreach ($t in 'cmd', 'powershell', 'curl', 'tar', 'certutil', 'msiexec', 'reg', 'where', 'pwsh', 'node', 'npm', 'msbuild', 'dotnet', 'gcc') { $c = Get-Command $t -ErrorAction SilentlyContinue Write-Host (" {0,-10} {1}" -f $t, $(if ($c) { $c.Source } else { 'ABSENT' })) } Section 'virginity: no toolchain on disk either' -# A binary can be absent from PATH and still be found by uv's own interpreter -# discovery or by py.exe's registry view -- that is exactly how the hosted Windows -# leg once reported `python ABSENT` and then installed with the runner's 3.13.14. -# Check the disk and the registry, not just PATH. +# A binary can be off PATH and still be found by uv's interpreter discovery or py.exe's +# registry view -- exactly how the hosted Windows leg once reported `python ABSENT` and +# then installed with the runner's 3.13.14. So check disk and registry too. $badPaths = @( 'C:\Python27', 'C:\Python3*', 'C:\Program Files\Python*', 'C:\Program Files (x86)\Python*', 'C:\Program Files\Git', 'C:\Program Files\CMake', 'C:\Program Files\Microsoft Visual Studio', 'C:\Program Files (x86)\Microsoft Visual Studio', 'C:\hostedtoolcache', 'C:\ProgramData\chocolatey' ) foreach ($p in $badPaths) { - # Wildcards can match several directories; take the first so the message names a - # real path instead of stringifying an array. + # Wildcards can match several dirs; take the first so the message names a real + # path instead of stringifying an array. $hit = @(Get-Item -Path $p -ErrorAction SilentlyContinue) | Select-Object -First 1 if ($hit) { Write-Host " PRESENT $($hit.FullName)" @@ -115,12 +111,12 @@ foreach ($scope in 'Machine', 'User') { # ── The VC++ runtime question the hosted leg cannot answer ──────────────────── Section 'VC++ runtime (honest measurement)' -# clean-machine-install-ci.yml carries an explicit HONESTY NOTE that the hosted image -# ships the VC++ 2015-2022 runtime in System32 and it cannot be removed without -# breaking the runner, so `import torch` succeeding there does NOT prove a no-winget -# machine has the runtime. This container is the only environment in CI that can -# answer it, so their absence is asserted, not merely recorded: if a future base image -# starts shipping them the lane silently degrades into another masked leg. +# The hosted image ships the VC++ 2015-2022 runtime in System32 and cannot lose it +# without breaking the runner (see the HONESTY NOTE in clean-machine-install-ci.yml), +# so `import torch` succeeding there does NOT prove a no-winget machine has the +# runtime. This container is the only environment in CI that can answer it, so their +# absence is asserted, not merely recorded: if a future base image starts shipping +# them the lane silently degrades into another masked leg. foreach ($dll in 'vcruntime140.dll', 'vcruntime140_1.dll', 'msvcp140.dll') { $p = Join-Path $env:WINDIR "System32\$dll" $present = Test-Path $p @@ -135,9 +131,9 @@ foreach ($k in 'HKLM:\SOFTWARE\Microsoft\VisualStudio\14.0\VC\Runtimes\x64', # ── Can the installer's transport work at all here? ─────────────────────────── Section 'outbound HTTPS and TLS' -# install.ps1 never sets [Net.ServicePointManager]::SecurityProtocol, so it inherits -# the .NET Framework default. Test the DEFAULT first: if that fails and Tls12 works, -# the installer has a real portability bug on hardened images, not a container quirk. +# install.ps1 never sets [Net.ServicePointManager]::SecurityProtocol, so it inherits the +# .NET Framework default. Test the DEFAULT first: if that fails and Tls12 works, the +# installer has a real portability bug on hardened images, not a container quirk. Write-Host "default SecurityProtocol: $([Net.ServicePointManager]::SecurityProtocol)" $probeUrls = @( 'https://www.python.org/ftp/python/', diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index 166092aa27..010bd91d25 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -3,42 +3,37 @@ # Proves Unsloth installs on a machine that has never seen a developer toolchain. # -# Why this exists: studio-mac-install-matrix.yml runs `install.sh --local --no-torch` -# on runners with Xcode CLT selected AND actions/setup-python preinstalled, so the -# macOS dependency gate never fires there -- and `--local` is precisely the mode that +# Why: studio-mac-install-matrix.yml runs `install.sh --local --no-torch` on runners +# with Xcode CLT selected AND actions/setup-python preinstalled, so the macOS +# dependency gate never fires there -- and `--local` is precisely the mode that # legitimately needs git. A brand-new Mac hits a hard `exit 1` no CI job covered. # -# Hosted runners are developer machines, so each job simulates absence. Two modes, -# answering different questions (see .github/scripts/clean-machine-env.sh): +# Hosted runners are developer machines, so each job simulates absence, two ways (see +# .github/scripts/clean-machine-env.sh): # mask -> the toolchain is genuinely unusable; does the install still work? # trace -> the toolchain works but is logged; does the installer ever call it? # Linux is the exception: containers are genuinely clean. # -# ── What each leg actually puts under test ──────────────────────────────────── -# install.sh / install.ps1 come from this ref, but they install unsloth FROM PyPI -- -# the consumer path, which has to stay that way -- so everything Python-side would -# come out of the RELEASED wheel (setup.sh, setup.ps1, install_python_stack.py and -# every requirements/constraints file they reach via Path(__file__)), and a branch -# changing any of them would get a green run proving nothing about the change. +# ── What `overlay` decides ──────────────────────────────────────────────────── +# install.sh / install.ps1 come from this ref but install unsloth FROM PyPI, the +# consumer path, which has to stay that way -- so everything Python-side would be the +# RELEASED wheel's (setup.sh, setup.ps1, install_python_stack.py and every +# requirements/constraints file they reach via Path(__file__)) and a branch changing +# any of them would get a green run proving nothing. `overlay: true` legs therefore +# re-point the venv at this ref before studio setup, via UNSLOTH_CI_SOURCE_OVERLAY: a +# `--no-deps` editable install of the checkout, so `import studio` resolves to the +# working tree and the setup-script lookup finds this ref's setup.sh / setup.ps1. NOT +# `install.sh --local`, which also pulls `unsloth-zoo @ git+https://...` and so needs +# the git these legs remove; an editable overlay resolves and clones nothing. # -# `overlay: true` legs therefore re-point the venv at this ref before studio setup -# runs, via UNSLOTH_CI_SOURCE_OVERLAY: a `--no-deps` editable install of the -# checkout, so `import studio` resolves to the working tree and the setup-script -# lookup finds this ref's setup.sh / setup.ps1. Deliberately NOT `install.sh -# --local`, which also pulls `unsloth-zoo @ git+https://...` and so needs the git -# these legs remove; an editable overlay resolves and clones nothing. -# -# Legs left on `overlay: false`, and why: -# mac */mask/pipe the `curl | sh` shape a user runs. Kept end-to-end on the -# released package so a broken PyPI release still shows up. -# mac macos-14/trace `notools` asserts the installer never reaches for git, and -# the editable build itself calls `git rev-parse` / -# `git archive` through setuptools-scm's file finder, so an -# overlay would answer the leg's own question for it. -# linux ubuntu2404-nonroot-notransport dies at the elevation gate before a venv -# exists. -# wsl only install.sh is copied into the distro; there is no -# source tree inside WSL to overlay from. +# Legs left on `overlay: false`: +# mac */mask/pipe the `curl | sh` shape a user runs, kept end-to-end on the +# released package so a broken PyPI release still shows up. +# mac macos-14/trace `notools` asserts the installer never reaches for git, and the +# editable build calls `git rev-parse` / `git archive` itself via +# setuptools-scm's file finder, answering the leg's own question. +# linux ubuntu2404-nonroot-notransport dies at the elevation gate before a venv exists. +# wsl only install.sh is copied in; no source tree inside WSL. name: Clean machine install @@ -57,8 +52,8 @@ on: - 'studio/prebuilt_core.py' - 'studio/node_prebuilt_pins.json' # The overlay exists so a constraints or requirements change is exercised here - # (see the header). The update-smoke jobs cannot stand in: they start from a - # preinstalled Python and full developer tooling. + # (see the header). update-smoke cannot stand in: it starts from a preinstalled + # Python and full developer tooling. - 'studio/backend/requirements/**' - '.github/scripts/clean-machine-*.sh' - '.github/scripts/assert-llama-loads.sh' @@ -70,24 +65,19 @@ on: push: branches: [main] # Same list as the PR filter: without it a direct push to main touching any of - # these skipped the workflow, so the post-merge backstop never happened. + # these skipped the workflow and the post-merge backstop never happened. paths: - 'install.sh' - 'install.ps1' - 'studio/setup.sh' - 'studio/setup.ps1' - 'studio/install_python_stack.py' - # setup.sh (727) and setup.ps1 (2343, 3630, 3916) call these directly, and the - # overlay makes them THIS ref's code, so they decide whether a clean machine gets - # a native prebuilt or falls back to a toolchain-dependent path. Left off the - # list, a change to one of them skipped the only workflow that can see it. - 'studio/install_*_prebuilt.py' - 'studio/prebuilt_core.py' - 'studio/node_prebuilt_pins.json' - 'studio/backend/requirements/**' - '.github/scripts/clean-machine-*.sh' - '.github/scripts/assert-llama-loads.sh' - # The virgin Windows container lane lives in this workflow too. - '.github/scripts/virgin-windows-*.ps1' - '.github/scripts/ensure-docker-daemon.ps1' - '.github/scripts/assert-nobuild.ps1' @@ -112,8 +102,8 @@ env: UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home # No wildcard bind -> no ifconfig.me / check-host.net calls on the startup path. UNSLOTH_STUDIO_DISABLE_PUBLIC_CHECK: '1' - # Without this, run_install_cmd (install.sh:193-243) sends every `uv pip install` to - # a temp file and DELETES it on success, so `nobuild` can only report "built: none". + # Without this run_install_cmd (install.sh:193-243) sends every `uv pip install` to a + # temp file and DELETES it on success, so `nobuild` can only report "built: none". UNSLOTH_VERBOSE: '1' jobs: @@ -129,11 +119,11 @@ jobs: fail-fast: false matrix: include: - # `overlay` decides whether this ref's Python is put under test at all; see - # the header. The pipe legs stay on the released package on purpose. + # `overlay` decides whether this ref's Python is under test at all; see the + # header. The pipe legs stay on the released package on purpose. # - # The reported failure, in the shape users run it, with torch because that - # is what a consumer gets. + # The reported failure, in the shape users run it, with torch because that is + # what a consumer gets. - {os: macos-14, mode: mask, delivery: pipe, flags: '', experimental: false, overlay: false} - {os: macos-14, mode: mask, delivery: file, flags: '', experimental: false, overlay: true} # What the desktop app runs: no tty, stdin closed, TAURI markers on. @@ -159,7 +149,7 @@ jobs: persist-credentials: false # No actions/setup-python on purpose: install.sh must bring its own uv-managed - # CPython, exactly as it must on a user's machine. + # CPython, as it must on a user's machine. - name: Record the pre-masking toolchain run: | @@ -169,7 +159,7 @@ jobs: echo "brew : $(command -v brew || echo none)" echo "cmake : $(command -v cmake || echo none)" echo "python3 : $(command -v python3 || echo none)" - # Neither is documented for these images, and both change what a binary is + # Neither is documented for these images and both change what a binary is # allowed to do. One line settles it for anyone reading the artifact. echo "spctl --status : $(spctl --status 2>&1 || true)" echo "csrutil status : $(csrutil status 2>&1 || true)" @@ -195,10 +185,10 @@ jobs: if: matrix.mode == 'trace' run: | # `notools` reads an absence, so a shim dir that never reached PATH looks - # exactly like an installer that touched nothing, and the one leg carrying - # that assertion would pass whatever the installer did. Prove the wrapper - # records before trusting an empty file. macOS never probes git off the - # --local path, so the call has to be explicit. + # exactly like an installer that touched nothing and the one leg carrying that + # assertion would pass whatever the installer did. Prove the wrapper records + # before trusting an empty file. macOS never probes git off the --local path, + # so the call has to be explicit. set -a; . ./clean-machine.env; set +a [ -n "$UNSLOTH_TOOL_TRACE" ] || { echo "::error::trace mode set no UNSLOTH_TOOL_TRACE"; exit 1; } git --version >/dev/null 2>&1 || true @@ -218,18 +208,18 @@ jobs: HF_TOKEN: ${{ github.event_name != 'pull_request' && secrets.HF_TOKEN || '' }} # Empty, and so ignored by install.sh, on the non-overlay legs. Empty for # `installer_source: published` too: the script under test is then - # production's and has no such hook, and overlaying this ref's Python onto it - # would report on neither honestly. + # production's, which has no such hook, and overlaying this ref's Python onto + # it would report on neither honestly. UNSLOTH_CI_SOURCE_OVERLAY: ${{ matrix.overlay && inputs.installer_source != 'published' && github.workspace || '' }} run: | set -a; . ./clean-machine.env; set +a set -o pipefail rc=0 FLAGS="${{ matrix.flags }}" - # A consumer has no CI=true, no GITHUB_*, no RUNNER_*: branching on any of - # them is a hidden dependency nobody outside CI exercises. Scoped to the - # installer's own process, so $GITHUB_OUTPUT below still resolves. `case` - # rather than `sed`, whose BRE has no \| alternation on macOS. + # A consumer has no CI=true, no GITHUB_*, no RUNNER_*: branching on any of them + # is a hidden dependency nobody outside CI exercises. Scoped to the installer's + # own process, so $GITHUB_OUTPUT below still resolves. `case` rather than + # `sed`, whose BRE has no \| alternation on macOS. CLEAN_ENV="" for v in $(env | cut -d= -f1); do case "$v" in CI|GITHUB_*|RUNNER_*) CLEAN_ENV="$CLEAN_ENV -u $v" ;; esac @@ -249,22 +239,21 @@ jobs: fi case "${{ matrix.delivery }}" in file) - # Plain file execution isolates "installer logic broken" from - # "curl-pipe delivery broken". + # Isolates "installer logic broken" from "curl-pipe delivery broken". env $CLEAN_ENV bash "$SCRIPT" $FLAGS 2>&1 | tee logs/install.log || rc=$? ;; pipe) # The shape users actually run. install.sh is ~150KB of top-level - # statements, so an early `exit` leaves the writer with a closed pipe - # -> `curl: (56)`. Piping a local file reproduces that faithfully - # without depending on unsloth.ai being current. The published case - # re-fetches rather than piping $SCRIPT: the live transport is half of - # what this delivery tests. + # statements, so an early `exit` leaves the writer with a closed pipe -> + # `curl: (56)`; piping a local file reproduces that faithfully without + # depending on unsloth.ai being current. The published case re-fetches + # rather than piping $SCRIPT: the live transport is half of what this + # delivery tests. if [ "${{ inputs.installer_source }}" = "published" ]; then curl -fsSL https://unsloth.ai/install.sh | env $CLEAN_ENV sh -s -- $FLAGS 2>&1 | tee logs/install.log || rc=$? else - # `sh -s --` with no further args would pass an empty positional, - # so only add the separator when there are flags to pass. + # `sh -s --` with no further args passes an empty positional, so add + # the separator only when there are flags. if [ -n "$FLAGS" ]; then cat install.sh | env $CLEAN_ENV sh -s -- $FLAGS 2>&1 | tee logs/install.log || rc=$? else @@ -273,27 +262,26 @@ jobs: fi ;; tauri) - # Exactly how the desktop app invokes it: no tty, stdin closed. - # --tauri rejects a custom UNSLOTH_STUDIO_HOME outright (it still uses - # the legacy ~/.unsloth/studio root), so the workspace-scoped value - # every other leg relies on must be dropped or the installer exits - # before doing any work. The runner is ephemeral, so the real home is - # as disposable as the override. + # Exactly how the desktop app invokes it: no tty, stdin closed. --tauri + # rejects a custom UNSLOTH_STUDIO_HOME outright (it still uses the legacy + # ~/.unsloth/studio root), so the workspace-scoped value every other leg + # relies on must be dropped or the installer exits before doing any work. + # The runner is ephemeral, so the real home is as disposable. env -u UNSLOTH_STUDIO_HOME $CLEAN_ENV \ bash "$SCRIPT" --tauri $FLAGS < /dev/null 2>&1 | tee logs/install.log || rc=$? ;; esac echo "install_rc=$rc" >> "$GITHUB_OUTPUT" echo "installer exit code: $rc" - # The pipe legs expose curl:(56); surface it rather than leaving it buried - # in a 4000-line log. + # The pipe legs expose curl:(56); surface it rather than leaving it buried in + # a 4000-line log. if grep -qE "curl: \(5[36]\)|Failure writing output to destination" logs/install.log; then echo "::warning::curl reported a broken pipe -- an early exit killed the reader" fi exit "$rc" - # install.sh ignores an unset UNSLOTH_CI_SOURCE_OVERLAY, so a typo in the matrix - # or the expression silently puts every leg back on the released wheel. + # install.sh ignores an unset UNSLOTH_CI_SOURCE_OVERLAY, so a typo in the matrix or + # the expression silently puts every leg back on the released wheel. - name: Assert this ref's Python was really put under test if: matrix.overlay && inputs.installer_source != 'published' && steps.install.outcome == 'success' run: | @@ -309,7 +297,7 @@ jobs: set -a; . ./clean-machine.env; set +a checks="nobuild" # `absent` ran only BEFORE the install, so an installer that quietly selected - # the CLT or installed a compiler left the leg green. Re-run it afterwards. + # the CLT or installed a compiler left the leg green. Re-run it after. [ "${{ matrix.mode }}" = "mask" ] && checks="$checks absent" [ "${{ matrix.mode }}" = "trace" ] && checks="$checks notools" UNSLOTH_CLEAN_ALLOW_WORKING='${{ matrix.allow_working }}' \ @@ -319,8 +307,8 @@ jobs: if: steps.install.outcome == 'success' run: | set -a; . ./clean-machine.env; set +a - # The tauri leg cannot honour UNSLOTH_STUDIO_HOME (see Install) and went to - # the legacy root, where llama.cpp sits at /llama.cpp and the venv at + # The tauri leg cannot honour UNSLOTH_STUDIO_HOME (see Install) and went to the + # legacy root, where llama.cpp sits at /llama.cpp and the venv at # /studio: so ~/.unsloth, not ~/.unsloth/studio. if [ "${{ matrix.delivery }}" = "tauri" ]; then HOME_DIR="$HOME/.unsloth" @@ -328,16 +316,16 @@ jobs: HOME_DIR="$UNSLOTH_STUDIO_HOME" fi STUDIO_HOME="$HOME_DIR" bash .github/scripts/assert-llama-loads.sh - # Rosetta 2 is on this runner and not on a fresh Mac, so llama-server - # launching above does not prove it would launch for a user. Assert the arch - # of every payload (llama.cpp, whisper.cpp, the Node prebuilt, uv) instead. + # Rosetta 2 is on this runner and not on a fresh Mac, so llama-server launching + # above does not prove it would launch for a user. Assert the arch of every + # payload (llama.cpp, whisper.cpp, the Node prebuilt, uv) instead. MACHO_ROOT="$HOME_DIR" bash .github/scripts/clean-machine-assert.sh macho - name: Restore the runner if: always() - # `|| true` swallowed everything, including a restore that genuinely broke. The - # file only exists once the strip step ran, and an earlier step can fail before - # that, so skip explicitly when it is absent and let a real failure surface. + # `|| true` swallowed everything, a genuinely broken restore included. The file + # only exists once the strip step ran and an earlier step can fail before that, + # so skip explicitly when it is absent and let a real failure surface. run: | if [ -f .clean-machine/restore.sh ]; then bash .clean-machine/restore.sh @@ -349,8 +337,8 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - # Two rows differ only in `flags`, so flags must be in the name: artifacts - # are immutable per run and the second upload 409s. + # Two rows differ only in `flags`, so flags must be in the name: artifacts are + # immutable per run and the second upload 409s. name: clean-mac-${{ matrix.os }}-${{ matrix.mode }}-${{ matrix.delivery }}${{ matrix.flags && format('-{0}', matrix.flags) || '' }} path: | logs/ @@ -389,17 +377,16 @@ jobs: experimental: false overlay: true # No elevation, but WITH the transport the advertised one-liner needs: not - # root, no sudo anywhere on the image, no toolchain, ca-certificates + curl - # and nothing else. Since #7547 the optional set (cmake, git, - # build-essential, libcurl4-openssl-dev) never escalates, so this must - # install end to end off prebuilt llama.cpp, and until now nothing proved it. + # root, no sudo anywhere on the image, no toolchain, ca-certificates + curl and + # nothing else. Since #7547 the optional set (cmake, git, build-essential, + # libcurl4-openssl-dev) never escalates, so this must install end to end off + # prebuilt llama.cpp, and until now nothing proved it. # # Gating, and overlay: true for the reason fedora41 is: the RELEASED # install_python_stack.py has no "skip triton kernels when git is missing" # guard, so without the overlay the run reaches the final step and dies there # on pure release lag (staging run 30421021166: venv, frontend, torch and - # extras all fine, then `Installing triton kernels (pip) failed`). The - # overlay removes that difference, exactly as it does on fedora. + # extras all fine, then `Installing triton kernels (pip) failed`). - label: ubuntu2404-nonroot image: ubuntu:24.04 runner: ubuntu-latest @@ -407,16 +394,15 @@ jobs: overlay: true nonroot: true # The same premise with the OTHER transport. install.sh's download() prefers - # curl and falls back to wget (install.sh:729-738), _http_get does the same - # for the connectivity probe (1019-1027) and the Radeon listing repeats it - # (3064-3067), and _check_linux_deps only calls the transport missing when - # BOTH are gone (2077-2079). So a box with wget and no curl -- a Debian - # netinst default, and every image where curl was deliberately removed -- is - # supported on paper and had never been run: the nonroot row above provisions - # ca-certificates AND curl, so curl won every probe and the wget branch was - # only ever reasoned from the code. Everything the row asserts is what the - # nonroot row asserts, plus curl proved absent for the whole run rather than - # merely unused. + # curl and falls back to wget (729-738), _http_get does the same for the + # connectivity probe (1019-1027), the Radeon listing repeats it (3064-3067), + # and _check_linux_deps calls the transport missing only when BOTH are gone + # (2077-2079). So a box with wget and no curl -- a Debian netinst default, and + # every image where curl was deliberately removed -- is supported on paper and + # had never been run: the nonroot row above provisions ca-certificates AND + # curl, so curl won every probe and the wget branch was only ever reasoned from + # the code. This row asserts everything that one does, plus curl proved absent + # for the whole run rather than merely unused. - label: ubuntu2404-nonroot-wget image: ubuntu:24.04 runner: ubuntu-latest @@ -425,9 +411,9 @@ jobs: nonroot: true wget_only: true # No elevation AND no transport. apt is the only way to get curl and reaching - # apt is what needs elevation, so failing is correct; the point is to pin the - # exact message and prove it is actionable rather than a bare `curl: (56)`. - # No overlay: it never reaches a venv to overlay into. + # apt needs elevation, so failing is correct; the point is to pin the exact + # message and prove it is actionable rather than a bare `curl: (56)`. No + # overlay: it never reaches a venv to overlay into. - label: ubuntu2404-nonroot-notransport image: ubuntu:24.04 runner: ubuntu-latest @@ -457,11 +443,11 @@ jobs: run: | # tar and gzip ride along on the overlay legs: with no actions/checkout here # (it needs git) the only way in for this ref's source is an archive over the - # same transport. Neither is a compiler, git or cmake, so the premise holds. - # Both are usually in the base image already; naming them makes it certain. + # same transport. Neither is a compiler, git or cmake, so the premise holds; + # both are usually in the base image already, naming them makes it certain. # - # One transport, never both: the wget-only row is testing install.sh's wget - # branch, and that branch is unreachable while curl is on the box. + # One transport, never both: the wget-only row tests install.sh's wget branch, + # which is unreachable while curl is on the box. if [ "${{ matrix.wget_only }}" = "true" ]; then pkgs="ca-certificates wget" else @@ -484,7 +470,7 @@ jobs: raw="https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/${GITHUB_SHA}" # Whichever transport this row provisioned: the wget-only leg has no curl, on # purpose, and this is the one download in the job that cannot go through - # install.sh's own helper. Same preference order as that helper. + # install.sh's own helper. Same preference order as it uses. dl() { if command -v curl >/dev/null 2>&1; then curl -fsSL "$1" -o "$2" else wget -q -O "$2" "$1"; fi @@ -501,8 +487,8 @@ jobs: wc -l install.sh # The overlay needs a source tree and these legs deliberately have no - # actions/checkout. codeload serves the same commit as a tarball over plain - # HTTPS, so this ref's Python gets in without a git client. + # actions/checkout. codeload serves the same commit as a tarball over plain HTTPS, + # so this ref's Python gets in without a git client. - name: Fetch this ref's source tree for the overlay if: matrix.overlay && inputs.installer_source != 'published' run: | @@ -520,7 +506,7 @@ jobs: # curl is what fetched install.sh above, so the no-transport leg cannot simply # never install it. Take it away again afterwards: from the installer's point of - # view the machine has no way to download anything, which is the case under test. + # view the machine can download nothing, which is the case under test. - name: Take the transport away again if: matrix.no_transport run: | @@ -538,20 +524,20 @@ jobs: run: | useradd -m tester # Switching user without a login shell keeps the caller's environment, so the - # workflow-wide UNSLOTH_STUDIO_HOME follows tester in, and install.sh - # validates that override in _resolve_studio_destinations (536-591), long - # before the elevation gate (829-905). Without a writable target these legs - # die on "cannot be created" rather than on anything they are asking about. + # workflow-wide UNSLOTH_STUDIO_HOME follows tester in, and install.sh validates + # that override in _resolve_studio_destinations (536-591), long before the + # elevation gate (829-905). Without a writable target these legs die on "cannot + # be created" rather than on anything they are asking about. mkdir -p "$UNSLOTH_STUDIO_HOME" chown -R tester logs install.sh "$UNSLOTH_STUDIO_HOME" # The editable overlay writes .egg-info next to the pyproject.toml, so the # source tree has to belong to tester too or the overlay fails on permissions - # rather than on anything this leg is asking about. + # rather than on anything this leg asks about. if [ -d ci-source ]; then chown -R tester ci-source; fi - # Not calling sudo is not the same as not having it: a leg that merely avoided - # the call would pass on an image where elevation was available all along, and - # the whole claim of these two rows is that there is no elevation to be had. + # Not calling sudo is not the same as not having it: a leg that merely avoided the + # call would pass on an image where elevation was available all along, and the + # whole claim of these two rows is that there is none to be had. - name: Prove the unprivileged user genuinely cannot elevate if: matrix.nonroot run: | @@ -562,8 +548,8 @@ jobs: exit 1 fi # Absent from disk, not merely off PATH: install.sh probes with `command -v` - # (install.sh:830), so a binary tester could not reach would still be a lie - # about the image. + # (830), so a binary tester could not reach would still be a lie about the + # image. for p in /usr/bin/sudo /bin/sudo /usr/local/bin/sudo /usr/sbin/sudo /sbin/sudo; do if [ -e "$p" ]; then echo "::error::$p exists, so this image is not sudo-free" @@ -575,8 +561,8 @@ jobs: exit 1 fi # The capability, not just the tool: the escalation install.sh would attempt - # writes the dpkg database, so an unwritable one is what actually makes - # `apt-get install` impossible for tester. + # writes the dpkg database, so an unwritable one is what makes `apt-get + # install` impossible for tester. if su tester -c 'test -w /var/lib/dpkg/status'; then echo "::error::tester can write the dpkg database, so it is effectively root" exit 1 @@ -584,16 +570,16 @@ jobs: echo "tester is unprivileged, has no sudo on disk, and cannot write dpkg state" # The mirror of the sudo proof above, for the other premise this row makes. Not - # calling curl is not the same as not having it: every transport site in - # install.sh probes with `command -v curl` and prefers it (731, 1022, 2078, 3064), - # so a leg that merely avoided the call would go on testing the curl branch and - # report the wget one green. + # calling curl is not the same as not having it: every transport site in install.sh + # probes with `command -v curl` and prefers it (731, 1022, 2078, 3064), so a leg + # that merely avoided the call would go on testing the curl branch and report the + # wget one green. - name: Prove wget is the only transport if: matrix.wget_only run: | - # Absent from disk, not merely off PATH, for the same reason the sudo check - # looks on disk: `command -v` is what install.sh asks, and a binary tester - # could not reach would still be a lie about the image. + # On disk, not merely off PATH, for the same reason the sudo check is: + # `command -v` is what install.sh asks, and a binary tester could not reach + # would still be a lie about the image. for p in /usr/bin/curl /bin/curl /usr/local/bin/curl /usr/sbin/curl /sbin/curl /snap/bin/curl; do if [ -e "$p" ]; then echo "::error::$p exists, so this leg is not wget-only" @@ -607,13 +593,13 @@ jobs: fi done # The package too, so a dependency that quietly pulled the binary back in is - # caught here rather than silently reinstating the curl branch. + # caught rather than silently reinstating the curl branch. if dpkg-query -W -f='${Status}' curl 2>/dev/null | grep -q 'install ok installed'; then echo "::error::the curl package is installed; the wget-only premise does not hold" exit 1 fi - # And tester really does have the other one, or the row is the no-transport - # case wearing a different label. + # And tester really has the other one, or the row is the no-transport case + # wearing a different label. su tester -c 'command -v wget' >/dev/null 2>&1 || { echo "::error::tester cannot reach wget, so this leg has no transport at all" exit 1 @@ -625,9 +611,9 @@ jobs: if: ${{ !matrix.nonroot }} run: | set -o pipefail - # Resolved here, not in `env:`, so it tracks the step's real working - # directory: a container job remaps the workspace, and github.workspace is not - # something this needs to depend on. + # Resolved here, not in `env:`, so it tracks the step's real working directory: + # a container job remaps the workspace, and this need not depend on + # github.workspace. if [ -d ci-source ]; then export UNSLOTH_CI_SOURCE_OVERLAY="$PWD/ci-source" echo "overlaying this ref's source from $UNSLOTH_CI_SOURCE_OVERLAY" @@ -639,9 +625,9 @@ jobs: echo "installer exit code: $rc" exit "$rc" - # The no-elevation case the whole workflow was missing: everything absent AND no - # way to become root, but the transport the documented one-liner needs is there. - # Gating, and asserted end to end by the same steps the root legs use. + # The no-elevation case the workflow was missing: everything absent AND no way to + # become root, but the transport the documented one-liner needs is there. Gating, + # and asserted end to end by the same steps the root legs use. - name: Install (unprivileged, no sudo) id: install_nonroot if: ${{ matrix.nonroot && !matrix.no_transport }} @@ -672,8 +658,7 @@ jobs: # KNOWN OUTCOME PIN. This row is required, and continue-on-error on the step above # would otherwise tolerate a bootstrap outage or an unrelated early exit exactly - # like the intended diagnostic, so every branch here that is not the pinned - # outcome exits 1. + # like the intended diagnostic, so every branch here but the pinned outcome exits 1. - name: Assert the no-transport outcome is the elevation gate if: always() && matrix.no_transport && steps.install_notransport.outcome != 'skipped' run: | @@ -684,9 +669,8 @@ jobs: [ -f logs/install.log ] || { echo "::error::the no-transport leg produced no install log"; exit 1; } tail -40 logs/install.log # Both, so a failure anywhere else is still red: it must be the transport that - # was missing, and it must be the no-sudo branch of _smart_apt_install - # (install.sh:899-903) that stopped it rather than a prompt, a dpkg lock or a - # network error. + # was missing, and the no-sudo branch of _smart_apt_install (install.sh:899-903) + # that stopped it, not a prompt, a dpkg lock or a network error. if ! grep -q "missing: curl" logs/install.log; then echo "::error::the installer never reported the transport as missing; it did not reach the elevation gate" exit 1 @@ -702,8 +686,8 @@ jobs: fi echo "::notice::known outcome: no transport and no way to elevate, refused with an actionable message" - # experimental, so without this check a bootstrap outage or an unrelated early - # exit is tolerated like the intended diagnostic. + # experimental, so without this a bootstrap outage or an unrelated early exit is + # tolerated like the intended diagnostic. - name: Assert the Fedora outcome is a known one if: always() && matrix.label == 'fedora41' run: | @@ -713,15 +697,15 @@ jobs: fi [ -f logs/install.log ] || { echo "::error::fedora leg produced no install log"; exit 1; } tail -40 logs/install.log - # install.sh comes from this ref, so which of the two accepted outcomes - # applies depends on which dependency gate this ref carries. + # install.sh comes from this ref, so which of the two accepted outcomes applies + # depends on which dependency gate this ref carries. if grep -q "using prebuilt llama.cpp (missing:" logs/install.log; then - # The gate no longer hard-stops on a non-apt distro: it warns the optional - # build tools are absent and carries on, and reaching that warning is what - # proves the Linux gate did not stop the install. Past it, the accepted - # failure used to be release lag: install.sh came from this ref but unsloth - # from PyPI, and the released install_python_stack.py has no "skip the - # triton kernels when git is missing" guard, so it fetched the git+https + # The gate no longer hard-stops on a non-apt distro: it warns that the + # optional build tools are absent and carries on, and reaching that warning + # is what proves the Linux gate did not stop the install. Past it, the + # accepted failure used to be release lag: install.sh came from this ref but + # unsloth from PyPI, and the released install_python_stack.py has no "skip + # the triton kernels when git is missing" guard, so it fetched the git+https # triton_kernels requirement with no git. The overlay makes that guard this # ref's own code, so the triton failure must NOT come back: accepting it # would be accepting a regression in the guard as release lag. @@ -735,15 +719,15 @@ jobs: echo "::error::fedora got past the dependency warning and still failed, with this ref's Python overlaid; there is no known-good outcome left to accept" exit 1 fi - # This ref still hard-exits on a non-apt package manager. Pin that message so - # a bootstrap outage or an unrelated early exit is not tolerated as the - # intended diagnostic. + # This ref still hard-exits on a non-apt package manager. Pin that message so a + # bootstrap outage or an unrelated early exit is not tolerated as the intended + # diagnostic. if ! grep -qiE "Automatic system package installation is supported on apt-based|Fedora/RHEL: sudo dnf install" logs/install.log; then echo "::error::fedora leg failed neither at the unsupported-package-manager gate nor past the dependency warning" exit 1 fi - # See the macOS job: proves the leg is testing what its matrix row claims. + # See the macOS job: proves the leg tests what its matrix row claims. - name: Assert this ref's Python was really put under test if: matrix.overlay && inputs.installer_source != 'published' && (steps.install_root.outcome == 'success' || steps.install_nonroot.outcome == 'success') run: | @@ -754,8 +738,8 @@ jobs: echo "overlay applied; this leg exercised this ref's Python" # nobuild only reads the log, so an installer that exits 0 having done nothing - # satisfies it. Unlike WSL and Windows, these required Linux rows had no check - # that the install produced anything runnable. + # satisfies it. Unlike WSL and Windows, these required Linux rows had no check that + # the install produced anything runnable. - name: Assert the install is actually usable if: steps.install_root.outcome == 'success' || steps.install_nonroot.outcome == 'success' run: | @@ -767,20 +751,20 @@ jobs: - name: Assert llama.cpp came from the prebuilt bundle if: steps.install_root.outcome == 'success' || steps.install_nonroot.outcome == 'success' run: | - # HONESTY NOTE: the ROOT legs START toolchain-free but do not stay that way. - # As root, _smart_apt_install's first `apt-get install` (install.sh:797-799) + # HONESTY NOTE: the ROOT legs START toolchain-free but do not stay that way. As + # root, _smart_apt_install's first `apt-get install` (install.sh:797-799) # succeeds before the _SMART_APT_OPTIONAL guard (814-821) can suppress # anything, so `cmake git build-essential libcurl4-openssl-dev` really are - # installed mid-run. That is product behaviour on any root Linux install, not - # a CI artefact. What must still hold is that nothing USED them: `nobuild` - # reads Python builds only, and llama.cpp is the one thing that silently falls - # back to a source compile once a compiler is around. The unprivileged leg - # never gets that far -- the optional set cannot escalate -- so there the same - # marker proves the prebuilt path won with no compiler on the machine at all. + # installed mid-run: product behaviour on any root Linux install, not a CI + # artefact. What must still hold is that nothing USED them. `nobuild` reads + # Python builds only, and llama.cpp is the one thing that silently falls back + # to a source compile once a compiler is around. The unprivileged leg never + # gets that far -- the optional set cannot escalate -- so there the same marker + # proves the prebuilt path won with no compiler on the machine at all. for t in cmake git gcc; do printf '%-6s %s\n' "$t" "$(command -v "$t" 2>/dev/null || echo ABSENT)" done - # install_llama_prebuilt.py:5629 writes this marker; a source-built tree has + # install_llama_prebuilt.py:5629 writes this marker and a source-built tree has # no such metadata (studio/setup.sh:1517), so its presence is the one # unambiguous "the prebuilt path won" signal. META="$UNSLOTH_STUDIO_HOME/llama.cpp/UNSLOTH_PREBUILT_INFO.json" @@ -794,8 +778,8 @@ jobs: head -c 800 "$META"; echo # The claim this leg exists to make: a user with no elevation gets a full install - # and the machine is no less clean afterwards. Without it the row would prove only - # that SOME install happened, which the root legs already show. + # and the machine is no less clean afterwards. Without it the row proves only that + # SOME install happened, which the root legs already show. - name: Assert the unprivileged install elevated nothing if: steps.install_nonroot.outcome == 'success' run: | @@ -815,16 +799,16 @@ jobs: grep -n "deps" logs/install.log | tail -20 || true exit 1 fi - # #7547 is what made the optional set stop escalating. If either prompt comes - # back, an unprivileged user is blocked on tools nothing here uses. + # #7547 made the optional set stop escalating. If either prompt comes back, an + # unprivileged user is blocked on tools nothing here uses. if grep -qE "We require sudo elevated permissions|No terminal to confirm on" logs/install.log; then echo "::error::the installer tried to elevate for the optional build tools; #7547's no-escalation guard has regressed" exit 1 fi # Absent at the start is not absent throughout, and only the whole-run claim makes - # the leg mean anything: every download the installer just did -- the uv bootstrap - # included (install.sh:2232) -- had to go through wget, and it did only if curl was + # the leg mean anything: every download the installer just did, the uv bootstrap + # included (install.sh:2232), had to go through wget, and it did only if curl was # never there to be preferred. - name: Re-prove curl never appeared, and that wget carried the install if: matrix.wget_only && steps.install_nonroot.outcome == 'success' @@ -839,10 +823,10 @@ jobs: echo "::error::curl resolves after the install, so the run did not stay wget-only" exit 1 fi - # _check_linux_deps (install.sh:2077-2079) calls the transport missing only - # when curl AND wget are both gone, and the elevation gate below it is what the - # notransport row pins. Reaching it here would mean wget was not recognised as - # a transport at all. + # _check_linux_deps (install.sh:2077-2079) calls the transport missing only when + # curl AND wget are both gone, and the elevation gate below it is what the + # notransport row pins. Reaching it here would mean wget was not recognised as a + # transport at all. if grep -q "missing: curl" logs/install.log; then echo "::error::install.sh reported the transport as missing on a box that has wget, so it does not accept wget as one" exit 1 @@ -871,15 +855,15 @@ jobs: # ── WSL ─────────────────────────────────────────────────────────────────── # install.sh carries ~126 lines of WSL-specific logic (the `linux|wsl` dependency - # branch, UNSLOTH_WSL_REROUTED, the Strix Halo reroute to 24.04) that had never run - # in CI: tests/sh/test_strixhalo_wsl_reroute.sh mocks the environment, which cannot - # catch anything about a real WSL. No third-party action either -- the official - # Ubuntu rootfs plus `wsl --import` is deterministic and checksum-verifiable. + # branch, UNSLOTH_WSL_REROUTED, the Strix Halo reroute to 24.04) that had never run in + # CI: tests/sh/test_strixhalo_wsl_reroute.sh mocks the environment, which cannot catch + # anything about a real WSL. No third-party action either -- the official Ubuntu rootfs + # plus `wsl --import` is deterministic and checksum-verifiable. # # Gating, deliberately: it is the only job that runs the real WSL branch and the only - # one that can catch a piped install being truncated (WSL shells out to Windows - # interop mid-script, and interop relays the stdin it inherited). There is no flake to - # absorb, and #7548 is in main, so this gates unconditionally. + # one that can catch a piped install being truncated (WSL shells out to Windows interop + # mid-script, and interop relays the stdin it inherited). No flake to absorb, and #7548 + # is in main, so this gates unconditionally. wsl: name: wsl ubuntu-24.04 runs-on: windows-latest @@ -907,7 +891,7 @@ jobs: wsl --import unsloth-ci "$PWD/wsl-dist/instance" "$PWD/wsl-dist/rootfs.tar.gz" --version 2 wsl -d unsloth-ci -- uname -a # A freshly imported rootfs is genuinely bare: no curl, git or compiler. The - # clean machine, not a simulation of one. + # clean machine, not a simulation. wsl -d unsloth-ci -- sh -c 'for t in curl wget git gcc cmake python3 sudo; do printf "%-8s %s\n" "$t" "$(command -v $t || echo ABSENT)"; done' - name: Install inside WSL, piped exactly as documented @@ -916,10 +900,9 @@ jobs: # Only ca-certificates + curl: the advertised one-liner cannot start without a # transport. Everything else must come from the installer. wsl -d unsloth-ci -u root -- sh -c 'apt-get update -qq && apt-get install -y -qq --no-install-recommends ca-certificates curl' 2>&1 | Tee-Object -FilePath logs/wsl-bootstrap.log - # A dispatch selecting `published` asks whether unsloth.ai's script works; - # running the checked-out one and reporting the leg green answered a different - # question under the same name. Empty on pull_request/push, so automatic runs - # stay on this ref. + # A dispatch selecting `published` asks whether unsloth.ai's script works, and + # running the checked-out one answered a different question under the same + # name. Empty on pull_request/push, so automatic runs stay on this ref. if ('${{ inputs.installer_source }}' -eq 'published') { Write-Host 'installer: published (unsloth.ai)' wsl -d unsloth-ci -u root -- sh -c 'curl -fsSL https://unsloth.ai/install.sh -o /root/install.sh' @@ -946,15 +929,15 @@ jobs: } # The pipe-integrity check. Interop (_maybe_reroute_strixhalo_to_2404 -> # powershell.exe, wsl.exe) relays the stdin it inherited, so before #7548 it - # drank the rest of the script and sh died on a half-read line. #7548's - # _unsloth_main wrapper makes sh parse the file first; a truncation here means - # that regressed. + # drank the rest of the piped script and sh died on a half-read line. #7548's + # _unsloth_main wrapper forces sh to parse the file in full first; a truncation + # here means that regressed. if (Select-String -Path logs/wsl-install.log ` -Pattern 'Syntax error: Unterminated quoted string' -Quiet) { Write-Host '::error::the piped install was truncated again; install.sh is no longer parsed in full before it runs' exit 1 } - # Printing the code discarded it, and the next step's CLI check does not + # Printing the code discarded it, and the next step's CLI check cannot # compensate: install.sh links the `unsloth` shim (4174-4182) BEFORE it reports # a failing studio/setup.sh (4219-4230), so a late setup failure leaves a shim # whose --version succeeds. @@ -984,8 +967,8 @@ jobs: Write-Host '::error::installer never reported ''platform wsl''; the WSL branch was not exercised' exit 1 } - # No `|| echo`: substituting a message for the missing CLI made the inner - # shell, this step and the job all succeed on an install that produced nothing. + # No `|| echo`: substituting a message for the missing CLI made the inner shell, + # this step and the job all succeed on an install that produced nothing. $verify = wsl -d unsloth-ci -u root -- sh -c 'set -e; test -x "$HOME/.unsloth/studio/unsloth_studio/bin/unsloth"; "$HOME/.unsloth/studio/unsloth_studio/bin/unsloth" --version' 2>&1 $verifyRc = $LASTEXITCODE $verify | Tee-Object -FilePath logs/wsl-verify.log @@ -1022,30 +1005,29 @@ jobs: winget: 'visible' experimental: false overlay: true - # The no-winget path (LTSC / Server / managed corporate machines) falls back - # to python.org + astral.sh and is untested today. It is also where - # Ensure-VCRedist silently does not run, leaving torch unable to load, hence - # the explicit `import torch` assert below. + # The no-winget path (LTSC / Server / managed corporate machines) falls back to + # python.org + astral.sh and is untested today. It is also where Ensure-VCRedist + # silently does not run, leaving torch unable to load, hence the explicit + # `import torch` assert below. # - # It used to stop at studio/setup.ps1's unconditional "Git is required but - # could not be installed automatically" gate -- no winget meant no way to fetch - # git -- and was carried as a pinned known failure until #7549 landed. #7549 - # relaxed that gate to the --local and llama.cpp source paths that actually use - # git (setup.ps1:1750-1759), so the row installs end to end and gates like any - # other; the assert below is what proves it took the relaxed branch rather than - # passing because git leaked back onto PATH. + # It used to stop at studio/setup.ps1's unconditional "Git is required but could + # not be installed automatically" gate -- no winget meant no way to fetch git -- + # and was a pinned known failure until #7549 relaxed that gate to the --local + # and llama.cpp source paths that actually use git (setup.ps1:1750-1759). The + # row now installs end to end and gates like any other; the assert below proves + # it took the relaxed branch rather than passing on git leaking back onto PATH. - os: windows-latest winget: 'masked' experimental: false overlay: true - # Windows on ARM gets a native ARM64 CPython, and torchaudio has never - # published a win_arm64 wheel at any version (nor have pyarrow and hf-transfer, - # which datasets pulls in), so the PyTorch step could not resolve and - # install.ps1 stopped at "Failed to install PyTorch". Pinned until #7549, which - # makes the installer prefer an x64 interpreter on an ARM64 host and bootstrap - # one when only ARM64 is installed (install.ps1:1160-1253, 1335-1353); x64 - # wheels run fine emulated. The row now gates, and the assert below checks the - # outcome that fix has to produce rather than the log line announcing it. + # Windows on ARM gets a native ARM64 CPython, and torchaudio has never published + # a win_arm64 wheel at any version (nor have pyarrow and hf-transfer, which + # datasets pulls in), so the PyTorch step could not resolve and install.ps1 + # stopped at "Failed to install PyTorch". Pinned until #7549, which makes the + # installer prefer an x64 interpreter on an ARM64 host and bootstrap one when + # only ARM64 is installed (install.ps1:1160-1253, 1335-1353); x64 wheels run + # fine emulated. The row now gates, and the assert below checks the outcome that + # fix has to produce rather than the log line announcing it. - os: windows-11-arm winget: 'visible' experimental: false @@ -1062,8 +1044,8 @@ jobs: shell: pwsh run: | New-Item -ItemType Directory -Force -Path logs | Out-Null - # Drop preinstalled Python, git, CMake, VS/LLVM and the WindowsApps aliases - # from PATH. A full Visual Studio uninstall is not realistic in CI (registry + + # Drop preinstalled Python, git, CMake, VS/LLVM and the WindowsApps aliases from + # PATH. A full Visual Studio uninstall is not realistic in CI (registry + # vswhere discovery, slow, may need a reboot), so PATH and env scrubbing is the # honest approximation, recorded as such. $drop = @('hostedtoolcache\windows\Python', 'WindowsApps', '\Git\', @@ -1071,9 +1053,9 @@ jobs: 'MSYS', 'mingw', 'Strawberry') # winget is an app-execution alias in ...\Local\Microsoft\WindowsApps, so the # blanket drop removed it on EVERY leg and winget=visible silently ran the same - # fallback as winget=masked. Resolve it before the scrub and hand it back - # through a shim, so the visible leg gets winget without the Store's python.exe - # alias. windows-11-arm has no winget on the hosted image + # fallback as winget=masked. Resolve it before the scrub and hand it back via a + # shim, so the visible leg gets winget without the Store's python.exe alias. + # windows-11-arm has no winget on the hosted image # (actions/runner-images#14083), so only windows-latest can carry it. $wantWinget = ('${{ matrix.winget }}' -ne 'masked') -and ('${{ matrix.os }}' -eq 'windows-latest') $wingetCmd = Get-Command winget -ErrorAction SilentlyContinue @@ -1102,7 +1084,7 @@ jobs: # Take the toolcache Python off disk, not just off PATH: py.exe lives in # C:\Windows (which must stay) and uv does its own interpreter discovery, so # both reach the toolcache whatever PATH says. That is how a leg printing - # `python ABSENT` still installed with the runner's 3.13.14. + # `python ABSENT` still installed the runner's 3.13.14. foreach ($tc in @("$env:AGENT_TOOLSDIRECTORY\Python", 'C:\hostedtoolcache\windows\Python')) { if ($tc -and (Test-Path $tc)) { try { Rename-Item -LiteralPath $tc -NewName 'Python.masked' -ErrorAction Stop @@ -1116,10 +1098,9 @@ jobs: # 1369/2797) merges the Machine and User registry PATHs back into $env:Path, so # a process-only scrub lasts until the first bootstrap refresh, after which # Git/CMake/VS/LLVM are back and the rest of the install is not clean. The - # runner is ephemeral, so rewrite the registry copies too. It is a merge, not a - # replace, so the shim above keeps resolving. Expand first: - # SetEnvironmentVariable rewrites REG_EXPAND_SZ as REG_SZ - # (dotnet/runtime#1442). + # runner is ephemeral, so rewrite the registry copies too. A merge, not a + # replace, so the shim above keeps resolving. Expand first: SetEnvironmentVariable + # rewrites REG_EXPAND_SZ as REG_SZ (dotnet/runtime#1442). foreach ($scope in 'Machine','User') { $raw = [System.Environment]::GetEnvironmentVariable('Path', $scope) if ([string]::IsNullOrWhiteSpace($raw)) { continue } @@ -1140,34 +1121,34 @@ jobs: shell: pwsh run: | $leaked = @() - # `py` too: the launcher lives in C:\Windows, which the scrub keeps, and it - # finds the toolcache Python the scrub only removed from PATH. + # `py` too: the launcher lives in C:\Windows, which the scrub keeps, and finds + # the toolcache Python the scrub only removed from PATH. foreach ($t in 'python','py','git','cmake','cl') { $f = Get-Command $t -ErrorAction SilentlyContinue Write-Host ("{0,-8} {1}" -f $t, $(if ($f) { $f.Source } else { 'ABSENT' })) if ($f -and $t -ne 'py') { $leaked += "$t -> $($f.Source)" } } - # The launcher binary may stay, but an interpreter it can still START is a - # leak: Find-CompatiblePython (install.ps1:1130-1153) probes `py` first, so a - # version registered outside the renamed toolcache dirs gets reused and the - # Python bootstrap never runs. + # The launcher binary may stay, but an interpreter it can still START is a leak: + # Find-CompatiblePython (install.ps1:1130-1153) probes `py` first, so a version + # registered outside the renamed toolcache dirs gets reused and the Python + # bootstrap never runs. if (Get-Command py -ErrorAction SilentlyContinue) { # -0p is the launcher's REGISTRY view, and the mask renames directories # without rewriting it, so -0p keeps naming paths that no longer exist: - # context for a failure, never evidence of one. Only a probe that STARTS counts. + # context for a failure, never evidence. Only a probe that STARTS counts. Write-Host "py -0p (stale registry entries; masked paths no longer exist on disk):" & py -0p 2>&1 | ForEach-Object { Write-Host " $_" } foreach ($v in '-3.11', '-3.12', '-3.13') { $out = & py $v -c "import sys; print(sys.executable)" 2>&1 $rc = $LASTEXITCODE - # Print every probe: when this check next fails it must say why. + # Print every probe: when this next fails it must say why. Write-Host ("py {0} -> exit {1}: {2}" -f $v, $rc, (($out | Out-String).Trim() -replace '\r?\n', ' / ')) if ($rc -eq 0) { $leaked += "py $v -> $out" } } # A FAILING probe is the outcome we want, but it leaves $LASTEXITCODE - # non-zero and cmdlets never reset it, and the runner appends - # `exit $LASTEXITCODE` to every pwsh step (actions/runner#351) -- so all - # three legs exited 1, silently, on machines that were in fact clean. + # non-zero, cmdlets never reset it, and the runner appends `exit + # $LASTEXITCODE` to every pwsh step (actions/runner#351) -- so all three legs + # exited 1, silently, on machines that were in fact clean. $global:LASTEXITCODE = 0 } # Printing alone could not fail: run 30365014702 logged `python ABSENT` then @@ -1184,15 +1165,15 @@ jobs: exit 1 } } elseif ('${{ matrix.os }}' -eq 'windows-latest' -and -not $winget) { - # Otherwise the visible leg quietly degrades into a second masked leg. + # Or the visible leg quietly degrades into a second masked leg. Write-Host '::error::winget is not resolvable on the visible leg; the winget bootstrap is not under test' exit 1 } foreach ($scope in 'Machine','User') { Write-Host ("{0} PATH after scrub: {1}" -f $scope, [System.Environment]::GetEnvironmentVariable('Path', $scope)) } - # Every failure above exits 1 explicitly, so reaching here means clean. Be - # explicit rather than let the runner's appended `exit $LASTEXITCODE` decide. + # Every failure above exits 1 explicitly, so reaching here means clean. Say so + # rather than let the runner's appended `exit $LASTEXITCODE` decide. exit 0 - name: Install @@ -1204,7 +1185,7 @@ jobs: run: | $ErrorActionPreference = 'Continue' # Windows ships its own published script (install.ps1:3), so `published` means - # something here too. Running the checked-out one regardless made a dispatch + # something here too: running the checked-out one regardless made a dispatch # asking about unsloth.ai report on this ref. Empty on pull_request/push, so # automatic runs stay on this ref. $script = './install.ps1' @@ -1216,21 +1197,21 @@ jobs: } else { Write-Host "installer: this ref ($env:GITHUB_SHA)" } - # No -SkipTorch: install.ps1's parser matches `--no-torch` only (112-142), so - # the token was silently dropped and every leg installed torch anyway. The - # assert below needs torch, so get it on purpose. Run under powershell.exe, not - # this pwsh 7 step: a clean Windows box ships Windows PowerShell 5.1 only, and - # the desktop launches it the same way (install.rs:325-339). + # No -SkipTorch: install.ps1's parser matches `--no-torch` only (112-142), so the + # token was silently dropped and every leg installed torch anyway. The assert + # below needs torch, so get it on purpose. Under powershell.exe, not this pwsh 7 + # step: a clean Windows box ships Windows PowerShell 5.1 only, and the desktop + # launches it the same way (install.rs:325-339). & powershell.exe -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass ` -File $script *>&1 | Tee-Object -FilePath logs/install.log $rc = $LASTEXITCODE Write-Host "installer exit code: $rc" exit $rc - # Windows asserted nothing about the install ITSELF: nobuild and the toolchain - # check only read the log, so an installer that exited 0 having produced nothing - # satisfied both. The Linux legs have had this since they stopped being pinned; - # these rows needed it more, because two of them are only just off a pin. + # Windows asserted nothing about the install ITSELF: nobuild and the toolchain check + # only read the log, so an installer that exited 0 having produced nothing satisfied + # both. The Linux legs have had this since they stopped being pinned; these rows + # needed it more, two of them being only just off a pin. - name: Assert the install is actually usable shell: pwsh run: | @@ -1248,21 +1229,21 @@ jobs: Write-Host "::error::installer exited 0 but left no unsloth CLI at $cli" exit 1 } - # Present is not the same as runnable: the console script imports the whole - # command tree, so a missing dependency or an unimportable extension surfaces - # here and nowhere else. --version is the one subcommand-free path. + # Present is not runnable: the console script imports the whole command tree, so + # a missing dependency or an unimportable extension surfaces here and nowhere + # else. --version is the one subcommand-free path. & $cli --version if ($LASTEXITCODE -ne 0) { Write-Host '::error::the unsloth CLI is on disk but does not run' exit 1 } - # What #7549 has to produce on this host, checked as an outcome rather than as the - # log line announcing it. torchaudio, pyarrow and hf-transfer publish no win_arm64 - # wheel at any version, so a native ARM64 interpreter cannot resolve the stack; - # the installer's answer is to prefer, and if necessary bootstrap, an x64 CPython - # and let it run emulated. Asked of the interpreter through sysconfig, not inferred - # from PROCESSOR_ARCHITECTURE, which describes the shell rather than the venv. + # What #7549 has to produce on this host, checked as an outcome rather than the log + # line announcing it. torchaudio, pyarrow and hf-transfer publish no win_arm64 wheel + # at any version, so a native ARM64 interpreter cannot resolve the stack; the + # installer's answer is to prefer, and if necessary bootstrap, an x64 CPython and + # let it run emulated. Asked of the interpreter through sysconfig, not inferred from + # PROCESSOR_ARCHITECTURE, which describes the shell rather than the venv. - name: Assert the ARM64 host installed against an x64 interpreter if: matrix.os == 'windows-11-arm' shell: pwsh @@ -1275,21 +1256,20 @@ jobs: Write-Host "::error::the venv was built from a '$tag' interpreter, so the x64 preference on ARM64 hosts has regressed and the missing win_arm64 wheels are back" exit 1 } - # The package that has never shipped a win_arm64 wheel, so its presence is what - # proves the emulated x64 stack really resolved rather than being skipped. - # Metadata, not an import: this asserts resolution, and the import is the job of - # the torch assert below. + # The package that has never shipped a win_arm64 wheel, so its presence proves + # the emulated x64 stack really resolved rather than being skipped. Metadata, + # not an import: this asserts resolution, the torch assert below does the import. & $venvPy -c "from importlib.metadata import version; print('torchaudio', version('torchaudio'))" if ($LASTEXITCODE -ne 0) { Write-Host '::error::torchaudio is not installed, so the x64 interpreter did not buy the wheels it was chosen for' exit 1 } - # The other half of what #7549 has to produce. This row is the only place the - # relaxed git gate matters: winget is masked, so there is no way to fetch git at - # all, and setup.ps1 used to refuse to continue without it. Assert the relaxed - # branch was taken, so the row cannot go green because git leaked back onto PATH - # and the gate was never reached. + # The other half of what #7549 has to produce. This row is the only place the relaxed + # git gate matters: winget is masked, so there is no way to fetch git at all, and + # setup.ps1 used to refuse to continue without it. Assert the relaxed branch was + # taken, so the row cannot go green on git leaking back onto PATH with the gate + # never reached. - name: Assert the no-winget path installed without git if: matrix.winget == 'masked' shell: pwsh @@ -1311,7 +1291,7 @@ jobs: } Write-Host 'no winget, no git, and the install completed anyway' - # See the macOS job: proves the leg is testing what its matrix row claims. + # See the macOS job: proves the leg tests what its matrix row claims. - name: Assert this ref's Python was really put under test if: matrix.overlay && inputs.installer_source != 'published' && steps.install.outcome == 'success' shell: pwsh @@ -1329,15 +1309,15 @@ jobs: # Windows checked nothing after the install, so setup.ps1 committing to a # llama.cpp SOURCE build would winget-install CMake (setup.ps1:816-822) and VS # Build Tools (845-857) and the leg still went green. Git is out of scope on - # purpose: bootstrapping it through winget (setup.ps1:1658-1661) is the consumer - # path the visible leg exercises. The VC++ runtime is a runtime, not a toolchain. + # purpose: bootstrapping it through winget (1658-1661) is the consumer path the + # visible leg exercises. The VC++ runtime is a runtime, not a toolchain. $bad = @() if (-not (Test-Path logs/install.log)) { Write-Host '::error::no install log, so nothing proves the install stayed toolchain-free' exit 1 } # The announcements inside Ensure-BuildToolsForLlamaSourceBuild, which runs only - # for a committed source build. Matched instead of the package ids, because + # for a committed source build. Matched instead of the package ids because # setup.ps1 PRINTS `winget install ...BuildTools` as manual advice when winget is # missing, and advice is not an install. foreach ($m in 'CMake not found -- installing via winget', @@ -1346,10 +1326,10 @@ jobs: $bad += "install log reports: $m" } } - # winget puts what it installs on the MACHINE PATH, which this step's own - # process PATH (scrubbed, from GITHUB_ENV) never sees, so read the registry - # copies back rather than ask Get-Command. The scrub already removed every - # entry matching these, so a match here means the install put one back. + # winget puts what it installs on the MACHINE PATH, which this step's own process + # PATH (scrubbed, from GITHUB_ENV) never sees, so read the registry copies back + # rather than ask Get-Command. The scrub removed every entry matching these, so + # a match here means the install put one back. foreach ($scope in 'Machine','User') { $raw = [System.Environment]::GetEnvironmentVariable('Path', $scope) if ([string]::IsNullOrWhiteSpace($raw)) { continue } @@ -1369,31 +1349,30 @@ jobs: if: always() && steps.install.outcome != 'skipped' shell: pwsh run: | - # The step above only catches a NEW CMake or VS Build Tools install. The - # image's Visual Studio survives a PATH scrub: setup.ps1's Find-VsBuildTools - # (763-800) reaches it through vswhere and a Program Files scan, and the - # visible leg logs `vs Visual Studio 18 2026 (vswhere)` on the same machine - # whose pre-flight printed `cl ABSENT`. So a dependency that lost its Windows - # wheel would compile against that MSVC and the leg would stay green, while - # macOS and Linux caught it. uv really does build sdists here (openai-whisper, - # antlr4-python3-runtime, randomname, argbind), so this is the live path. + # The step above only catches a NEW CMake or VS Build Tools install. The image's + # Visual Studio survives a PATH scrub: setup.ps1's Find-VsBuildTools (763-800) + # reaches it through vswhere and a Program Files scan, and the visible leg logs + # `vs Visual Studio 18 2026 (vswhere)` on the same machine whose pre-flight + # printed `cl ABSENT`. So a dependency that lost its Windows wheel would compile + # against that MSVC and the leg would stay green while macOS and Linux caught + # it. uv really does build sdists here (openai-whisper, antlr4-python3-runtime, + # randomname, argbind), so this is the live path. & "$env:GITHUB_WORKSPACE/.github/scripts/assert-nobuild.ps1" -LogPath logs/install.log - name: Assert torch loads, and record what that does and does not prove if: steps.install.outcome == 'success' shell: pwsh run: | - # HONESTY NOTE: the image ships the VC++ 2015-2022 runtime in System32 and it - # cannot be removed without breaking the runner, so `import torch` succeeding - # does NOT prove a clean no-winget machine has it -- Test-VCRedistInstalled - # (setup.ps1:875) finds the preinstalled DLL and Ensure-VCRedist (891) - # short-circuits. Record that, then assert what CAN fail. + # HONESTY NOTE: the image ships the VC++ 2015-2022 runtime in System32 and cannot + # lose it without breaking the runner, so `import torch` succeeding does NOT + # prove a clean no-winget machine has it -- Test-VCRedistInstalled (setup.ps1:875) + # finds the preinstalled DLL and Ensure-VCRedist (891) short-circuits. Record + # that, then assert what CAN fail. $sys32 = Join-Path $env:WINDIR 'System32\vcruntime140_1.dll' Write-Host "preinstalled System32 vcruntime140_1.dll: $(Test-Path $sys32)" - # The managed interpreter, with no fallback to whatever `python` resolves to: - # the usability assert above already hard-fails when it is missing, and a - # fallback would answer this question with an interpreter the install did not - # create. + # The managed interpreter, with no fallback to whatever `python` resolves to: the + # usability assert above already hard-fails when it is missing, and a fallback + # would answer this with an interpreter the install did not create. $py = Join-Path $env:UNSLOTH_STUDIO_HOME 'unsloth_studio\Scripts\python.exe' & $py -c "import ctypes.util, sys; print('VCRUNTIME140:', ctypes.util.find_library('vcruntime140'))" & $py -c "import torch; print('torch', torch.__version__)" @@ -1417,21 +1396,21 @@ jobs: if-no-files-found: warn # ── Windows, genuinely virgin: the same install inside a Windows container ──── - # The `win` legs above only SIMULATE absence, and two things they structurally - # cannot test are the VC++ 2015-2022 runtime (it ships in the runner image's System32 - # and cannot be removed without breaking the runner) and a Windows with no Microsoft - # Store at all rather than a winget hidden from PATH. A servercore container answers - # both, so this lane lives here: same premise, same path filters, masked next to real. + # The `win` legs above only SIMULATE absence, and two things they structurally cannot + # test are the VC++ 2015-2022 runtime (it ships in the runner image's System32 and + # cannot be removed without breaking the runner) and a Windows with no Microsoft Store + # at all rather than a winget hidden from PATH. A servercore container answers both, so + # this lane lives here: same premise, same path filters, masked next to real. # # Constraints, all load-bearing: - # * `container:` is Linux-only on the Actions runner (actions/runner#1402), so - # docker is driven from ordinary `run:` steps and the payload goes in by - # `docker cp` -- actions/checkout inside the container would need git. + # * `container:` is Linux-only on the Actions runner (actions/runner#1402), so docker + # is driven from ordinary `run:` steps and the payload goes in by `docker cp` -- + # actions/checkout inside the container would need git. # * servercore, not nanoserver: install.ps1 needs Windows PowerShell 5.1, which # nanoserver does not ship at all. - # * windows-2022, not windows-latest: process isolation needs the host and - # container builds to match, and only the 2022 image pre-caches ltsc2022. - # windows-latest is Server 2025 and caches no Windows images. + # * windows-2022, not windows-latest: process isolation needs the host and container + # builds to match, and only the 2022 image pre-caches ltsc2022. windows-latest is + # Server 2025 and caches no Windows images. windows_container_probe: name: virgin win container / probe runs-on: windows-2022 @@ -1444,9 +1423,9 @@ jobs: fetch-depth: 1 persist-credentials: false - # Docker is on every windows-2022 image but is not always already running: one - # spike leg died in 21s on npipe:////./pipe/docker_engine, which misreads as - # "Windows containers are unavailable". + # Docker is on every windows-2022 image but is not always already running: one spike + # leg died in 21s on npipe:////./pipe/docker_engine, which misreads as "Windows + # containers are unavailable". - name: Ensure the Docker daemon is running shell: pwsh run: ./.github/scripts/ensure-docker-daemon.ps1 @@ -1467,9 +1446,9 @@ jobs: - name: Start the container shell: pwsh run: | - # Never refresh a cached image: process isolation needs the container build - # <= the host build, and MCR has shipped a patched image ahead of the host - # before (actions/runner-images#11582 broke Windows containers for ~2 weeks). + # Never refresh a cached image: process isolation needs the container build <= + # the host build, and MCR has shipped a patched image ahead of the host before + # (actions/runner-images#11582 broke Windows containers for ~2 weeks). if ((docker images --format '{{.Repository}}:{{.Tag}}') -contains $env:IMAGE) { Write-Host "using the runner's pre-cached $env:IMAGE (no pull)" } else { @@ -1477,8 +1456,8 @@ jobs: docker pull $env:IMAGE if ($LASTEXITCODE -ne 0) { Write-Host "::error::could not pull $env:IMAGE"; exit 1 } } - # A keepalive entrypoint so each assertion can be its own `docker exec`, and - # therefore its own step with its own exit code. + # A keepalive entrypoint so each assertion can be its own `docker exec`, and so + # its own step with its own exit code. docker run -d --name virgin $env:IMAGE cmd /c "ping -t localhost >nul" if ($LASTEXITCODE -ne 0) { Write-Host "::error::could not start a container from $env:IMAGE"; exit 1 } Write-Host "isolation: $(docker inspect virgin --format '{{.HostConfig.Isolation}}')" @@ -1521,13 +1500,13 @@ jobs: matrix: include: # The consumer path: install.ps1 from this ref, unsloth from PyPI. So - # studio/setup.ps1 comes out of the RELEASED wheel, which is why this row and - # the overlay one below do not currently reach the same place: see the pin on - # the Install step. + # studio/setup.ps1 comes out of the RELEASED wheel, which is why this row and the + # overlay one below do not currently reach the same place: see the pin on the + # Install step. - overlay: false # This ref's studio/setup.ps1 and install_python_stack.py, via # UNSLOTH_CI_SOURCE_OVERLAY (install.ps1:2643). Without it a branch changing - # setup.ps1 gets a green run that proves nothing about the change. + # setup.ps1 gets a green run proving nothing about the change. - overlay: true steps: @@ -1556,7 +1535,7 @@ jobs: docker exec virgin cmd /c "mkdir C:\ci-out" # Re-run here and not only in `probe`: different runner, and an install leg that - # skipped the check would be reporting on an environment it never verified. + # skipped the check would report on an environment it never verified. - name: Assert the container is genuinely virgin shell: pwsh run: | @@ -1565,20 +1544,20 @@ jobs: *>&1 | Tee-Object -FilePath logs/virginity.log exit $LASTEXITCODE - # AFTER the virginity assertion, so that assertion still proves what it says. A - # fresh container ships an almost empty trusted-root store while a real desktop - # fills it via automatic root update, so seeding makes this MORE representative. - # Needed because studio/install_node_prebuilt.py downloads Node with bare + # AFTER the virginity assertion, so that assertion still proves what it says. A fresh + # container ships an almost empty trusted-root store while a real desktop fills it + # via automatic root update, so seeding makes this MORE representative. Needed + # because studio/install_node_prebuilt.py downloads Node with bare # urllib.request.urlopen, reads the empty Windows ROOT store and gets # CERTIFICATE_VERIFY_FAILED; uv and pip bundle certifi. Reported separately. - name: Seed the container's trusted root CA store shell: pwsh run: | - # -generateSSTFromWU pulls each root from ctldl.windowsupdate.com, and that - # host times out often enough to be the leg's main flake (staging run - # 30423072537 died on WinHttp 12002 while the sibling row seeded fine). - # Retry, but never tolerate a total failure: without the roots, Node's - # urllib download later fails with CERTIFICATE_VERIFY_FAILED. + # -generateSSTFromWU pulls each root from ctldl.windowsupdate.com, which times + # out often enough to be the leg's main flake (staging run 30423072537 died on + # WinHttp 12002 while the sibling row seeded fine). Retry, but never tolerate a + # total failure: without the roots Node's urllib download later fails with + # CERTIFICATE_VERIFY_FAILED. for ($i = 1; $i -le 3; $i++) { docker exec virgin cmd /c "certutil -generateSSTFromWU C:\roots.sst && certutil -addstore -f Root C:\roots.sst" ` *>&1 | Select-Object -Last 15 @@ -1595,16 +1574,16 @@ jobs: id: install shell: pwsh # RELEASE-LAG PIN, overlay=false only. A Server Core container has no Microsoft - # Store and therefore no winget, ever, and studio/setup.ps1 used to hard-stop on - # a winget-only git gate and reach for winget again for the VC++ runtime. #7549 - # relaxed both, and this branch has it -- but the released wheel does not: the - # setup.ps1 inside unsloth 2026.7.5 (uploaded 2026-07-23, and #7549 landed on the - # 28th) still carries the old gate, so the row that deliberately installs from - # PyPI still cannot get past it. That is release lag, not a product gap, and - # nothing in this branch can change it; the overlay row runs the same install - # against this ref's setup.ps1 and gates unconditionally. The step below accepts - # only that exact signature and hard-errors the moment the released wheel catches - # up. + # Store and so no winget, ever, and studio/setup.ps1 used to hard-stop on a + # winget-only git gate and reach for winget again for the VC++ runtime. #7549 + # relaxed both and this branch has it, but the released wheel does not: unpacking + # unsloth 2026.7.5 (uploaded 2026-07-23, #7549 landed on the 28th) shows its + # setup.ps1 still carrying the old gate, so the row that deliberately installs + # from PyPI cannot get past it. Release lag, not a product gap; nothing in this + # branch can change it, only the next RELEASE, not a merge. The overlay row runs + # the same install against this ref's setup.ps1 and gates unconditionally, and + # the step below accepts only that exact signature, hard-erroring the moment the + # released wheel catches up. continue-on-error: ${{ !matrix.overlay }} run: | $overlayArg = if ('${{ matrix.overlay }}' -eq 'true') { 'C:\ci' } else { '' } @@ -1613,31 +1592,30 @@ jobs: -Overlay "$overlayArg" *>&1 | Tee-Object -FilePath logs/install-outer.log exit $LASTEXITCODE - # The overlay row runs this ref's studio/setup.ps1, so it carries #7549 and has - # to install end to end. The in-container harness already asserts the venv - # interpreter, the unsloth CLI, `import torch`, the no-winget path, the overlay - # marker and nobuild, and exits 1 listing every failure -- so the Install step - # gating is most of the assertion. What is added here is the part this lane alone - # can prove. + # The overlay row runs this ref's studio/setup.ps1, so it carries #7549 and has to + # install end to end. The in-container harness already asserts the venv interpreter, + # the unsloth CLI, `import torch`, the no-winget path, the overlay marker and + # nobuild, and exits 1 listing every failure -- so the Install step gating is most + # of the assertion. Added here is the part this lane alone can prove. - name: Assert the virgin container install proved what this lane exists for if: matrix.overlay shell: pwsh run: | $log = Get-Content logs/install-outer.log -Raw - # A `docker exec` that lost its container also exits 0, so read the harness's - # own verdict rather than trusting the exit code alone. + # A `docker exec` that lost its container also exits 0, so read the harness's own + # verdict rather than trust the exit code alone. if (-not ($log -match 'VIRGIN WINDOWS CONTAINER INSTALL PASSED')) { Write-Host '::error::the install step exited 0 but the in-container harness never printed its passing verdict' exit 1 } - # The overlay hook is this PR's own feature and gates unconditionally: without - # it this row would be indistinguishable from the released-wheel one. + # The overlay hook is this PR's own feature and gates unconditionally: without it + # this row is indistinguishable from the released-wheel one. if (-not ($log -match 'CI: overlaying source checkout')) { Write-Host '::error::the overlay row never overlaid the checkout, so it only tested the released package' exit 1 } - # Git: no Store, no winget, no git, and nothing on the consumer path needs it. - # The relaxed gate is the only reason this row gets past setup.ps1 at all. + # Git: no Store, no winget, no git, and nothing on the consumer path needs it. The + # relaxed gate is the only reason this row gets past setup.ps1 at all. if ($log -match 'Git is required but could not be installed automatically') { Write-Host '::error::studio/setup.ps1 stopped at the unconditional git gate; the relax to --local and llama.cpp source-build installs has regressed' exit 1 @@ -1646,10 +1624,10 @@ jobs: Write-Host '::error::setup.ps1 never reported git as absent-but-not-required, so this container was not gitless and the relaxed gate went untested' exit 1 } - # VC++: this container is the ONLY environment in the workflow whose System32 - # does not already ship the 2015-2022 runtime (the hosted Windows legs cannot - # remove it without breaking the runner), so it is the only place the direct - # aka.ms download can be proved to run rather than be short-circuited by + # VC++: this container is the ONLY environment in the workflow whose System32 does + # not already ship the 2015-2022 runtime (the hosted legs cannot remove it + # without breaking the runner), so it is the only place the direct aka.ms + # download can be proved to run rather than be short-circuited by # Test-VCRedistInstalled. Both halves: the fallback was taken, and it worked. if (-not ($log -match 'downloading the runtime directly')) { Write-Host '::error::Ensure-VCRedist never took the direct-download fallback, so a container with no VC++ runtime and no winget did not exercise it' @@ -1659,16 +1637,16 @@ jobs: Write-Host '::error::the direct VC++ runtime download ran but left the runtime uninstalled' exit 1 } - # The harness already ran `import torch` against the managed interpreter, which - # is what actually needs VCRUNTIME140_1.dll; this is the announcement that the - # DLL got there rather than having been there all along. + # The harness already ran `import torch` against the managed interpreter, which is + # what needs VCRUNTIME140_1.dll; this announces that the DLL got there rather + # than having been there all along. Write-Host '::notice::no Store, no winget, no git and no preinstalled VC++ runtime, and the install completed anyway' # RELEASE-LAG PIN (overlay=false). See the Install step: this row installs unsloth - # from PyPI on purpose, and the released setup.ps1 predates #7549. continue-on-error - # would otherwise tolerate a bootstrap outage or an unrelated early exit exactly - # like the intended diagnostic, so every branch here that is not the pinned failure - # exits 1 and fails the (required) job. + # from PyPI on purpose and the released setup.ps1 predates #7549. continue-on-error + # would otherwise tolerate a bootstrap outage or an unrelated early exit exactly like + # the intended diagnostic, so every branch here but the pinned failure exits 1 and + # fails the (required) job. - name: Assert the released-wheel row failed only on release lag if: always() && !matrix.overlay && steps.install.outcome != 'skipped' shell: pwsh @@ -1683,8 +1661,8 @@ jobs: } $log = Get-Content logs/install-outer.log -Raw # The pinned signature is the OLD gate wording, which #7549 deleted. Its - # disappearance from a released wheel is the flip condition, and until then a - # failure anywhere else has to be red. + # disappearance from a released wheel is the flip condition; until then a failure + # anywhere else has to be red. $gitGate = $log -match 'Git is required but could not be installed automatically' $vcGate = $log -match 'torch failed to import' if (-not ($gitGate -or $vcGate)) { @@ -1692,14 +1670,14 @@ jobs: exit 1 } # `-or` on its own is too generous. virgin-windows-install.ps1:97 runs the torch - # assertion whenever the venv interpreter exists, whatever the installer did, - # and this image has no VC++ runtime, so ANY failure after venv creation -- a - # Node download, a setup step, a bad prebuilt -- arrives here carrying the - # $vcGate text and was accepted as the pinned outcome. Enumerate what the - # harness actually recorded instead: it prints one `::error::` per entry - # of its $failures list (that script:151), and every one has to be a pinned - # gate. Anchored, because it also dumps the install log tail indented two spaces - # and those copies must not count. + # assertion whenever the venv interpreter exists, whatever the installer did, and + # this image has no VC++ runtime, so ANY failure after venv creation -- a Node + # download, a setup step, a bad prebuilt -- arrives here carrying the $vcGate + # text and was accepted as the pinned outcome. Enumerate what the harness + # actually recorded instead: it prints one `::error::` per entry of its + # $failures list (that script:151), and every one has to be a pinned gate. + # Anchored, because it also dumps the install log tail indented two spaces and + # those copies must not count. $recorded = @(Get-Content logs/install-outer.log | ForEach-Object { if ($_ -match '^::error::(.+)$') { $Matches[1].Trim() } }) Write-Host "recorded failures: $($recorded.Count)" @@ -1708,7 +1686,7 @@ jobs: Write-Host '::error::the container install failed but recorded no ::error:: line, so nothing identifies which gate stopped it' exit 1 } - # The git gate makes install.ps1 exit non-zero; the missing runtime makes the + # The git gate makes install.ps1 exit non-zero, the missing runtime makes the # torch assert fail. Nothing else is pinned. $pinned = @('^installer exited \d+$', '^torch failed to import from the managed Python') $unexpected = @($recorded | Where-Object { $r = $_; -not ($pinned | Where-Object { $r -match $_ }) }) diff --git a/.github/workflows/desktop-app-clean-machine-ci.yml b/.github/workflows/desktop-app-clean-machine-ci.yml index e5746a2343..7e95aa8c26 100644 --- a/.github/workflows/desktop-app-clean-machine-ci.yml +++ b/.github/workflows/desktop-app-clean-machine-ci.yml @@ -13,15 +13,15 @@ # installs, the binary is present, of the right architecture, and clears the gatekeeper # checks a user hits (macOS quarantine + codesign, Windows installer exit); the process # STAYS UP past its preflight, where an unhappy app dies; and it writes tauri.log with a -# preflight disposition, the field that read `ManagedReady` over an unbootable venv in -# the bug report. Linux gets the strongest check: a real webview under Xvfb. +# preflight disposition, the field that read `ManagedReady` over an unbootable venv in the +# bug report. Linux gets the strongest check: a real webview under Xvfb. name: Desktop app clean machine on: - # Also on PRs touching this job or the stripping scripts: dispatch resolves the - # workflow from the DEFAULT branch, so a new or edited file on a feature branch can - # never be dispatched and would first run only after merging blind. + # Also on PRs touching this job or the stripping scripts: dispatch resolves the workflow + # from the DEFAULT branch, so a new or edited file on a feature branch can never be + # dispatched and would first run only after merging blind. pull_request: paths: - '.github/workflows/desktop-app-clean-machine-ci.yml' @@ -50,21 +50,21 @@ concurrency: cancel-in-progress: true permissions: - # Drafts are listed only to a token with push access, and every desktop-v* release in - # this repo is a draft, so `contents: read` cannot see the bundle under test at all. + # Drafts are listed only to a token with push access, and every desktop-v* release here + # is a draft, so `contents: read` cannot see the bundle under test at all. contents: write env: - # release-desktop.yml publishes into github.repository, so a nightly aimed anywhere - # else goes green over a broken production bundle. unsloth-test/unsloth-test holds - # one frozen release, so the schedule was re-testing the same fixture forever. + # release-desktop.yml publishes into github.repository, so a nightly aimed anywhere else + # goes green over a broken production bundle. unsloth-test/unsloth-test holds one frozen + # release, so the schedule was re-testing the same fixture forever. REL_REPO: ${{ inputs.release_repo || github.repository }} - # Empty unless dispatched: a pinned tag is an immutable fixture, so a nightly against - # it could never catch a newly published broken bundle. Each download step then - # resolves the newest desktop-v* release, drafts included -- every desktop-v* release - # here is cut as a draft, so --exclude-drafts matched nothing and every leg died - # resolving. releases/tags/ 404s for a draft, but gh looks drafts up over GraphQL, - # so `gh release download ` still fetches their assets. + # Empty unless dispatched: a pinned tag is an immutable fixture, so a nightly against it + # could never catch a newly published broken bundle. Each download step then resolves + # the newest desktop-v* release, drafts included -- every desktop-v* release here is cut + # as a draft, so --exclude-drafts matched nothing and every leg died resolving. + # releases/tags/ 404s for a draft, but gh looks drafts up over GraphQL, so `gh + # release download ` still fetches their assets. REL_TAG: ${{ inputs.release_tag || '' }} UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home UNSLOTH_STUDIO_DISABLE_PUBLIC_CHECK: '1' @@ -73,8 +73,8 @@ jobs: # ── macOS: .dmg, Apple Silicon ──────────────────────────────────────────── macos: # A fork PR's token is read-only however this workflow declares permissions, so it - # cannot list the draft releases every desktop-v* bundle is published as. Skip - # rather than fail: it is a property of the trigger, not a broken release. + # cannot list the draft releases every desktop-v* bundle is published as. Skip rather + # than fail: a property of the trigger, not a broken release. if: github.event.pull_request.head.repo.fork != true name: desktop macOS ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -98,8 +98,8 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | mkdir -p dl logs - # Desktop releases are prereleases (never repo-wide "latest") and drafts, so - # the newest desktop-v* tag has to be resolved explicitly. See REL_TAG above. + # Desktop releases are prereleases (never repo-wide "latest") and drafts, so the + # newest desktop-v* tag has to be resolved explicitly. See REL_TAG above. if [ -z "$REL_TAG" ]; then REL_TAG="$(gh release list --repo "$REL_REPO" --limit 100 \ --json tagName,createdAt \ @@ -118,10 +118,9 @@ jobs: ls -la dl - name: Strip the developer toolchain - # `inputs` exists only for workflow_dispatch, so elsewhere strip_toolchain is - # '' -- and loose equality coerces both '' and false to 0, making `!= false` - # FALSE, so automatic runs would keep the very toolchain this removes. Gate on - # the event instead. + # `inputs` exists only for workflow_dispatch, so elsewhere strip_toolchain is '' -- + # and loose equality coerces both '' and false to 0, making `!= false` FALSE, so + # automatic runs would keep the very toolchain this removes. Gate on the event. if: ${{ github.event_name != 'workflow_dispatch' || inputs.strip_toolchain }} run: | bash .github/scripts/clean-machine-env.sh mask --remove @@ -131,8 +130,8 @@ jobs: - name: Mount and install run: | DMG="$(ls dl/*.dmg | head -1)" - # A real download is quarantined, and Gatekeeper treats that differently - # from a locally built bundle: a genuine failure mode. + # A real download is quarantined, and Gatekeeper treats that differently from a + # locally built bundle: a genuine failure mode. xattr -w com.apple.quarantine \ "0081;$(printf %x $(date +%s));Safari;" "$DMG" 2>/dev/null || true hdiutil attach "$DMG" -nobrowse -quiet -mountpoint /Volumes/UnslothCI @@ -148,8 +147,8 @@ jobs: BIN="$APP/Contents/MacOS/$(/usr/libexec/PlistBuddy -c 'Print :CFBundleExecutable' "$APP/Contents/Info.plist")" file "$BIN" # `lipo -archs` prints and exits 0 for a thin x86_64 binary, and `|| true` - # swallowed even that, so architecture was never asserted. lipo is an xcrun - # shim, gone once the strip moved CommandLineTools aside; file is base system. + # swallowed even that, so architecture was never asserted. lipo is an xcrun shim, + # gone once the strip moved CommandLineTools aside; file is base system. ARCHS="$(lipo -archs "$BIN" 2>/dev/null || true)" [ -n "$ARCHS" ] || ARCHS="$(file -b "$BIN")" echo "architectures: $ARCHS" @@ -157,14 +156,14 @@ jobs: *arm64*|*aarch64*) ;; *) echo "::error::the aarch64 .dmg carries no arm64 binary ($ARCHS)"; exit 1 ;; esac - # Report rather than gate: an unnotarised beta is expected to fail - # assessment, but a user WILL hit this, so it must be visible. + # Report rather than gate: an unnotarised beta is expected to fail assessment, but + # a user WILL hit this, so it must be visible. codesign -dv --verbose=2 "$APP" 2>&1 | head -20 || true spctl -a -vvv -t install "$APP" 2>&1 | head -5 || \ echo "::warning::Gatekeeper assessment failed -- users see 'cannot be opened' unless notarised" - # The bundled installer is what actually failed for users, and `::error::` is - # only an annotation that `echo` exits 0 from, so `|| echo` let a bundle with - # no installer pass. + # The bundled installer is what actually failed for users, and `::error::` is only + # an annotation that `echo` exits 0 from, so `|| echo` let a bundle with no + # installer pass. if [ -f "$APP/Contents/Resources/install.sh" ]; then echo "bundled install.sh present" else @@ -179,15 +178,15 @@ jobs: APP="$(ls -d /Applications/*Unsloth*.app | head -1)" # A headless runner never clicks Install: preflight sets `not_installed` and # returns (use-tauri-backend.ts:252-254) while startup-screen.tsx:388-389 waits - # for the button, so launching alone sits there for 90s without ever running - # the bundled installer. Invoke it as src-tauri/src/install.rs does: --tauri, - # stdin closed, no tty. --tauri rejects a custom studio home - # (install.sh:102-114), so drop the override. - # KNOWN OUTCOME PIN, retire when the desktop release catches up to #7547. - # REL_TAG predates #7547, so the bundle's own install.sh still hard-exits on - # the Xcode CLT gate that #7547 replaced with a warning. No change to this PR - # can move that; only a new release can. _check_macos_deps is the function - # #7547 added, so finding it means the release caught up and this pin must go. + # for the button, so launching alone sits there for 90s without ever running the + # bundled installer. Invoke it as src-tauri/src/install.rs does: --tauri, stdin + # closed, no tty. --tauri rejects a custom studio home (install.sh:102-114), so + # drop the override. + # KNOWN OUTCOME PIN, retire when the desktop release catches up to #7547. REL_TAG + # predates #7547, so the bundle's own install.sh still hard-exits on the Xcode + # CLT gate that #7547 replaced with a warning. Only a new release can move that, + # not this PR. _check_macos_deps is the function #7547 added, so finding it means + # the release caught up and this pin must go. SH="$APP/Contents/Resources/install.sh" if grep -q '_check_macos_deps' "$SH"; then echo "::error::the bundled install.sh now carries #7547; delete this pin block and let the venv + torch assertions below run unconditionally" @@ -210,8 +209,8 @@ jobs: PY="$HOME/.unsloth/studio/unsloth_studio/bin/python" [ -x "$PY" ] || { echo "::error::bundled installer left no venv at $PY"; exit 1; } "$PY" -V - # install.rs passes only --tauri, so torch is part of first launch: without - # this the venv check passes a bundle whose only failure is the torch install. + # install.rs passes only --tauri, so torch is part of first launch: without this + # the venv check passes a bundle whose only failure is the torch install. "$PY" -c "import torch; print('torch', torch.__version__)" - name: Launch and prove it stays up @@ -251,18 +250,18 @@ jobs: done # Everything above is `|| true`, so this step could not fail while the header # sells the tauri.log disposition as an acceptance criterion. setup_logging - # (src-tauri/src/main.rs:50-67) opens tauri.log unconditionally at process - # start, so no log means the binary never got that far, and the disposition - # line is the field the bug report turned on: a process that hangs before - # preflight must not pass. + # (src-tauri/src/main.rs:50-67) opens tauri.log unconditionally at process start, + # so no log means the binary never got that far, and the disposition line is the + # field the bug report turned on: a process that hangs before preflight must not + # pass. [ "${found:-0}" = "1" ] || { echo "::error::the app wrote no tauri.log; it never reached setup_logging"; exit 1; } [ "${disposition:-0}" = "1" ] || { echo "::error::tauri.log records no desktop_preflight disposition; the app never completed preflight"; exit 1; } - name: Restore the runner if: always() - # `|| true` swallowed everything, including a restore that genuinely broke. The - # file only exists once the strip step ran, and an earlier step can fail before - # that, so skip explicitly when it is absent and let a real failure surface. + # `|| true` swallowed everything, a genuinely broken restore included. The file + # only exists once the strip step ran, and an earlier step can fail before that, + # so skip explicitly when it is absent and let a real failure surface. run: | if [ -f .clean-machine/restore.sh ]; then bash .clean-machine/restore.sh @@ -281,9 +280,7 @@ jobs: # ── Linux: .deb and .AppImage, with a real webview under Xvfb ──────────── linux: - # A fork PR's token is read-only however this workflow declares permissions, so it - # cannot list the draft releases every desktop-v* bundle is published as. Skip - # rather than fail: it is a property of the trigger, not a broken release. + # See the macOS job: a fork PR's token cannot list drafts, so skip rather than fail. if: github.event.pull_request.head.repo.fork != true name: desktop linux ${{ matrix.kind }} runs-on: ubuntu-22.04 @@ -306,14 +303,12 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | mkdir -p dl logs - # Desktop releases are prereleases (never repo-wide "latest") and drafts, so - # the newest desktop-v* tag has to be resolved explicitly. See REL_TAG above. + # See the macOS job. Loud on purpose: no bundle means nothing to prove. if [ -z "$REL_TAG" ]; then REL_TAG="$(gh release list --repo "$REL_REPO" --limit 100 \ --json tagName,createdAt \ --jq '[.[] | select(.tagName | startswith("desktop-v"))] | sort_by(.createdAt) | reverse | .[0].tagName // empty')" - # Loud on purpose: there is no bundle to test, so passing would prove nothing. [ -n "$REL_TAG" ] || { echo "::error::no desktop-v* release visible in $REL_REPO -- either none has been cut, or this token cannot list drafts (needs contents: write)" exit 1 @@ -326,14 +321,14 @@ jobs: ls -la dl - name: Strip the developer toolchain - # Same gate as macOS: without it the Linux rows ignored strip_toolchain and ran - # the bundled installer with the runner's git, gcc, cmake and make in /usr/bin. + # Same gate as macOS: without it the Linux rows ignored strip_toolchain and ran the + # bundled installer with the runner's git, gcc, cmake and make in /usr/bin. # # BEFORE the bundle install, as macOS and Windows already do: dpkg runs the # package's own maintainer scripts, so installing first let them see the hosted # image's toolchain. Nothing in that install needs a masked tool -- - # clean-machine-env.sh moves aside only $TOOLS, leaving the package manager - # itself -- and the current bundle ships a postrm and no install-time script. + # clean-machine-env.sh moves aside only $TOOLS, leaving the package manager itself + # -- and the current bundle ships a postrm and no install-time script. if: ${{ github.event_name != 'workflow_dispatch' || inputs.strip_toolchain }} run: | bash .github/scripts/clean-machine-env.sh mask --remove @@ -342,9 +337,9 @@ jobs: - name: Install with NO dev tooling, only runtime libs run: | - # Deliberately not build-essential/cmake/git: a user installing a .deb has - # none of that. Xvfb and WebKit are runtime requirements, and apt pulls the - # .deb's declared deps, so a wrong dependency list fails here. + # Deliberately not build-essential/cmake/git: a user installing a .deb has none + # of that. Xvfb and WebKit are runtime requirements, and apt pulls the .deb's + # declared deps, so a wrong dependency list fails here. sudo apt-get update -qq sudo apt-get install -y -qq --no-install-recommends xvfb if [ "${{ matrix.kind }}" = "deb" ]; then @@ -362,12 +357,12 @@ jobs: echo "BIN=$BIN" >> "$GITHUB_ENV" echo "binary: $BIN" - # The strip runs before this, but `apt-get install ./dl/*.deb` then pulls the - # bundle's DECLARED dependencies, so a release that adds git, cmake or a compiler - # to that list puts one back in /usr/bin and both required Linux rows still pass. - # `absent` ran only beforehand, so re-run it here, before the bundled installer. - # The current dependency closure is 65 packages of runtime libs and no toolchain, - # so this is green today and only a new dependency can turn it red. + # The strip runs before this, but `apt-get install ./dl/*.deb` then pulls the bundle's + # DECLARED dependencies, so a release that adds git, cmake or a compiler to that list + # puts one back in /usr/bin and both required Linux rows still pass. `absent` ran only + # beforehand, so re-run it here, before the bundled installer. The current dependency + # closure is 65 packages of runtime libs and no toolchain, so this is green today and + # only a new dependency can turn it red. - name: Re-assert the toolchain is still absent after the package install if: ${{ github.event_name != 'workflow_dispatch' || inputs.strip_toolchain }} run: | @@ -379,39 +374,39 @@ jobs: set -a; [ -f ./clean-machine.env ] && . ./clean-machine.env; set +a set -o pipefail # The launch step below only proves the process stayed alive: on a fresh home - # preflight reports not_installed and the app waits on the install screen for - # a click (use-tauri-backend.ts:252-254, startup-screen.tsx:388-389), so a - # bundle whose embedded install.sh was missing or broken passed both Linux - # rows. tauri.conf.json:56-59 ships it as a bundle resource, so find it there - # and run it as install.rs does. + # preflight reports not_installed and the app waits on the install screen for a + # click (use-tauri-backend.ts:252-254, startup-screen.tsx:388-389), so a bundle + # whose embedded install.sh was missing or broken passed both Linux rows. + # tauri.conf.json:56-59 ships it as a bundle resource, so find it there and run + # it as install.rs does. if [ "${{ matrix.kind }}" = "deb" ]; then SH="$(dpkg -L "$(dpkg-deb -f dl/*.deb Package)" | grep -E '/install\.sh$' | head -1)" else - # ls returns a bare filename here, and a command word with no slash is - # resolved through PATH, not the cwd, so this needs the ./ prefix. + # ls returns a bare filename here, and a command word with no slash resolves + # through PATH, not the cwd, so this needs the ./ prefix. (cd dl && "./$(ls *.AppImage | head -1)" --appimage-extract >/dev/null) SH="$(find dl/squashfs-root -name install.sh -type f | head -1)" fi [ -n "$SH" ] && [ -f "$SH" ] || { echo "::error::the bundle ships no install.sh resource"; exit 1; } echo "bundled installer: $SH" - # KNOWN OUTCOME PIN, retire when the desktop release catches up to #7547. - # The bundle carries its own install.sh, and REL_TAG predates #7547, so on a - # stripped runner it still exits 2 at the NEED_SUDO handshake for the optional - # set instead of falling through to prebuilt llama.cpp. No change to this PR - # can move that; only a new release can. _SMART_APT_OPTIONAL is the guard #7547 - # added, so finding it means the release caught up and this pin must go. + # KNOWN OUTCOME PIN, retire when the desktop release catches up to #7547. The + # bundle carries its own install.sh and REL_TAG predates #7547, so on a stripped + # runner it still exits 2 at the NEED_SUDO handshake for the optional set instead + # of falling through to prebuilt llama.cpp. Only a new release can move that, not + # this PR. _SMART_APT_OPTIONAL is the guard #7547 added, so finding it means the + # release caught up and this pin must go. if grep -q '_SMART_APT_OPTIONAL' "$SH"; then echo "::error::the bundled install.sh now carries #7547; delete this pin block and let the venv + torch assertions below run unconditionally" exit 1 fi # --tauri rejects a custom studio home (install.sh:102-114), so drop the - # workspace-scoped override, and close stdin as install.rs does. + # workspace-scoped override; close stdin as install.rs does. rc=0 env -u UNSLOTH_STUDIO_HOME \ bash "$SH" --tauri < /dev/null 2>&1 | tee logs/bundled-install.log || rc=$? echo "bundled installer exit code: $rc" - # Exit code AND the exact optional set, so a different NEED_SUDO list or any - # other non-zero exit is still a failure. + # Exit code AND the exact optional set, so a different NEED_SUDO list or any other + # non-zero exit is still a failure. if [ "$rc" -eq 2 ] && grep -qE '^\[TAURI:NEED_SUDO\] cmake git build-essential libcurl4-openssl-dev[[:space:]]*$' logs/bundled-install.log; then echo "::notice::known pre-#7547 outcome: the shipped bundle's install.sh asked to elevate for the optional set and exited 2. Not a regression here; the next desktop release retires this pin." exit 0 @@ -429,9 +424,9 @@ jobs: - name: Launch under Xvfb and prove it stays up run: | set -a; [ -f ./clean-machine.env ] && . ./clean-machine.env; set +a - # Linux is the one platform where a hosted runner can give the app a real - # display, so this is the strongest "does the UI come up" check available - # without self-hosted hardware. + # Linux is the one platform where a hosted runner can give the app a real display, + # so this is the strongest "does the UI come up" check available without + # self-hosted hardware. xvfb-run -a --server-args="-screen 0 1440x900x24" \ "$BIN" > logs/app-stdout.log 2>&1 & APP_PID=$! @@ -459,20 +454,15 @@ jobs: found=1 if grep -qE "desktop_preflight completed disposition=" "$f"; then disposition=1; fi done - # Same acceptance criterion the macOS rows enforce. Everything above is - # `|| true` and the loop skips a missing log, so without these two lines the - # step could not fail. setup_logging (src-tauri/src/main.rs:50-67) opens - # tauri.log at process start, so no log means the binary never got that far, - # and the launch step only proves liveness: an app hanging before preflight - # completes would otherwise pass both Linux rows. + # Same acceptance criterion the macOS rows enforce, and for the same reason: + # everything above is `|| true` and the loop skips a missing log, so without + # these two lines the step could not fail. [ "${found:-0}" = "1" ] || { echo "::error::the app wrote no tauri.log; it never reached setup_logging"; exit 1; } [ "${disposition:-0}" = "1" ] || { echo "::error::tauri.log records no desktop_preflight disposition; the app never completed preflight"; exit 1; } - name: Restore the runner if: always() - # `|| true` swallowed everything, including a restore that genuinely broke. The - # file only exists once the strip step ran, and an earlier step can fail before - # that, so skip explicitly when it is absent and let a real failure surface. + # See the macOS job: `|| true` would swallow a genuinely broken restore. run: | if [ -f .clean-machine/restore.sh ]; then bash .clean-machine/restore.sh @@ -491,9 +481,7 @@ jobs: # ── Windows: NSIS setup.exe, silent install ────────────────────────────── windows: - # A fork PR's token is read-only however this workflow declares permissions, so it - # cannot list the draft releases every desktop-v* bundle is published as. Skip - # rather than fail: it is a property of the trigger, not a broken release. + # See the macOS job: a fork PR's token cannot list drafts, so skip rather than fail. if: github.event.pull_request.head.repo.fork != true name: desktop windows runs-on: windows-latest @@ -512,14 +500,12 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | mkdir -p dl logs - # Desktop releases are prereleases (never repo-wide "latest") and drafts, so - # the newest desktop-v* tag has to be resolved explicitly. See REL_TAG above. + # See the macOS job. Loud on purpose: no bundle means nothing to prove. if [ -z "$REL_TAG" ]; then REL_TAG="$(gh release list --repo "$REL_REPO" --limit 100 \ --json tagName,createdAt \ --jq '[.[] | select(.tagName | startswith("desktop-v"))] | sort_by(.createdAt) | reverse | .[0].tagName // empty')" - # Loud on purpose: there is no bundle to test, so passing would prove nothing. [ -n "$REL_TAG" ] || { echo "::error::no desktop-v* release visible in $REL_REPO -- either none has been cut, or this token cannot list drafts (needs contents: write)" exit 1 @@ -531,24 +517,21 @@ jobs: ls -la dl - name: Strip the developer toolchain - # `inputs` exists only for workflow_dispatch, so on pull_request and schedule - # `inputs.strip_toolchain` is '' -- and loose equality coerces both '' and - # false to 0, making `!= false` FALSE, so automatic runs would keep the hosted - # toolchain this workflow exists to remove. Gate on the event instead. + # Same gate as macOS, for the same `inputs`-coercion reason. if: ${{ github.event_name != 'workflow_dispatch' || inputs.strip_toolchain }} shell: pwsh run: | $drop = @('hostedtoolcache\windows\Python', 'WindowsApps', '\Git\', 'CMake', 'Microsoft Visual Studio', 'BuildTools', 'LLVM', 'MSYS', 'mingw') - # winget is an app-execution alias under ...\Local\Microsoft\WindowsApps, so - # the WindowsApps fragment -- there to take the Store's python.exe alias away - # -- drops the OS package manager with it. winget is not developer tooling; - # every consumer Windows machine this bundle ships to has it, and the bundled + # winget is an app-execution alias under ...\Local\Microsoft\WindowsApps, so the + # WindowsApps fragment -- there to take the Store's python.exe alias away -- + # drops the OS package manager with it. winget is not developer tooling: every + # consumer Windows machine this bundle ships to has it, and the bundled # install.ps1 reaches for it for the git that studio/setup.ps1:1657-1669 still - # gates on unconditionally. Without it this lane only re-runs the no-winget - # fallback that clean-machine-install-ci.yml already covers and pins on its - # winget=masked row, and it does so as a hard failure. Resolve winget before - # the scrub and hand it back through a shim, exactly as that workflow does. + # gates on unconditionally. Without it this lane only re-runs, as a hard failure, + # the no-winget fallback clean-machine-install-ci.yml already covers and pins on + # its winget=masked row. Resolve winget before the scrub and hand it back through + # a shim, exactly as that workflow does. $wingetCmd = Get-Command winget -ErrorAction SilentlyContinue if (-not $wingetCmd) { Write-Host '::error::winget was not on PATH before the strip; this image ships it and the bundled installer needs it' @@ -564,9 +547,8 @@ jobs: } "PATH=$shim;$((& $scrub ($env:PATH -split ';')) -join ';')" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 - # Take the toolcache Python off disk, not just off PATH: py.exe lives in - # C:\Windows (which must stay) and uv does its own interpreter discovery, - # so both reach the toolcache whatever PATH says. + # Off disk, not just off PATH: py.exe lives in C:\Windows (which must stay) and + # uv does its own discovery, so both reach the toolcache whatever PATH says. foreach ($tc in @("$env:AGENT_TOOLSDIRECTORY\Python", 'C:\hostedtoolcache\windows\Python')) { if ($tc -and (Test-Path $tc)) { try { Rename-Item -LiteralPath $tc -NewName 'Python.masked' -ErrorAction Stop @@ -574,13 +556,12 @@ jobs: catch { Write-Host "::error::could not mask $tc ($($_.Exception.Message)); the job would not be clean"; exit 1 } } } - # The bundled install.ps1 this job runs calls Refresh-SessionPath (318-337), - # which merges the Machine and User registry PATHs back into $env:Path, so a - # process-only scrub lasts until the first refresh and Git/CMake/VS/LLVM come - # back from the registry. The runner is ephemeral, so rewrite the registry - # copies too. (A merge keeps what the process already had, which is why the - # winget shim above survives.) Expand - # first: SetEnvironmentVariable rewrites REG_EXPAND_SZ as REG_SZ + # The bundled install.ps1 this job runs calls Refresh-SessionPath (318-337), which + # merges the Machine and User registry PATHs back into $env:Path, so a + # process-only scrub lasts until the first refresh and Git/CMake/VS/LLVM come back + # from the registry. The runner is ephemeral, so rewrite the registry copies too. + # (A merge keeps what the process already had, which is why the winget shim above + # survives.) Expand first: SetEnvironmentVariable rewrites REG_EXPAND_SZ as REG_SZ # (dotnet/runtime#1442). foreach ($scope in 'Machine','User') { $raw = [System.Environment]::GetEnvironmentVariable('Path', $scope) @@ -599,11 +580,11 @@ jobs: exit 0 - name: Verify the strip took effect - # PATH written to $GITHUB_ENV only applies to LATER steps, so the scrub can - # only be checked from here. The drop list above is heuristic path-fragment - # matching: if a runner image moves any of these tools outside those fragments, - # the bundled install.ps1 reuses the survivor and this job still calls itself - # clean. Same assertion the installer workflow runs, same reason. + # PATH written to $GITHUB_ENV only applies to LATER steps, so the scrub can only be + # checked from here. The drop list above is heuristic path-fragment matching: if a + # runner image moves any of these tools outside those fragments, the bundled + # install.ps1 reuses the survivor and this job still calls itself clean. Same + # assertion the installer workflow runs, same reason. if: ${{ github.event_name != 'workflow_dispatch' || inputs.strip_toolchain }} shell: pwsh run: | @@ -615,8 +596,8 @@ jobs: } # `py` itself lives in C:\Windows and stays. Only an interpreter it can still # START is a leak, because Find-CompatiblePython (install.ps1:1130-1153) probes - # `py` first. `py -0p` is just the launcher's REGISTRY view, which still names - # the paths the rename removed, so a start attempt is the only real evidence. + # `py` first. `py -0p` is only the launcher's REGISTRY view, which still names the + # paths the rename removed, so a start attempt is the only real evidence. if (Get-Command py -ErrorAction SilentlyContinue) { foreach ($v in '-3.11', '-3.12', '-3.13') { $out = & py $v -c "import sys; print(sys.executable)" 2>&1 @@ -624,16 +605,15 @@ jobs: Write-Host ("py {0} -> exit {1}: {2}" -f $v, $rc, (($out | Out-String).Trim() -replace '\r?\n', ' / ')) if ($rc -eq 0) { $leaked += "py $v -> $out" } } - # A failing probe is the outcome we want, but it leaves $LASTEXITCODE - # non-zero and the runner appends `exit $LASTEXITCODE` to every pwsh step - # (actions/runner#351), so the step would exit 1 with nothing printed on a - # machine that is in fact clean. + # A failing probe is the outcome we want, but it leaves $LASTEXITCODE non-zero + # and the runner appends `exit $LASTEXITCODE` to every pwsh step + # (actions/runner#351), so the step would exit 1 on a machine that is clean. $global:LASTEXITCODE = 0 } # The shim is the only reason winget resolves after the WindowsApps drop. It # survives the installer's own refreshes because Refresh-SessionPath # (install.ps1:318-337) and setup.ps1's Refresh-Environment MERGE the current - # $env:Path back in rather than replace it -- but assert it, or this lane + # $env:Path back in rather than replace it -- but assert that, or this lane # silently degrades into the no-winget leg the installer workflow already pins. $winget = Get-Command winget -ErrorAction SilentlyContinue Write-Host ("winget {0}" -f $(if ($winget) { $winget.Source } else { 'ABSENT' })) @@ -651,8 +631,8 @@ jobs: shell: pwsh run: | $exe = (Get-ChildItem dl/*setup.exe | Select-Object -First 1).FullName - # /S is the NSIS silent switch: a user double-clicks, but an installer that - # cannot run unattended cannot be scripted or MDM-deployed either. + # /S is the NSIS silent switch: a user double-clicks, but an installer that cannot + # run unattended cannot be scripted or MDM-deployed either. $p = Start-Process -FilePath $exe -ArgumentList '/S' -Wait -PassThru Write-Host "installer exit: $($p.ExitCode)" if ($p.ExitCode -ne 0) { Write-Host "::error::silent install failed"; exit 1 } @@ -666,8 +646,8 @@ jobs: - name: Run the bundled installer, the path first launch takes shell: pwsh run: | - # The launch step below only proves the process stayed alive: on a fresh - # profile the app waits for a click on Install (use-tauri-backend.ts:252-254, + # The launch step below only proves the process stayed alive: on a fresh profile + # the app waits for a click on Install (use-tauri-backend.ts:252-254, # startup-screen.tsx:388-389), so this job passed on a bundle whose embedded # install.ps1 was missing or broken. tauri.conf.json:56-59 ships it as a bundle # resource, so find it where NSIS put it and invoke it as install.rs:326-341. @@ -697,8 +677,8 @@ jobs: exit 1 } & $py -V - # install.rs passes only --tauri, so torch is part of first launch, and a venv - # that cannot import it is the unbootable environment from the report. + # install.rs passes only --tauri, so torch is part of first launch, and a venv that + # cannot import it is the unbootable environment from the report. & $py -c "import torch; print('torch', torch.__version__)" if ($LASTEXITCODE -ne 0) { Write-Host '::error::the bundled install produced a venv with no working torch' @@ -738,11 +718,8 @@ jobs: -SimpleMatch -Quiet) { $disposition = $true } } } - # Same acceptance criterion macOS and Linux enforce. Test-Path, Get-Content and + # Same acceptance criterion macOS and Linux enforce: Test-Path, Get-Content and # Select-String cannot fail, so without these two lines the step was decoration. - # setup_logging (src-tauri/src/main.rs:50-67) opens tauri.log unconditionally at - # process start, so no log means the binary never got that far, and an app that - # hangs before preflight would otherwise pass. if (-not $found) { Write-Host '::error::the app wrote no tauri.log; it never reached setup_logging' exit 1 diff --git a/install.ps1 b/install.ps1 index a26de28a19..11de4aec5d 100644 --- a/install.ps1 +++ b/install.ps1 @@ -2766,18 +2766,17 @@ exit 0 # text, ignored unless UNSLOTH_CI_SOURCE_OVERLAY names a directory holding a # pyproject.toml. # - # The clean-machine legs run THIS script from a branch, but it installs - # unsloth from PyPI, the consumer path, so everything Python-side comes out - # of the released wheel (studio/setup.ps1, install_python_stack.py and every - # requirements/constraints file they reach via Path(__file__)) and the - # workflow meant to validate a branch could not. `& $UnslothExe studio setup` - # below goes through the CLI, and an editable overlay makes _PACKAGE_ROOT in - # unsloth_cli/commands/studio.py resolve to the working tree by PEP 660 - # __file__, so setup.ps1 comes from the branch unchanged. NOT --local: that - # also installs `unsloth-zoo @ git+https://github.com/unslothai/unsloth-zoo`, - # which genuinely needs git, and git absence is what the masked leg proves. - # Editable + --no-deps resolves nothing and clones nothing, so it survives - # git, cmake and MSVC all missing. + # The clean-machine legs run THIS script from a branch but install unsloth + # from PyPI, the consumer path, so everything Python-side (studio/setup.ps1, + # install_python_stack.py and every requirements/constraints file they reach + # via Path(__file__)) would be the released wheel's and a branch could not be + # validated. `& $UnslothExe studio setup` below goes through the CLI, and an + # editable overlay makes _PACKAGE_ROOT in unsloth_cli/commands/studio.py + # resolve to the working tree by PEP 660 __file__, so setup.ps1 comes from + # this ref. NOT --local: that also installs `unsloth-zoo @ + # git+https://github.com/unslothai/unsloth-zoo`, which genuinely needs git, + # and git absence is what the masked leg proves; editable + --no-deps + # resolves and clones nothing, so it survives git, cmake and MSVC all missing. if ($env:UNSLOTH_CI_SOURCE_OVERLAY) { $CiOverlayRoot = $env:UNSLOTH_CI_SOURCE_OVERLAY if (-not (Test-Path -LiteralPath (Join-Path $CiOverlayRoot "pyproject.toml"))) { @@ -2785,8 +2784,8 @@ exit 0 return (Exit-InstallFailure "UNSLOTH_CI_SOURCE_OVERLAY has no pyproject.toml: $CiOverlayRoot") } substep "CI: overlaying source checkout (editable, no deps): $CiOverlayRoot" - # Retry: the editable build downloads its pinned build backend from PyPI, - # so it carries the same transient-network risk as every other step. + # Retry: the editable build fetches its build backend from PyPI, same + # transient-network risk as every other step. $CiOverlayExit = Invoke-InstallCommandRetry -Label "overlay CI source checkout" -Command { uv pip install --python $VenvPython --no-deps -e $CiOverlayRoot } if ($CiOverlayExit -ne 0) { return (Exit-InstallFailure "Failed to overlay the CI source checkout (exit code $CiOverlayExit)" $CiOverlayExit) diff --git a/install.sh b/install.sh index 28975a4060..d474a6becf 100755 --- a/install.sh +++ b/install.sh @@ -4189,16 +4189,15 @@ fi # Not a consumer knob: no flag, absent from --help, ignored unless # UNSLOTH_CI_SOURCE_OVERLAY names a directory holding a pyproject.toml. # -# The clean-machine legs run THIS script from a branch, but it installs unsloth -# from PyPI, the consumer path. Everything Python-side then comes out of the -# released wheel (studio/setup.sh, setup.ps1, install_python_stack.py and every -# requirements/constraints file they reach via Path(__file__)), so the workflow -# meant to validate a branch could not. An editable overlay re-points -# `import studio` at the working tree, and the importlib.resources lookup below -# then finds the branch's setup.sh unchanged. NOT --local: that also installs -# `unsloth-zoo @ git+https://github.com/unslothai/unsloth-zoo`, which genuinely -# needs git, and git absence is what these legs prove. Editable + --no-deps -# resolves nothing and clones nothing, so it survives git, cmake and the C/C++ +# The clean-machine legs run THIS script from a branch but install unsloth from +# PyPI, the consumer path, so everything Python-side (studio/setup.sh, setup.ps1, +# install_python_stack.py and every requirements/constraints file they reach via +# Path(__file__)) would be the released wheel's and a branch could not be +# validated. An editable overlay re-points `import studio` at the working tree, so +# the importlib.resources lookup below finds this ref's setup.sh. NOT --local: +# that also installs `unsloth-zoo @ git+https://github.com/unslothai/unsloth-zoo`, +# which genuinely needs git, and git absence is what these legs prove; editable + +# --no-deps resolves and clones nothing, so it survives git, cmake and the C/C++ # compilers all being gone. if [ -n "${UNSLOTH_CI_SOURCE_OVERLAY:-}" ]; then if [ ! -f "$UNSLOTH_CI_SOURCE_OVERLAY/pyproject.toml" ]; then @@ -4206,8 +4205,8 @@ if [ -n "${UNSLOTH_CI_SOURCE_OVERLAY:-}" ]; then exit 1 fi substep "CI: overlaying source checkout (editable, no deps): $UNSLOTH_CI_SOURCE_OVERLAY" - # Retry: the editable build downloads its pinned build backend from PyPI, so - # it carries the same transient-network risk as every other install step. + # Retry: the editable build fetches its build backend from PyPI, same + # transient-network risk as every other install step. run_install_cmd_retry "overlay CI source checkout" uv pip install --python "$_VENV_PY" \ --no-deps -e "$UNSLOTH_CI_SOURCE_OVERLAY" fi diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py index c3b80cdee3..bac5773625 100644 --- a/studio/install_python_stack.py +++ b/studio/install_python_stack.py @@ -2790,8 +2790,7 @@ def pip_install_try( env = _install_env_for_cmd(cmd), ) if result.returncode == 0: - # Same reasoning as pip_install below: `nobuild` can only catch a source - # build that reaches the log. + # As pip_install below: `nobuild` only catches a build that reaches the log. if VERBOSE and result.stdout: print(_redact_install_output(result.stdout)) return True @@ -2849,14 +2848,13 @@ def pip_install( **_windows_hidden_subprocess_kwargs(), ) if result.returncode == 0: - # Echo successful output under UNSLOTH_VERBOSE, as install.sh's + # Echo success under UNSLOTH_VERBOSE, as install.sh's # run_install_cmd does. Without it the dependency phase never - # reached the install log, and clean-machine-assert.sh's `nobuild` - # greps that log for uv's "Building ==" -- so a source - # build in this step, the one installing studio.txt where an - # sdist-only dependency actually shows up, reported "built: none" - # and the leg stayed green. Redacted: uv echoes index URLs with - # credentials. + # reached the install log that clean-machine-assert.sh's `nobuild` + # greps for uv's "Building ==", so a source build in this + # step -- the studio.txt install, where sdist-only dependencies + # actually show up -- reported "built: none" and stayed green. + # Redacted: uv echoes index URLs with credentials. if VERBOSE and result.stdout: print(_redact_install_output(result.stdout)) return