CI: prove the installer works on a machine with no developer toolchain
No job has ever run the installer on a machine without one. studio-mac-install-matrix.yml is the only macOS installer job and it runs 'bash install.sh --local --no-torch' on runners that already have the Xcode CLT selected and setup-python preinstalled, so the CLT gate never fires there, and --local is precisely the mode that legitimately needs git. Repo-wide there was zero coverage of xcode-select or CommandLineTools outside install.sh itself. clean-machine-install-ci.yml runs the installer on a genuinely stripped machine. macOS legs move /var/db/xcode_select_link, /Library/Developer/CommandLineTools, /Applications/Xcode*.app and Homebrew aside, so xcode-select -p, git, cc and clang really do fail, and restore unconditionally afterwards. Removing the select-link alone is not enough: xcode-select falls through to a full Xcode.app and re-arms /usr/bin/git. Linux legs use containers, which are genuinely clean. Windows legs cover winget visible and masked, plus windows-11-arm. A WSL leg covers the 126 lines of WSL-specific install.sh logic that had no runtime test. Each macOS leg runs four deliveries: pipe (the advertised command, and the shape that turns an early exit into curl (56)), file (separates installer logic from pipe delivery), no-torch, and tauri (stdin closed, no tty, as the desktop app invokes it). One leg records every toolchain invocation and asserts the trace, which is the real deliverable: proof the installer never reached for a compiler rather than proof it happened to succeed. The asserts test that tools do NOT WORK rather than that they are absent from PATH. On a real virgin Mac /usr/bin/git and /usr/bin/cc exist as CLT stubs, so 'command -v git' succeeds and only running it tells the truth. desktop-app-clean-machine-ci.yml installs and launches the SHIPPED desktop app release on a stripped machine, covering Gatekeeper and quarantine on macOS, NSIS silent install on Windows, and Xvfb with WebKit2GTK on Linux. Known limit, stated plainly: hosted macOS runners are developer machines. Masking reproduces this bug and proves the installer does not invoke a toolchain, but it cannot prove no hidden dependency exists on a truly virgin Mac. An ephemeral-VM lane is the follow-up.
This commit is contained in:
Daniel Han
parent
af2439683a
commit
c5b4f5ee86
4 changed files with 1174 additions and 0 deletions
127
.github/scripts/clean-machine-assert.sh
vendored
Executable file
127
.github/scripts/clean-machine-assert.sh
vendored
Executable file
|
|
@ -0,0 +1,127 @@
|
|||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
|
||||
#
|
||||
# Assert the clean-machine contract after an install attempt.
|
||||
#
|
||||
# absent The toolchain really was absent for the whole run. Guards against a
|
||||
# leg that "passed" only because masking silently failed, or because
|
||||
# the installer quietly installed Xcode CLT behind our back.
|
||||
# notools The trace recorded no compiler/git/brew invocation (trace mode).
|
||||
# nobuild The install log shows no source build (no sdist, no cmake, no
|
||||
# "Building wheel"). This is the wheels-only contract.
|
||||
#
|
||||
# Usage: bash .github/scripts/clean-machine-assert.sh absent notools nobuild
|
||||
set -uo pipefail
|
||||
|
||||
LOG="${INSTALL_LOG:-logs/install.log}"
|
||||
TRACE="${UNSLOTH_TOOL_TRACE:-}"
|
||||
rc=0
|
||||
|
||||
fail() { echo "::error::$*"; rc=1; }
|
||||
ok() { echo "[assert] OK $*"; }
|
||||
|
||||
for check in "$@"; do
|
||||
case "$check" in
|
||||
|
||||
absent)
|
||||
# Deliberately NOT a `command -v` check. On a real virgin Mac /usr/bin/git and
|
||||
# /usr/bin/cc EXIST as Xcode CLT stubs, so `command -v git` SUCCEEDS -- running
|
||||
# it is what fails ("xcrun: error: invalid active developer path"). Asserting on
|
||||
# `command -v` would therefore be unfaithful and would fail on a correctly masked
|
||||
# runner. The honest invariant is: the tool must not WORK.
|
||||
if xcode-select -p >/dev/null 2>&1; then
|
||||
fail "xcode-select -p still resolves to $(xcode-select -p 2>/dev/null); not a clean Mac"
|
||||
else
|
||||
ok "xcode-select -p fails (the gate a virgin Mac hits)"
|
||||
fi
|
||||
for tool in git cc clang cmake; do
|
||||
command -v "$tool" >/dev/null 2>&1 || { ok "$tool not on PATH"; continue; }
|
||||
if "$tool" --version >/dev/null 2>&1; then
|
||||
fail "toolchain still usable: '$tool --version' succeeded ($(command -v "$tool")); masking failed"
|
||||
else
|
||||
ok "$tool present but non-functional (CLT stub), as on a clean Mac"
|
||||
fi
|
||||
done
|
||||
# brew is a plain binary with no stub, so absence from PATH is the right test.
|
||||
if command -v brew >/dev/null 2>&1; then
|
||||
fail "Homebrew still on PATH at $(command -v brew); masking failed"
|
||||
else
|
||||
ok "brew absent"
|
||||
fi
|
||||
;;
|
||||
|
||||
notools)
|
||||
if [ -z "$TRACE" ] || [ ! -f "$TRACE" ]; then
|
||||
fail "notools requested but no trace file (\$UNSLOTH_TOOL_TRACE=$TRACE)"
|
||||
else
|
||||
# git is legitimate under --local (it installs unsloth-zoo from a git URL);
|
||||
# UNSLOTH_ALLOW_TOOLS lets that leg allow-list it explicitly.
|
||||
allow="${UNSLOTH_ALLOW_TOOLS:-}"
|
||||
hits=""
|
||||
while IFS=$'\t' read -r tool rest; do
|
||||
[ -n "$tool" ] || continue
|
||||
case " $allow " in *" $tool "*) continue ;; esac
|
||||
# `xcode-select -p` ASKS whether a toolchain is selected; it cannot build
|
||||
# anything. The installer has to ask in order to tell the user whether a
|
||||
# source build is available, and the whole point of the fix is that it then
|
||||
# carries on without one. Treating the question as toolchain USE would fail
|
||||
# the very leg that proves the toolchain was never used. `--install`, which
|
||||
# pops the CLT installer, stays a hit.
|
||||
if [ "$tool" = "xcode-select" ]; then
|
||||
case "$rest" in
|
||||
-p|--print-path|-v|--version|"") continue ;;
|
||||
esac
|
||||
fi
|
||||
hits="$hits $tool"
|
||||
done < "$TRACE"
|
||||
if [ -n "$hits" ]; then
|
||||
fail "installer invoked toolchain:$(echo "$hits" | tr ' ' '\n' | sort -u | tr '\n' ' ')"
|
||||
echo "---- tool trace ----"; sort -u "$TRACE" | head -50
|
||||
else
|
||||
ok "no compiler/git/brew invocation recorded"
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
|
||||
nobuild)
|
||||
# "Built an sdist" is NOT the same as "needed a compiler". Four packages on the
|
||||
# macOS path are sdist-only PURE PYTHON projects that build fine with no
|
||||
# toolchain (verified by resolving each against cp313/macos-arm64):
|
||||
# openai-whisper, argbind, randomname -- no version ever ships a wheel
|
||||
# antlr4-python3-runtime==4.9.3 -- pinned below the 4.13.2 wheel
|
||||
# Failing on those would be a false alarm, so the contract asserted here is
|
||||
# "nothing that needs a COMPILER was built", with that allowlist subtracted.
|
||||
# UNSLOTH_ALLOW_SDIST can extend it.
|
||||
_allow="openai-whisper argbind randomname antlr4-python3-runtime ${UNSLOTH_ALLOW_SDIST:-}"
|
||||
if [ ! -f "$LOG" ]; then
|
||||
fail "nobuild requested but $LOG is missing"
|
||||
else
|
||||
_built="$(grep -oiE "building wheel for [a-z0-9._-]+" "$LOG" 2>/dev/null \
|
||||
| sed -E 's/.* for //' | tr 'A-Z' 'a-z' | sort -u || true)"
|
||||
_bad=""
|
||||
for pkg in $_built; do
|
||||
case " $_allow " in *" $pkg "*) continue ;; esac
|
||||
_bad="$_bad $pkg"
|
||||
done
|
||||
if [ -n "$_bad" ]; then
|
||||
fail "built from source:$_bad -- these must resolve to wheels on a clean machine"
|
||||
else
|
||||
[ -n "$_built" ] && say_built="$(echo "$_built" | tr '\n' ' ')" || say_built="none"
|
||||
ok "no non-allowlisted source build (built: $say_built)"
|
||||
fi
|
||||
# Independent of package names: a compiler error means a toolchain was needed.
|
||||
if grep -qiE "error: command '(cc|gcc|clang|cl)' failed|no such file or directory: 'cc'|clang: error|cargo: not found|error: linker \`cc\` not found" "$LOG"; then
|
||||
fail "compiler invocation appears in the install log"
|
||||
grep -iE "error: command '(cc|gcc|clang|cl)' failed|clang: error" "$LOG" | head -10
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
|
||||
*)
|
||||
fail "unknown check '$check'"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
exit "$rc"
|
||||
165
.github/scripts/clean-machine-env.sh
vendored
Executable file
165
.github/scripts/clean-machine-env.sh
vendored
Executable file
|
|
@ -0,0 +1,165 @@
|
|||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
|
||||
#
|
||||
# Simulate a virgin developer machine on a GitHub-hosted runner, so the installer
|
||||
# is exercised the way a real user's brand-new Mac / PC exercises it.
|
||||
#
|
||||
# Two modes, because "the tool is absent" and "the installer never called the tool"
|
||||
# CANNOT be simulated by the same mechanism:
|
||||
#
|
||||
# mask Make the toolchain genuinely ABSENT. Scrubs PATH down to the OS
|
||||
# defaults and (with --remove) moves the real toolchain aside. After
|
||||
# this, `command -v git` correctly FAILS, which is what a clean Mac does.
|
||||
# A failing "poison shim" on PATH would do the opposite -- `command -v`
|
||||
# finds it and reports the tool as present -- so shims are NOT used here.
|
||||
#
|
||||
# trace Leave the toolchain working, but route it through logging wrappers that
|
||||
# record the invocation and then exec the real binary. Proves whether the
|
||||
# installer ever REACHES for a compiler/git, without changing behaviour.
|
||||
#
|
||||
# Writes shell exports to $CLEAN_ENV_FILE (default ./clean-machine.env) for the
|
||||
# caller to `source`. Nothing is exported globally, so other workflow steps
|
||||
# (checkout, upload-artifact) keep a normal environment.
|
||||
#
|
||||
# Usage:
|
||||
# bash .github/scripts/clean-machine-env.sh mask [--remove]
|
||||
# bash .github/scripts/clean-machine-env.sh trace
|
||||
# source ./clean-machine.env
|
||||
set -uo pipefail
|
||||
|
||||
MODE="${1:-}"
|
||||
REMOVE=0
|
||||
[ "${2:-}" = "--remove" ] && REMOVE=1
|
||||
|
||||
case "$MODE" in
|
||||
mask|trace) ;;
|
||||
*) echo "usage: $0 {mask|trace} [--remove]" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
OS="$(uname -s)"
|
||||
WORK="${CLEAN_MACHINE_DIR:-$PWD/.clean-machine}"
|
||||
ENV_FILE="${CLEAN_ENV_FILE:-$PWD/clean-machine.env}"
|
||||
TRACE="$WORK/tool-invocations.log"
|
||||
BIN="$WORK/bin"
|
||||
RESTORE="$WORK/restore.sh"
|
||||
mkdir -p "$BIN"
|
||||
: > "$TRACE"
|
||||
: > "$ENV_FILE"
|
||||
printf '#!/usr/bin/env bash\n# Undo clean-machine-env.sh --remove. Safe to run twice.\nset -uo pipefail\n' > "$RESTORE"
|
||||
chmod +x "$RESTORE"
|
||||
|
||||
# The toolchain we care about: a consumer install must need none of it.
|
||||
TOOLS="xcode-select xcrun clang clang++ cc c++ gcc g++ git cmake make brew ninja cargo rustc"
|
||||
|
||||
note() { echo "[clean-machine] $*"; }
|
||||
|
||||
# ── PATH scrub ────────────────────────────────────────────────────────────────
|
||||
# Keep only OS-default system dirs. Drops Homebrew, the hosted Python toolcache,
|
||||
# setup-* shims, pipx, cargo, and every other preinstalled developer dir.
|
||||
scrub_path() {
|
||||
local keep out=""
|
||||
if [ "$OS" = "Darwin" ]; then
|
||||
keep="/usr/bin:/bin:/usr/sbin:/sbin"
|
||||
else
|
||||
keep="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
fi
|
||||
local IFS=":"
|
||||
for d in $keep; do
|
||||
[ -d "$d" ] && out="${out:+$out:}$d"
|
||||
done
|
||||
echo "$out"
|
||||
}
|
||||
|
||||
# ── mask ──────────────────────────────────────────────────────────────────────
|
||||
if [ "$MODE" = "mask" ]; then
|
||||
NEWPATH="$(scrub_path)"
|
||||
{
|
||||
echo "export PATH='$NEWPATH'"
|
||||
# DEVELOPER_DIR must be UNSET, not pointed at a fake path: `xcode-select -p`
|
||||
# honours DEVELOPER_DIR and prints it verbatim with exit 0, so setting it to a
|
||||
# nonexistent dir makes the probe SUCCEED -- the exact opposite of a clean Mac,
|
||||
# where DEVELOPER_DIR is unset and the missing /var/db/xcode_select_link is what
|
||||
# makes `xcode-select -p` fail.
|
||||
echo "unset DEVELOPER_DIR || true"
|
||||
echo "unset SDKROOT CC CXX CFLAGS CXXFLAGS LDFLAGS CMAKE_GENERATOR CMAKE_PREFIX_PATH || true"
|
||||
echo "export HOMEBREW_NO_AUTO_UPDATE=1"
|
||||
echo "export UNSLOTH_CLEAN_MACHINE=1"
|
||||
} >> "$ENV_FILE"
|
||||
|
||||
if [ "$REMOVE" = "1" ] && [ "$OS" = "Darwin" ]; then
|
||||
# Best-effort real removal. Each step is independent and recorded in
|
||||
# restore.sh so an `if: always()` step can put the runner back.
|
||||
# /var/db/xcode_select_link is exactly what `xcode-select -p` reads, so
|
||||
# removing it reproduces a virgin Mac's gate precisely. `xcode-select --reset`
|
||||
# is NOT enough: it can reselect a full Xcode.app.
|
||||
if [ -e /var/db/xcode_select_link ]; then
|
||||
if sudo rm -f /var/db/xcode_select_link 2>/dev/null; then
|
||||
note "removed /var/db/xcode_select_link"
|
||||
echo "sudo xcode-select --switch /Library/Developer/CommandLineTools 2>/dev/null || true" >> "$RESTORE"
|
||||
else
|
||||
note "WARN could not remove /var/db/xcode_select_link"
|
||||
fi
|
||||
fi
|
||||
# Moving the CLT dir aside turns /usr/bin/{cc,clang,git} into dead shims, so
|
||||
# the run also proves the install needs no compiler at all.
|
||||
if [ -d /Library/Developer/CommandLineTools ]; then
|
||||
if sudo mv /Library/Developer/CommandLineTools /Library/Developer/CommandLineTools.masked 2>/dev/null; then
|
||||
note "moved CommandLineTools aside"
|
||||
echo "sudo mv /Library/Developer/CommandLineTools.masked /Library/Developer/CommandLineTools 2>/dev/null || true" >> "$RESTORE"
|
||||
else
|
||||
note "WARN could not move CommandLineTools"
|
||||
fi
|
||||
fi
|
||||
# Xcode.app must go too. With the select link removed AND CommandLineTools moved,
|
||||
# `xcode-select -p` does not fail -- it falls through to whatever Xcode bundle the
|
||||
# runner image ships (observed: /Applications/Xcode_16.4.app/Contents/Developer),
|
||||
# which re-arms /usr/bin/git and /usr/bin/cc and silently un-cleans the machine.
|
||||
# A rename is instant regardless of bundle size: same filesystem, no copy.
|
||||
for app in /Applications/Xcode*.app; do
|
||||
[ -d "$app" ] || continue
|
||||
if sudo mv "$app" "${app}.masked" 2>/dev/null; then
|
||||
note "moved $(basename "$app") aside"
|
||||
echo "sudo mv '${app}.masked' '$app' 2>/dev/null || true" >> "$RESTORE"
|
||||
else
|
||||
note "WARN could not move $app"
|
||||
fi
|
||||
done
|
||||
for brewdir in /opt/homebrew /usr/local/Homebrew; do
|
||||
if [ -d "$brewdir" ]; then
|
||||
if sudo mv "$brewdir" "${brewdir}.masked" 2>/dev/null; then
|
||||
note "moved $brewdir aside"
|
||||
echo "sudo mv '${brewdir}.masked' '$brewdir' 2>/dev/null || true" >> "$RESTORE"
|
||||
else
|
||||
note "WARN could not move $brewdir"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── trace ─────────────────────────────────────────────────────────────────────
|
||||
if [ "$MODE" = "trace" ]; then
|
||||
for tool in $TOOLS; do
|
||||
real="$(command -v "$tool" 2>/dev/null || true)"
|
||||
[ -n "$real" ] || continue
|
||||
# Wrapper logs the call then execs the REAL binary, so behaviour is unchanged
|
||||
# and the trace answers "did the installer reach for this?" honestly.
|
||||
cat > "$BIN/$tool" <<WRAP
|
||||
#!/bin/sh
|
||||
printf '%s\t%s\n' "$tool" "\$*" >> "$TRACE"
|
||||
exec "$real" "\$@"
|
||||
WRAP
|
||||
chmod +x "$BIN/$tool"
|
||||
done
|
||||
{
|
||||
echo "export PATH='$BIN:$PATH'"
|
||||
echo "export UNSLOTH_TOOL_TRACE='$TRACE'"
|
||||
echo "export UNSLOTH_CLEAN_MACHINE=trace"
|
||||
} >> "$ENV_FILE"
|
||||
fi
|
||||
|
||||
note "mode=$MODE remove=$REMOVE"
|
||||
note "env file: $ENV_FILE"
|
||||
note "trace: $TRACE"
|
||||
note "restore: $RESTORE"
|
||||
Loading…
Add table
Add a link
Reference in a new issue