Tighten the clean-machine comments
Compress the comment blocks across the clean-machine workflows and scripts. The explanations of why each check is written the way it is stay; the padding, restatement and duplication go. No code or workflow logic changes.
This commit is contained in:
danielhanchen
parent
90ec9462a0
commit
231fcc3cf0
4 changed files with 224 additions and 253 deletions
68
.github/scripts/clean-machine-assert.sh
vendored
68
.github/scripts/clean-machine-assert.sh
vendored
|
|
@ -4,15 +4,14 @@
|
|||
#
|
||||
# Assert the clean-machine contract after an install attempt.
|
||||
#
|
||||
# absent The toolchain really was absent for the whole run. Guards against a
|
||||
# leg that "passed" only because masking silently failed, or because
|
||||
# the installer quietly installed Xcode CLT behind our back.
|
||||
# absent The toolchain really was absent for the whole run. Guards against a leg
|
||||
# that "passed" only because masking silently failed, or because the
|
||||
# installer quietly installed Xcode CLT behind our back.
|
||||
# notools The trace recorded no compiler/git/brew invocation (trace mode).
|
||||
# nobuild The install log shows no source build (no sdist, no cmake, no
|
||||
# "Building wheel" from pip and no "Building <pkg>==<ver>" from uv).
|
||||
# This is the wheels-only contract. It needs UNSLOTH_VERBOSE=1 on the
|
||||
# installer, otherwise run_install_cmd (install.sh:193-243) throws the
|
||||
# uv output away on success and there is nothing here to read.
|
||||
# nobuild The wheels-only contract: no "Building wheel" from pip, no
|
||||
# "Building <pkg>==<ver>" from uv. Needs UNSLOTH_VERBOSE=1, else
|
||||
# run_install_cmd (install.sh:193-243) discards the uv output on success
|
||||
# and there is nothing here to read.
|
||||
#
|
||||
# Usage: bash .github/scripts/clean-machine-assert.sh absent notools nobuild
|
||||
set -uo pipefail
|
||||
|
|
@ -28,11 +27,9 @@ for check in "$@"; do
|
|||
case "$check" in
|
||||
|
||||
absent)
|
||||
# Deliberately NOT a `command -v` check. On a real virgin Mac /usr/bin/git and
|
||||
# /usr/bin/cc EXIST as Xcode CLT stubs, so `command -v git` SUCCEEDS -- running
|
||||
# it is what fails ("xcrun: error: invalid active developer path"). Asserting on
|
||||
# `command -v` would therefore be unfaithful and would fail on a correctly masked
|
||||
# runner. The honest invariant is: the tool must not WORK.
|
||||
# Deliberately NOT `command -v`: on a virgin Mac /usr/bin/{git,cc} EXIST as CLT
|
||||
# stubs, so `command -v` succeeds and only RUNNING them fails ("xcrun: error:
|
||||
# invalid active developer path"). The honest invariant is: must not WORK.
|
||||
if xcode-select -p >/dev/null 2>&1; then
|
||||
fail "xcode-select -p still resolves to $(xcode-select -p 2>/dev/null); not a clean Mac"
|
||||
else
|
||||
|
|
@ -41,10 +38,10 @@ for check in "$@"; do
|
|||
for tool in git cc clang cmake; do
|
||||
command -v "$tool" >/dev/null 2>&1 || { ok "$tool not on PATH"; continue; }
|
||||
if "$tool" --version >/dev/null 2>&1; then
|
||||
# On Intel runners /usr/bin/git keeps working once the CLT are gone, so it
|
||||
# is not CLT-provided there and no masking can remove it. cc and clang do
|
||||
# become stubs, and the consumer path needs no git on macOS, so report it
|
||||
# rather than calling the simulation broken.
|
||||
# On Intel runners /usr/bin/git is not CLT-provided and keeps working once
|
||||
# the CLT are gone, so no masking can remove it. cc and clang do become
|
||||
# stubs and the macOS consumer path needs no git, so report rather than
|
||||
# call the simulation broken.
|
||||
case " ${UNSLOTH_CLEAN_ALLOW_WORKING:-} " in
|
||||
*" $tool "*)
|
||||
echo "[assert] NOTE $tool still works ($(command -v "$tool")); allowed on this runner"
|
||||
|
|
@ -68,19 +65,18 @@ for check in "$@"; do
|
|||
if [ -z "$TRACE" ] || [ ! -f "$TRACE" ]; then
|
||||
fail "notools requested but no trace file (\$UNSLOTH_TOOL_TRACE=$TRACE)"
|
||||
else
|
||||
# git is legitimate under --local (it installs unsloth-zoo from a git URL);
|
||||
# UNSLOTH_ALLOW_TOOLS lets that leg allow-list it explicitly.
|
||||
# git is legitimate under --local (unsloth-zoo comes from a git URL), so that
|
||||
# leg allow-lists it via UNSLOTH_ALLOW_TOOLS.
|
||||
allow="${UNSLOTH_ALLOW_TOOLS:-}"
|
||||
hits=""
|
||||
while IFS=$'\t' read -r tool rest; do
|
||||
[ -n "$tool" ] || continue
|
||||
case " $allow " in *" $tool "*) continue ;; esac
|
||||
# `xcode-select -p` ASKS whether a toolchain is selected; it cannot build
|
||||
# anything. The installer has to ask in order to tell the user whether a
|
||||
# source build is available, and the whole point of the fix is that it then
|
||||
# carries on without one. Treating the question as toolchain USE would fail
|
||||
# the very leg that proves the toolchain was never used. `--install`, which
|
||||
# pops the CLT installer, stays a hit.
|
||||
# `xcode-select -p` only ASKS whether a toolchain is selected; the installer
|
||||
# has to ask, and the point of the fix is that it carries on without one.
|
||||
# Counting the question as toolchain USE would fail the very leg that proves
|
||||
# the toolchain was never used. `--install`, which pops the CLT installer,
|
||||
# stays a hit.
|
||||
if [ "$tool" = "xcode-select" ]; then
|
||||
case "$rest" in
|
||||
-p|--print-path|-v|--version|"") continue ;;
|
||||
|
|
@ -98,25 +94,21 @@ for check in "$@"; do
|
|||
;;
|
||||
|
||||
nobuild)
|
||||
# "Built an sdist" is NOT the same as "needed a compiler". Four packages on the
|
||||
# macOS path are sdist-only PURE PYTHON projects that build fine with no
|
||||
# toolchain (verified by resolving each against cp313/macos-arm64):
|
||||
# "Built an sdist" is NOT "needed a compiler". Four packages on the macOS path
|
||||
# are sdist-only PURE PYTHON (verified against cp313/macos-arm64):
|
||||
# openai-whisper, argbind, randomname -- no version ever ships a wheel
|
||||
# antlr4-python3-runtime==4.9.3 -- pinned below the 4.13.2 wheel
|
||||
# Failing on those would be a false alarm, so the contract asserted here is
|
||||
# "nothing that needs a COMPILER was built", with that allowlist subtracted.
|
||||
# UNSLOTH_ALLOW_SDIST can extend it.
|
||||
# Failing on those is a false alarm, so the contract is "nothing needing a
|
||||
# COMPILER was built". UNSLOTH_ALLOW_SDIST extends the allowlist.
|
||||
_allow="openai-whisper argbind randomname antlr4-python3-runtime ${UNSLOTH_ALLOW_SDIST:-}"
|
||||
if [ ! -f "$LOG" ]; then
|
||||
fail "nobuild requested but $LOG is missing"
|
||||
else
|
||||
# The installer runs `uv pip install`, and uv does NOT use pip's phrasing.
|
||||
# It prints ` Building <name>==<version>` and ` Built <name>==<version>`
|
||||
# to stderr, as plain lines once stderr is not a TTY (astral-sh/uv#11165), so
|
||||
# the pip-only pattern left _built empty on every uv source build. Match both
|
||||
# spellings. Requiring `==` or ` @ ` after the name keeps this off the
|
||||
# installer's own lowercase "building frontend..." progress text. Strip ANSI
|
||||
# first so a coloured run (FORCE_COLOR) still parses.
|
||||
# uv does NOT use pip's phrasing: it prints `Building <name>==<version>` to
|
||||
# stderr (astral-sh/uv#11165), so the pip-only pattern left _built empty on
|
||||
# every uv source build. Match both spellings. Requiring `==` or ` @ ` after
|
||||
# the name keeps this off the installer's own lowercase "building frontend..."
|
||||
# progress text. Strip ANSI first so a coloured run (FORCE_COLOR) parses.
|
||||
_esc=$(printf '\033')
|
||||
_built="$(sed -E "s/${_esc}\[[0-9;]*[A-Za-z]//g" "$LOG" 2>/dev/null \
|
||||
| grep -oiE "building wheel for [a-z0-9._-]+|building [a-z0-9._-]+(==| @ )" \
|
||||
|
|
|
|||
68
.github/scripts/clean-machine-env.sh
vendored
68
.github/scripts/clean-machine-env.sh
vendored
|
|
@ -2,25 +2,20 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
|
||||
#
|
||||
# Simulate a virgin developer machine on a GitHub-hosted runner, so the installer
|
||||
# is exercised the way a real user's brand-new Mac / PC exercises it.
|
||||
# Simulate a virgin developer machine on a GitHub-hosted runner. Two modes, because
|
||||
# "the tool is absent" and "the installer never called the tool" cannot be simulated
|
||||
# by the same mechanism:
|
||||
#
|
||||
# Two modes, because "the tool is absent" and "the installer never called the tool"
|
||||
# CANNOT be simulated by the same mechanism:
|
||||
# mask Make the toolchain genuinely ABSENT: scrub PATH to OS defaults and (with
|
||||
# --remove) move the real toolchain aside, so `command -v git` correctly
|
||||
# FAILS, as on a clean Mac. A failing "poison shim" would do the opposite --
|
||||
# `command -v` finds it and reports the tool as present -- so no shims here.
|
||||
# trace Leave the toolchain working but route it through logging wrappers that log
|
||||
# the call then exec the real binary, proving whether the installer ever
|
||||
# REACHES for a compiler/git without changing behaviour.
|
||||
#
|
||||
# mask Make the toolchain genuinely ABSENT. Scrubs PATH down to the OS
|
||||
# defaults and (with --remove) moves the real toolchain aside. After
|
||||
# this, `command -v git` correctly FAILS, which is what a clean Mac does.
|
||||
# A failing "poison shim" on PATH would do the opposite -- `command -v`
|
||||
# finds it and reports the tool as present -- so shims are NOT used here.
|
||||
#
|
||||
# trace Leave the toolchain working, but route it through logging wrappers that
|
||||
# record the invocation and then exec the real binary. Proves whether the
|
||||
# installer ever REACHES for a compiler/git, without changing behaviour.
|
||||
#
|
||||
# Writes shell exports to $CLEAN_ENV_FILE (default ./clean-machine.env) for the
|
||||
# caller to `source`. Nothing is exported globally, so other workflow steps
|
||||
# (checkout, upload-artifact) keep a normal environment.
|
||||
# Writes shell exports to $CLEAN_ENV_FILE (default ./clean-machine.env) to `source`;
|
||||
# nothing is exported globally, so other steps keep a normal environment.
|
||||
#
|
||||
# Usage:
|
||||
# bash .github/scripts/clean-machine-env.sh mask [--remove]
|
||||
|
|
@ -55,8 +50,8 @@ TOOLS="xcode-select xcrun clang clang++ cc c++ gcc g++ git cmake make brew ninja
|
|||
note() { echo "[clean-machine] $*"; }
|
||||
|
||||
# ── PATH scrub ────────────────────────────────────────────────────────────────
|
||||
# Keep only OS-default system dirs. Drops Homebrew, the hosted Python toolcache,
|
||||
# setup-* shims, pipx, cargo, and every other preinstalled developer dir.
|
||||
# Keep only OS-default system dirs: drops Homebrew, the hosted Python toolcache,
|
||||
# setup-* shims, pipx, cargo and every other preinstalled developer dir.
|
||||
scrub_path() {
|
||||
local keep out=""
|
||||
if [ "$OS" = "Darwin" ]; then
|
||||
|
|
@ -76,11 +71,9 @@ if [ "$MODE" = "mask" ]; then
|
|||
NEWPATH="$(scrub_path)"
|
||||
{
|
||||
echo "export PATH='$NEWPATH'"
|
||||
# DEVELOPER_DIR must be UNSET, not pointed at a fake path: `xcode-select -p`
|
||||
# honours DEVELOPER_DIR and prints it verbatim with exit 0, so setting it to a
|
||||
# nonexistent dir makes the probe SUCCEED -- the exact opposite of a clean Mac,
|
||||
# where DEVELOPER_DIR is unset and the missing /var/db/xcode_select_link is what
|
||||
# makes `xcode-select -p` fail.
|
||||
# UNSET, not a fake path: `xcode-select -p` honours DEVELOPER_DIR and prints it
|
||||
# verbatim with exit 0, so a nonexistent dir makes the probe SUCCEED. On a clean
|
||||
# Mac it is unset and the missing xcode_select_link is what makes the probe fail.
|
||||
echo "unset DEVELOPER_DIR || true"
|
||||
echo "unset SDKROOT CC CXX CFLAGS CXXFLAGS LDFLAGS CMAKE_GENERATOR CMAKE_PREFIX_PATH || true"
|
||||
echo "export HOMEBREW_NO_AUTO_UPDATE=1"
|
||||
|
|
@ -88,11 +81,10 @@ if [ "$MODE" = "mask" ]; then
|
|||
} >> "$ENV_FILE"
|
||||
|
||||
if [ "$REMOVE" = "1" ] && [ "$OS" = "Darwin" ]; then
|
||||
# Best-effort real removal. Each step is independent and recorded in
|
||||
# restore.sh so an `if: always()` step can put the runner back.
|
||||
# /var/db/xcode_select_link is exactly what `xcode-select -p` reads, so
|
||||
# removing it reproduces a virgin Mac's gate precisely. `xcode-select --reset`
|
||||
# is NOT enough: it can reselect a full Xcode.app.
|
||||
# Best-effort real removal; each step is independent and recorded in restore.sh
|
||||
# so an `if: always()` step can put the runner back. xcode_select_link is exactly
|
||||
# what `xcode-select -p` reads, so removing it reproduces a virgin Mac's gate.
|
||||
# `xcode-select --reset` is NOT enough: it can reselect a full Xcode.app.
|
||||
if [ -e /var/db/xcode_select_link ]; then
|
||||
if sudo rm -f /var/db/xcode_select_link 2>/dev/null; then
|
||||
note "removed /var/db/xcode_select_link"
|
||||
|
|
@ -101,8 +93,8 @@ if [ "$MODE" = "mask" ]; then
|
|||
note "WARN could not remove /var/db/xcode_select_link"
|
||||
fi
|
||||
fi
|
||||
# Moving the CLT dir aside turns /usr/bin/{cc,clang,git} into dead shims, so
|
||||
# the run also proves the install needs no compiler at all.
|
||||
# Moving the CLT dir aside turns /usr/bin/{cc,clang,git} into dead shims, so the
|
||||
# run also proves the install needs no compiler at all.
|
||||
if [ -d /Library/Developer/CommandLineTools ]; then
|
||||
if sudo mv /Library/Developer/CommandLineTools /Library/Developer/CommandLineTools.masked 2>/dev/null; then
|
||||
note "moved CommandLineTools aside"
|
||||
|
|
@ -111,11 +103,11 @@ if [ "$MODE" = "mask" ]; then
|
|||
note "WARN could not move CommandLineTools"
|
||||
fi
|
||||
fi
|
||||
# Xcode.app must go too. With the select link removed AND CommandLineTools moved,
|
||||
# `xcode-select -p` does not fail -- it falls through to whatever Xcode bundle the
|
||||
# runner image ships (observed: /Applications/Xcode_16.4.app/Contents/Developer),
|
||||
# which re-arms /usr/bin/git and /usr/bin/cc and silently un-cleans the machine.
|
||||
# A rename is instant regardless of bundle size: same filesystem, no copy.
|
||||
# Xcode.app must go too: with the link removed AND CommandLineTools moved,
|
||||
# `xcode-select -p` still does not fail, it falls through to the image's Xcode
|
||||
# bundle (observed: /Applications/Xcode_16.4.app/Contents/Developer), which
|
||||
# re-arms /usr/bin/{git,cc} and silently un-cleans the machine. A rename is
|
||||
# instant regardless of bundle size: same filesystem, no copy.
|
||||
for app in /Applications/Xcode*.app; do
|
||||
[ -d "$app" ] || continue
|
||||
if sudo mv "$app" "${app}.masked" 2>/dev/null; then
|
||||
|
|
@ -143,8 +135,8 @@ if [ "$MODE" = "trace" ]; then
|
|||
for tool in $TOOLS; do
|
||||
real="$(command -v "$tool" 2>/dev/null || true)"
|
||||
[ -n "$real" ] || continue
|
||||
# Wrapper logs the call then execs the REAL binary, so behaviour is unchanged
|
||||
# and the trace answers "did the installer reach for this?" honestly.
|
||||
# Logs the call then execs the REAL binary: behaviour unchanged, so the trace
|
||||
# answers "did the installer reach for this?" honestly.
|
||||
cat > "$BIN/$tool" <<WRAP
|
||||
#!/bin/sh
|
||||
printf '%s\t%s\n' "$tool" "\$*" >> "$TRACE"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue