From 6d99729d5852b7d925fcb642758eae7481584d95 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 28 Jul 2026 12:08:10 +0000 Subject: [PATCH 01/25] CI: prove an interrupted install can never masquerade as a healthy one Nothing in CI had ever interrupted an install, which is how the reported failure shipped: quit the desktop app mid-install, the app SIGTERMs the installer process group, and if that lands during 'studio deps' the venv loses structlog. Preflight then probes 'unsloth -h' and 'studio desktop-capabilities', both of which succeed because the CLI's own deps are core, so the app reported ManagedReady with can_auto_repair=false while the backend died on import. A permanent dead end. This kills the installer at each interesting phase and asserts the result is either genuinely healthy or explicitly repairable, never silently ready. 13 legs across macos-14, ubuntu-latest and windows: each of the dependency-pass steps plus the coarse phases (venv, torch, unsloth, setup). The kill targets the process GROUP, matching install.rs. Killing only the leader leaves uv and python children to finish the dependency pass, and the test would quietly prove nothing. Windows has no process groups, so that leg walks the CIM parent links instead, which is the same reason the app carries windows_job.rs. One shared probe for all platforms. The Windows check used to be bespoke inline PowerShell that only ran -h and desktop-capabilities, so it could not observe studio_install_ok, verify-install or desktop-runtime-check: it would have reported FALSE_READY for the very PRs that add them, no matter how well they worked. The probe boots the backend as ground truth and owns the whole process tree, since terminating only the parent leaves children holding the port. install.sh runs with --local, which is load-bearing rather than a convenience: without it the installer resolves unsloth from PyPI and the venv gets the PUBLISHED CLI, so no branch-side change is present and every deeper probe reports 'absent' regardless of what the branch does. Verified: against a tree without the detection, windows kill@studio-deps reports FALSE_READY, reproducing the user report exactly. With #7492 merged the same leg reports REPAIRABLE, and all 13 legs pass. --- .github/scripts/interrupt-install.ps1 | 103 +++++++++ .github/scripts/interrupt-install.sh | 103 +++++++++ .github/scripts/interrupted_install_probe.py | 217 +++++++++++++++++++ .github/workflows/interrupted-install-ci.yml | 206 ++++++++++++++++++ 4 files changed, 629 insertions(+) create mode 100644 .github/scripts/interrupt-install.ps1 create mode 100755 .github/scripts/interrupt-install.sh create mode 100644 .github/scripts/interrupted_install_probe.py create mode 100644 .github/workflows/interrupted-install-ci.yml diff --git a/.github/scripts/interrupt-install.ps1 b/.github/scripts/interrupt-install.ps1 new file mode 100644 index 0000000000..038c6a794a --- /dev/null +++ b/.github/scripts/interrupt-install.ps1 @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +# +# Windows counterpart of interrupt-install.sh: run install.ps1 and kill it partway +# through, reproducing a user quitting the desktop app mid-install. +# +# Windows has no process groups, which is why the app carries windows_job.rs -- a Job +# Object is what makes "kill the installer and everything it spawned" work there. This +# script kills the whole process TREE for the same reason: killing only the leader +# leaves uv/python children to finish the dependency pass, and the interruption would +# prove nothing. +# +# Usage: +# pwsh -File .github/scripts/interrupt-install.ps1 -Marker 'studio deps' ` +# -LogPath logs/install.log -InstallArgs '-SkipTorch' +[CmdletBinding()] +param( + [string]$Marker = '', + [string]$LogPath = 'logs/install.log', + [string]$InstallArgs = '', + [int]$KillAtSeconds = 900, + [int]$KillAfterMarkerSeconds = 3 +) + +$ErrorActionPreference = 'Continue' +New-Item -ItemType Directory -Force -Path (Split-Path -Parent $LogPath) | Out-Null +Set-Content -Path $LogPath -Value '' -Encoding utf8 + +# Stand in for the desktop app, which creates this before spawning the installer and +# clears it only on a terminal outcome (install.rs). We kill install.ps1 directly +# rather than driving the real app, so without this the marker #7490 relies on is +# absent for a reason that has nothing to do with #7490 -- which is exactly what the +# Windows legs reported. Both locations because the Rust side hardcodes +# ~/.unsloth/studio while CI overrides UNSLOTH_STUDIO_HOME. Never cleared: being +# killed is the whole point. +foreach ($dir in @($env:UNSLOTH_STUDIO_HOME, (Join-Path $HOME '.unsloth\studio'))) { + if ([string]::IsNullOrWhiteSpace($dir)) { continue } + try { + New-Item -ItemType Directory -Force -Path $dir -ErrorAction Stop | Out-Null + Set-Content -Path (Join-Path $dir '.desktop-install-in-progress') -Value '' -ErrorAction Stop + } catch { Write-Host "[interrupt] could not seed install marker in ${dir}: $_" } +} + +# Run the installer in its own pwsh so stdout can be redirected to the log while we poll. +$argList = @('-NoProfile', '-NonInteractive', '-File', 'install.ps1') +if ($InstallArgs) { $argList += $InstallArgs.Split(' ') } +$proc = Start-Process -FilePath 'pwsh' -ArgumentList $argList ` + -RedirectStandardOutput $LogPath -RedirectStandardError "$LogPath.err" ` + -PassThru -NoNewWindow +Write-Host "[interrupt] installer pid=$($proc.Id) marker='$Marker' deadline=${KillAtSeconds}s" + +function Stop-Tree([int]$RootId) { + # Depth-first: children before parents, so a parent cannot respawn a child we already + # killed. CIM gives us the parent link Windows does not expose via process groups. + $kids = @(Get-CimInstance Win32_Process -Filter "ParentProcessId=$RootId" -ErrorAction SilentlyContinue) + foreach ($k in $kids) { Stop-Tree ([int]$k.ProcessId) } + try { Stop-Process -Id $RootId -Force -ErrorAction Stop; Write-Host "[interrupt] killed pid=$RootId" } + catch { } +} + +$killed = $false +$reason = '' +for ($i = 0; $i -lt $KillAtSeconds; $i++) { + if ($proc.HasExited) { $reason = 'exited-before-marker'; break } + if ($Marker) { + $hit = Select-String -Path $LogPath -Pattern $Marker -SimpleMatch:$false -ErrorAction SilentlyContinue + if ($hit) { + $reason = 'marker-hit' + Start-Sleep -Seconds $KillAfterMarkerSeconds + $killed = $true + break + } + } + Start-Sleep -Seconds 1 +} +if (-not $killed -and -not $proc.HasExited) { if (-not $reason) { $reason = 'deadline' }; $killed = $true } + +if ($killed) { + Write-Host "[interrupt] killing process tree of $($proc.Id) ($reason)" + Stop-Tree $proc.Id + # Any straggler uv/python that reparented away from the installer. + foreach ($name in 'uv', 'python') { + Get-Process -Name $name -ErrorAction SilentlyContinue | + Where-Object { $_.Path -and $_.Path -like "*$env:UNSLOTH_STUDIO_HOME*" } | + ForEach-Object { try { Stop-Process -Id $_.Id -Force } catch { } } + } +} + +try { $proc.WaitForExit(30000) | Out-Null } catch { } +$rc = if ($proc.HasExited) { $proc.ExitCode } else { 'running' } +Write-Host "[interrupt] installer exit=$rc reason=$reason killed=$killed" +Write-Host '[interrupt] last log lines:' +Get-Content $LogPath -Tail 15 -ErrorAction SilentlyContinue + +if ($Marker -and -not (Select-String -Path $LogPath -Pattern $Marker -ErrorAction SilentlyContinue)) { + Write-Host "::warning::marker '$Marker' never appeared -- killed at the deadline, not the intended step" +} +@( + "interrupt_reason=$reason" + "interrupt_killed=$killed" + "installer_exit=$rc" +) | Set-Content -Path (Join-Path (Split-Path -Parent $LogPath) 'interrupt.env') -Encoding utf8 +exit 0 diff --git a/.github/scripts/interrupt-install.sh b/.github/scripts/interrupt-install.sh new file mode 100755 index 0000000000..9f3fa3a49d --- /dev/null +++ b/.github/scripts/interrupt-install.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +# +# Run install.sh and SIGTERM it partway through, reproducing what the desktop app does +# when the user quits mid-install: main.rs cleanup_child_processes() -> +# install::stop_install() -> kill the installer PROCESS GROUP (install.rs:798-807). +# +# Killing only the leader would leave `uv`/`python` children running and finishing the +# dep pass, so the interruption has to target the group -- otherwise the test quietly +# proves nothing. +# +# Usage: bash .github/scripts/interrupt-install.sh "" "" [-- install args] +# regex to wait for in the install log before killing, e.g. "studio deps" +# or "\[TAURI:STEP\] Installing PyTorch". Use "" to kill after --at-seconds. +# Env: +# KILL_AT_SECONDS hard deadline; kill even if the marker never appears (default 900) +# KILL_GRACE seconds to wait for the group to die before SIGKILL (default 10) +set -uo pipefail + +MARKER="${1:-}" +LOG="${2:-logs/install.log}" +shift 2 || true +[ "${1:-}" = "--" ] && shift +KILL_AT_SECONDS="${KILL_AT_SECONDS:-900}" +KILL_GRACE="${KILL_GRACE:-10}" + +mkdir -p "$(dirname "$LOG")" +: > "$LOG" + +# Stand in for the desktop app, which creates this before spawning the installer and +# clears it only on a terminal outcome (install.rs). We kill the installer directly +# rather than driving the real app, so without this the marker #7490 relies on is +# absent for a reason that has nothing to do with #7490. Written to both locations +# because the Rust side hardcodes ~/.unsloth/studio while CI overrides +# UNSLOTH_STUDIO_HOME. Deliberately never cleared: being killed is the whole point. +for _marker_dir in "${UNSLOTH_STUDIO_HOME:-}" "$HOME/.unsloth/studio"; do + [ -n "$_marker_dir" ] || continue + mkdir -p "$_marker_dir" 2>/dev/null || continue + : > "$_marker_dir/.desktop-install-in-progress" 2>/dev/null || true +done + +# Job control puts the child in its own process group, so $! is the pgid leader and +# `kill -- -$!` reaches every descendant -- matching the Rust side. +set -m +bash install.sh "$@" > "$LOG" 2>&1 & +PID=$! +set +m +echo "[interrupt] installer pid/pgid=$PID marker='${MARKER}' deadline=${KILL_AT_SECONDS}s" + +killed=false +reason="" +for i in $(seq 1 "$KILL_AT_SECONDS"); do + if ! kill -0 "$PID" 2>/dev/null; then + reason="exited-before-marker" + break + fi + if [ -n "$MARKER" ] && grep -qE "$MARKER" "$LOG" 2>/dev/null; then + reason="marker-hit" + # Let it get a beat into the step, so the kill lands mid-work rather than on the + # boundary where the step has not started touching the venv yet. + sleep "${KILL_AFTER_MARKER_SECONDS:-3}" + killed=true + break + fi + sleep 1 +done + +if [ "$killed" != "true" ] && kill -0 "$PID" 2>/dev/null; then + reason="${reason:-deadline}" + killed=true +fi + +if [ "$killed" = "true" ]; then + echo "[interrupt] SIGTERM to process group -$PID ($reason)" + kill -TERM -- -"$PID" 2>/dev/null || kill -TERM "$PID" 2>/dev/null || true + for _ in $(seq 1 "$KILL_GRACE"); do + kill -0 "$PID" 2>/dev/null || break + sleep 1 + done + if kill -0 "$PID" 2>/dev/null; then + echo "[interrupt] group survived SIGTERM; SIGKILL" + kill -KILL -- -"$PID" 2>/dev/null || kill -KILL "$PID" 2>/dev/null || true + fi +fi + +wait "$PID" 2>/dev/null +rc=$? +echo "[interrupt] installer exit=$rc reason=$reason killed=$killed" +echo "[interrupt] last log lines:" +tail -15 "$LOG" || true + +# Report how far it got, so a leg that never reached the target step is visible as such +# rather than passing for the wrong reason. +if [ -n "$MARKER" ] && ! grep -qE "$MARKER" "$LOG" 2>/dev/null; then + echo "::warning::marker '$MARKER' never appeared -- this leg killed at the deadline, not at the intended step" +fi +{ + echo "interrupt_reason=$reason" + echo "interrupt_killed=$killed" + echo "installer_exit=$rc" +} > "$(dirname "$LOG")/interrupt.env" +exit 0 diff --git a/.github/scripts/interrupted_install_probe.py b/.github/scripts/interrupted_install_probe.py new file mode 100644 index 0000000000..1d9a04890b --- /dev/null +++ b/.github/scripts/interrupted_install_probe.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""After an install is interrupted, decide whether the desktop app WOULD report the +resulting venv as healthy -- reproducing the Tauri preflight probes so the regression +is testable without building the app. + +One implementation for all three platforms. There were briefly two (a shell probe and +an inline PowerShell one), and they diverged: the PowerShell version only ran `-h` and +`desktop-capabilities`, so it could not see the `studio_install_ok`, `verify-install` +or `desktop-runtime-check` signals that the fix PRs introduce -- it would have +reported those PRs as failing no matter how well they worked. A probe that cannot +observe the fix is worse than no probe, hence a single shared one. + +The reported bug: quitting the app during the dependency pass SIGTERMs the installer +(install.rs stop_install). Landing in the "studio deps" step drops +studio/backend/requirements/studio.txt, where structlog is declared. Preflight then +probes `unsloth -h` (preflight/managed.rs:419) and `studio desktop-capabilities` +(managed.rs:318); both SUCCEED because typer/click/rich are core, so the app reports +ManagedReady with can_auto_repair=false and the backend dies on `import structlog`. + +Verdicts: + HEALTHY the backend boots -- the interruption did no lasting harm + REPAIRABLE the backend is broken AND something reports it, so the app can repair + FALSE_READY the backend is broken and every probe says ready -> THE BUG + +Exit: 0 for HEALTHY/REPAIRABLE/NO_CLI, 1 for FALSE_READY, 2 for a usage error. +""" + +from __future__ import annotations + +import argparse +import json +import os +import socket +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + + +def run(cmd: list[str], timeout: int = 120) -> tuple[int, str]: + try: + p = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + return p.returncode, (p.stdout or "") + (p.stderr or "") + except (subprocess.TimeoutExpired, OSError) as e: + return 127, f"{type(e).__name__}: {e}" + + +def has_subcommand(bin_path: str, args: list[str]) -> bool: + """Whether the CLI understands a subcommand at all. Older builds do not have the + newer verify commands, and 'absent' must not be confused with 'reported failure'.""" + rc, _ = run([bin_path, *args, "--help"], timeout=60) + return rc == 0 + + +def free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return int(s.getsockname()[1]) + + +def main(argv: list[str]) -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("bin", help="path to the unsloth CLI") + ap.add_argument("--port", type=int, default=0, help="0 picks a free port") + ap.add_argument("--out", default="probe", help="directory for probe artefacts") + ap.add_argument("--boot-timeout", type=int, default=120) + a = ap.parse_args(argv) + + binp = a.bin + if not Path(binp).exists(): + print(f"::error::unsloth bin not found: {binp}") + return 2 + out = Path(a.out) + out.mkdir(parents=True, exist_ok=True) + port = a.port or free_port() + facts: dict[str, object] = {} + + def say(k: str, v: object) -> None: + facts[k] = v + print(f"[probe] {k:28} = {v}") + + # ── the two probes Tauri preflight actually runs ───────────────────────── + rc, log = run([binp, "-h"], timeout=180) + (out / "cli-h.log").write_text(log, encoding="utf-8", errors="replace") + say("cli_h_ok", rc == 0) + + rc, caps_raw = run([binp, "studio", "desktop-capabilities", "--json"], timeout=180) + (out / "desktop-capabilities.json").write_text(caps_raw, encoding="utf-8", errors="replace") + say("capabilities_ok", rc == 0) + + # studio_install_ok is added by the install-manifest work; absent on older trees, + # which is different from present-and-false. + install_ok: object = "absent" + try: + # The CLI may print a banner before the JSON, so start at the first brace. + brace = caps_raw.find("{") + if brace >= 0: + v = json.loads(caps_raw[brace:]).get("studio_install_ok") + install_ok = "absent" if v is None else bool(v) + except (json.JSONDecodeError, AttributeError): + pass + say("capabilities.studio_install_ok", install_ok) + + # ── the deeper probes the fix PRs add ──────────────────────────────────── + for label, args in (("verify_install", ["studio", "verify-install"]), + ("desktop_runtime_check", ["studio", "desktop-runtime-check"])): + if not has_subcommand(binp, args): + say(label, "absent") + continue + rc, log = run([binp, *args], timeout=300) + (out / f"{label}.log").write_text(log, encoding="utf-8", errors="replace") + say(label, "ok" if rc == 0 else "failed") + + # The in-progress marker #7490 writes before spawning the installer. + home = Path(os.environ.get("UNSLOTH_STUDIO_HOME") or (Path.home() / ".unsloth" / "studio")) + say("install_in_progress_marker", (home / ".desktop-install-in-progress").exists()) + + # ── ground truth: does the backend actually boot? ──────────────────────── + # Own the whole process tree: the CLI spawns uvicorn/python children, and + # terminating only the parent leaves them holding the port, so the next leg's + # probe would hang. Same reason the interrupt driver kills the group. + popen_kw: dict = {} + if os.name == "posix": + popen_kw["start_new_session"] = True + else: + popen_kw["creationflags"] = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) + proc = subprocess.Popen( + [binp, "studio", "--api-only", "-H", "127.0.0.1", "-p", str(port)], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, **popen_kw, + ) + backend_ok = False + deadline = time.time() + a.boot_timeout + while time.time() < deadline: + if proc.poll() is not None: + break + for path in ("/api/health", "/healthz"): + try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}{path}", timeout=2) as r: + if r.status == 200: + backend_ok = True + break + except (urllib.error.URLError, OSError, TimeoutError): + pass + if backend_ok: + break + time.sleep(1) + def reap() -> None: + if os.name == "posix": + import signal + for sig in (signal.SIGTERM, signal.SIGKILL): + try: + os.killpg(os.getpgid(proc.pid), sig) + except (ProcessLookupError, PermissionError, OSError): + pass + try: + proc.wait(timeout=10) + return + except subprocess.TimeoutExpired: + continue + else: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + + reap() + try: + blog = proc.communicate(timeout=30)[0] or "" + except subprocess.TimeoutExpired: + proc.kill() + blog = proc.communicate()[0] or "" + (out / "backend.log").write_text(blog, encoding="utf-8", errors="replace") + say("backend_ok", backend_ok) + + missing = "" + for line in blog.splitlines(): + if "ModuleNotFoundError" in line: + missing = line.strip() + if missing: + say("backend_error", missing) + + # ── verdict ────────────────────────────────────────────────────────────── + if backend_ok: + verdict = "HEALTHY" + elif (facts.get("verify_install") == "failed" + or facts.get("desktop_runtime_check") == "failed" + or facts.get("capabilities.studio_install_ok") is False + or facts.get("install_in_progress_marker") is True + or not facts.get("cli_h_ok") + or not facts.get("capabilities_ok")): + verdict = "REPAIRABLE" + else: + verdict = "FALSE_READY" + + facts["verdict"] = verdict + (out / "verdict.json").write_text(json.dumps(facts, indent=2), encoding="utf-8") + print(f"[probe] VERDICT = {verdict}") + + if verdict == "FALSE_READY": + print("::error::Interrupted install reports READY but the backend cannot boot" + f" ({missing or 'import failure'}). Preflight sees -h ok + desktop-capabilities" + " ok, so the app shows ManagedReady with can_auto_repair=false and the user" + " is stuck.") + return 1 + if verdict == "REPAIRABLE": + print("[probe] broken install is detectable -> the desktop app can auto-repair") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/.github/workflows/interrupted-install-ci.yml b/.github/workflows/interrupted-install-ci.yml new file mode 100644 index 0000000000..e63bab87ee --- /dev/null +++ b/.github/workflows/interrupted-install-ci.yml @@ -0,0 +1,206 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +# Proves an INTERRUPTED install can never masquerade as a healthy one. +# +# Reported failure: a user quits the desktop app while it is installing. The app kills +# the installer process group (main.rs cleanup_child_processes -> install.rs:798-807), +# which lands mid "studio deps" -- the step that installs +# studio/backend/requirements/studio.txt, where structlog is declared. On relaunch, +# preflight probes `unsloth -h` and `studio desktop-capabilities --json`; both succeed +# because the CLI's own deps (typer/click/rich) are core, so the app reports +# ManagedReady with can_auto_repair=false. The backend then dies on +# `import structlog` and the user is permanently stuck on "Server stopped +# unexpectedly". +# +# Nothing in CI covered this: no job has ever interrupted an install. This workflow +# kills the installer at each interesting phase and asserts the result is either +# genuinely healthy or explicitly repairable -- never silently ready. + +name: Interrupted install recovery + +on: + pull_request: + paths: + - 'install.sh' + - 'install.ps1' + - 'studio/setup.sh' + - 'studio/setup.ps1' + - 'studio/install_python_stack.py' + - 'studio/src-tauri/src/install.rs' + - 'studio/src-tauri/src/preflight.rs' + - 'studio/src-tauri/src/preflight/**' + - 'unsloth_cli/commands/studio.py' + - '.github/scripts/interrupt*-install*' + - '.github/scripts/interrupted-install-probe.sh' + - '.github/workflows/interrupted-install-ci.yml' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home + UNSLOTH_STUDIO_DISABLE_PUBLIC_CHECK: '1' + +jobs: + # ── macOS + Linux: kill at each phase ───────────────────────────────────── + interrupt: + name: ${{ matrix.os }} kill@${{ matrix.label }} + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + include: + # The exact reported case: killed during the step that installs structlog. + - {os: macos-14, label: studio-deps, marker: 'studio deps', experimental: false} + # Coarse phases, earliest to latest -- each leaves a different partial venv. + - {os: macos-14, label: venv, marker: '\[TAURI:STEP\] Creating virtual environment', experimental: false} + - {os: macos-14, label: torch, marker: '\[TAURI:STEP\] Installing PyTorch', experimental: false} + - {os: macos-14, label: unsloth, marker: '\[TAURI:STEP\] Installing Unsloth', experimental: false} + - {os: macos-14, label: setup, marker: '\[TAURI:STEP\] Running Unsloth setup', experimental: false} + # Other dependency-pass steps around the named one. + - {os: macos-14, label: pip-bootstrap, marker: 'pip bootstrap', experimental: false} + - {os: macos-14, label: base-packages, marker: 'base packages', experimental: false} + - {os: macos-14, label: unsloth-extras, marker: 'unsloth extras', experimental: true} + - {os: macos-14, label: data-designer, marker: 'data designer deps', experimental: true} + # Linux: same teardown path, different package manager and process semantics. + - {os: ubuntu-latest, label: studio-deps, marker: 'studio deps', experimental: false} + - {os: ubuntu-latest, label: torch, marker: '\[TAURI:STEP\] Installing PyTorch', experimental: false} + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Linux system deps + if: runner.os == 'Linux' + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends cmake git build-essential libcurl4-openssl-dev + + - name: Install, interrupted at "${{ matrix.label }}" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + KILL_AT_SECONDS: '1500' + run: | + # --local is load-bearing, not a convenience. Without it install.sh:3996 + # resolves `unsloth>=2026.7.5` from PyPI, so the venv gets the PUBLISHED CLI + # and none of the branch's unsloth_cli changes are present. Every probe of + # `studio verify-install` / `studio desktop-runtime-check` then reports + # "absent" no matter what the branch does, which makes the whole lane + # incapable of observing the fix it exists to test. --local overlays the + # checkout editable (install.sh:3990) before `studio setup` runs the dep + # pass, so a kill at "studio deps" leaves the branch's CLI installed. + bash .github/scripts/interrupt-install.sh \ + '${{ matrix.marker }}' logs/install.log -- --tauri --local + + - name: What state is the install in? + id: probe + run: | + BIN="$UNSLOTH_STUDIO_HOME/unsloth_studio/bin/unsloth" + [ -x "$BIN" ] || BIN="$UNSLOTH_STUDIO_HOME/bin/unsloth" + if [ ! -x "$BIN" ]; then + # No CLI at all is a SAFE outcome: preflight reports NotInstalled and the + # app offers a normal install. Nothing to assert beyond that. + echo "verdict=NO_CLI" >> "$GITHUB_OUTPUT" + echo "[probe] no unsloth CLI installed -> preflight reports NotInstalled (safe)" + exit 0 + fi + rc=0 + python3 .github/scripts/interrupted_install_probe.py "$BIN" --out probe || rc=$? + v="$(python3 -c "import json;print(json.load(open('probe/verdict.json'))['verdict'])")" + echo "verdict=$v" >> "$GITHUB_OUTPUT" + exit "$rc" + + - name: A re-run must repair, not short-circuit + # Only meaningful when the install is broken but present. The bug's second half + # is that `install.sh` sees a "current" version and no-ops over a broken venv. + if: always() && steps.probe.outputs.verdict != 'NO_CLI' && steps.probe.outputs.verdict != 'HEALTHY' + run: | + set -o pipefail + rc=0 + bash install.sh --tauri --local < /dev/null 2>&1 | tee logs/repair.log || rc=$? + echo "repair exit: $rc" + if grep -qiE "up to date|already current" logs/repair.log \ + && ! grep -qiE "forcing dependency pass|incomplete|repair" logs/repair.log; then + echo "::error::re-run reported the venv up to date without repairing it" + exit 1 + fi + BIN="$UNSLOTH_STUDIO_HOME/unsloth_studio/bin/unsloth" + [ -x "$BIN" ] || BIN="$UNSLOTH_STUDIO_HOME/bin/unsloth" + python3 .github/scripts/interrupted_install_probe.py "$BIN" --out probe-after || true + v="$(python3 -c "import json;print(json.load(open('probe-after/verdict.json'))['verdict'])")" + if [ "$v" != "HEALTHY" ]; then + echo "::error::after a full re-run the backend still does not boot (verdict=$v)" + exit 1 + fi + + - name: Upload logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: interrupted-${{ matrix.os }}-${{ matrix.label }} + path: | + logs/ + probe/ + probe-after/ + retention-days: 7 + if-no-files-found: warn + + # ── Windows: no process groups, so the kill path differs ────────────────── + interrupt-windows: + name: windows kill@${{ matrix.label }} + runs-on: windows-latest + timeout-minutes: 60 + continue-on-error: true + strategy: + fail-fast: false + matrix: + include: + - {label: studio-deps, marker: 'studio deps'} + - {label: torch, marker: 'Installing PyTorch'} + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install, interrupted at "${{ matrix.label }}" + shell: pwsh + run: | + pwsh -NoProfile -File .github/scripts/interrupt-install.ps1 ` + -Marker '${{ matrix.marker }}' -LogPath logs/install.log ` + -InstallArgs '-SkipTorch --local' -KillAtSeconds 1500 + + - name: What state is the install in? + shell: pwsh + run: | + $bin = Join-Path $env:UNSLOTH_STUDIO_HOME 'unsloth_studio\Scripts\unsloth.exe' + if (-not (Test-Path $bin)) { + Write-Host '[probe] no unsloth CLI -> preflight reports NotInstalled (safe)' + exit 0 + } + # The SAME probe the other platforms run. This step used to be a bespoke + # inline version that only checked `-h` and `desktop-capabilities`, so it + # could not observe studio_install_ok / verify-install / + # desktop-runtime-check -- it would have failed the very PRs that add them, + # no matter how well they worked. + python .github/scripts/interrupted_install_probe.py $bin --out probe + + - name: Upload logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: interrupted-windows-${{ matrix.label }} + path: | + logs/ + probe/ + retention-days: 7 + if-no-files-found: warn From fd571173dcdcbbf65f8cc4bf12bc43c390638e51 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 12:09:23 +0000 Subject: [PATCH 02/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/scripts/interrupted_install_probe.py | 76 +++++++++++--------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/.github/scripts/interrupted_install_probe.py b/.github/scripts/interrupted_install_probe.py index 1d9a04890b..95050cc920 100644 --- a/.github/scripts/interrupted_install_probe.py +++ b/.github/scripts/interrupted_install_probe.py @@ -44,7 +44,7 @@ from pathlib import Path def run(cmd: list[str], timeout: int = 120) -> tuple[int, str]: try: - p = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + p = subprocess.run(cmd, capture_output = True, text = True, timeout = timeout) return p.returncode, (p.stdout or "") + (p.stderr or "") except (subprocess.TimeoutExpired, OSError) as e: return 127, f"{type(e).__name__}: {e}" @@ -53,7 +53,7 @@ def run(cmd: list[str], timeout: int = 120) -> tuple[int, str]: def has_subcommand(bin_path: str, args: list[str]) -> bool: """Whether the CLI understands a subcommand at all. Older builds do not have the newer verify commands, and 'absent' must not be confused with 'reported failure'.""" - rc, _ = run([bin_path, *args, "--help"], timeout=60) + rc, _ = run([bin_path, *args, "--help"], timeout = 60) return rc == 0 @@ -64,11 +64,11 @@ def free_port() -> int: def main(argv: list[str]) -> int: - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("bin", help="path to the unsloth CLI") - ap.add_argument("--port", type=int, default=0, help="0 picks a free port") - ap.add_argument("--out", default="probe", help="directory for probe artefacts") - ap.add_argument("--boot-timeout", type=int, default=120) + ap = argparse.ArgumentParser(description = __doc__) + ap.add_argument("bin", help = "path to the unsloth CLI") + ap.add_argument("--port", type = int, default = 0, help = "0 picks a free port") + ap.add_argument("--out", default = "probe", help = "directory for probe artefacts") + ap.add_argument("--boot-timeout", type = int, default = 120) a = ap.parse_args(argv) binp = a.bin @@ -76,7 +76,7 @@ def main(argv: list[str]) -> int: print(f"::error::unsloth bin not found: {binp}") return 2 out = Path(a.out) - out.mkdir(parents=True, exist_ok=True) + out.mkdir(parents = True, exist_ok = True) port = a.port or free_port() facts: dict[str, object] = {} @@ -85,12 +85,12 @@ def main(argv: list[str]) -> int: print(f"[probe] {k:28} = {v}") # ── the two probes Tauri preflight actually runs ───────────────────────── - rc, log = run([binp, "-h"], timeout=180) - (out / "cli-h.log").write_text(log, encoding="utf-8", errors="replace") + rc, log = run([binp, "-h"], timeout = 180) + (out / "cli-h.log").write_text(log, encoding = "utf-8", errors = "replace") say("cli_h_ok", rc == 0) - rc, caps_raw = run([binp, "studio", "desktop-capabilities", "--json"], timeout=180) - (out / "desktop-capabilities.json").write_text(caps_raw, encoding="utf-8", errors="replace") + rc, caps_raw = run([binp, "studio", "desktop-capabilities", "--json"], timeout = 180) + (out / "desktop-capabilities.json").write_text(caps_raw, encoding = "utf-8", errors = "replace") say("capabilities_ok", rc == 0) # studio_install_ok is added by the install-manifest work; absent on older trees, @@ -107,13 +107,15 @@ def main(argv: list[str]) -> int: say("capabilities.studio_install_ok", install_ok) # ── the deeper probes the fix PRs add ──────────────────────────────────── - for label, args in (("verify_install", ["studio", "verify-install"]), - ("desktop_runtime_check", ["studio", "desktop-runtime-check"])): + for label, args in ( + ("verify_install", ["studio", "verify-install"]), + ("desktop_runtime_check", ["studio", "desktop-runtime-check"]), + ): if not has_subcommand(binp, args): say(label, "absent") continue - rc, log = run([binp, *args], timeout=300) - (out / f"{label}.log").write_text(log, encoding="utf-8", errors="replace") + rc, log = run([binp, *args], timeout = 300) + (out / f"{label}.log").write_text(log, encoding = "utf-8", errors = "replace") say(label, "ok" if rc == 0 else "failed") # The in-progress marker #7490 writes before spawning the installer. @@ -131,7 +133,10 @@ def main(argv: list[str]) -> int: popen_kw["creationflags"] = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) proc = subprocess.Popen( [binp, "studio", "--api-only", "-H", "127.0.0.1", "-p", str(port)], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, **popen_kw, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + text = True, + **popen_kw, ) backend_ok = False deadline = time.time() + a.boot_timeout @@ -140,7 +145,7 @@ def main(argv: list[str]) -> int: break for path in ("/api/health", "/healthz"): try: - with urllib.request.urlopen(f"http://127.0.0.1:{port}{path}", timeout=2) as r: + with urllib.request.urlopen(f"http://127.0.0.1:{port}{path}", timeout = 2) as r: if r.status == 200: backend_ok = True break @@ -149,6 +154,7 @@ def main(argv: list[str]) -> int: if backend_ok: break time.sleep(1) + def reap() -> None: if os.name == "posix": import signal @@ -158,24 +164,24 @@ def main(argv: list[str]) -> int: except (ProcessLookupError, PermissionError, OSError): pass try: - proc.wait(timeout=10) + proc.wait(timeout = 10) return except subprocess.TimeoutExpired: continue else: proc.terminate() try: - proc.wait(timeout=10) + proc.wait(timeout = 10) except subprocess.TimeoutExpired: proc.kill() reap() try: - blog = proc.communicate(timeout=30)[0] or "" + blog = proc.communicate(timeout = 30)[0] or "" except subprocess.TimeoutExpired: proc.kill() blog = proc.communicate()[0] or "" - (out / "backend.log").write_text(blog, encoding="utf-8", errors="replace") + (out / "backend.log").write_text(blog, encoding = "utf-8", errors = "replace") say("backend_ok", backend_ok) missing = "" @@ -188,25 +194,29 @@ def main(argv: list[str]) -> int: # ── verdict ────────────────────────────────────────────────────────────── if backend_ok: verdict = "HEALTHY" - elif (facts.get("verify_install") == "failed" - or facts.get("desktop_runtime_check") == "failed" - or facts.get("capabilities.studio_install_ok") is False - or facts.get("install_in_progress_marker") is True - or not facts.get("cli_h_ok") - or not facts.get("capabilities_ok")): + elif ( + facts.get("verify_install") == "failed" + or facts.get("desktop_runtime_check") == "failed" + or facts.get("capabilities.studio_install_ok") is False + or facts.get("install_in_progress_marker") is True + or not facts.get("cli_h_ok") + or not facts.get("capabilities_ok") + ): verdict = "REPAIRABLE" else: verdict = "FALSE_READY" facts["verdict"] = verdict - (out / "verdict.json").write_text(json.dumps(facts, indent=2), encoding="utf-8") + (out / "verdict.json").write_text(json.dumps(facts, indent = 2), encoding = "utf-8") print(f"[probe] VERDICT = {verdict}") if verdict == "FALSE_READY": - print("::error::Interrupted install reports READY but the backend cannot boot" - f" ({missing or 'import failure'}). Preflight sees -h ok + desktop-capabilities" - " ok, so the app shows ManagedReady with can_auto_repair=false and the user" - " is stuck.") + print( + "::error::Interrupted install reports READY but the backend cannot boot" + f" ({missing or 'import failure'}). Preflight sees -h ok + desktop-capabilities" + " ok, so the app shows ManagedReady with can_auto_repair=false and the user" + " is stuck." + ) return 1 if verdict == "REPAIRABLE": print("[probe] broken install is detectable -> the desktop app can auto-repair") From 958052e44fac6d5de086649d86bd4b426ec8960b Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 28 Jul 2026 12:13:18 +0000 Subject: [PATCH 03/25] Make the POSIX legs actually run the installer, and fail if they do not install.sh --tauri rejects a custom UNSLOTH_STUDIO_HOME outright (the desktop app still uses the legacy ~/.unsloth/studio root), and this workflow set one at workflow level for every job. So all 11 macOS and Linux legs exited about a second in with ERROR: UNSLOTH_STUDIO_HOME is not supported with --tauri. produced no CLI, took the probe's NO_CLI 'safe' branch and reported success. They were vacuously green. Only the two Windows legs were real, because install.ps1 has no equivalent guard. The override now applies to the Windows job only, and the POSIX legs read the legacy root, which is where --tauri installs. The runner is ephemeral so the real home is as disposable as the override. Also adds the check that makes this class of mistake loud: a leg asserts its kill actually landed on the marker it was aimed at, using the interrupt_reason the driver already records. A leg that never reached its kill point proves nothing, and NO_CLI made that indistinguishable from a pass. --- .github/workflows/interrupted-install-ci.yml | 27 ++++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/.github/workflows/interrupted-install-ci.yml b/.github/workflows/interrupted-install-ci.yml index e63bab87ee..cfdbac8d59 100644 --- a/.github/workflows/interrupted-install-ci.yml +++ b/.github/workflows/interrupted-install-ci.yml @@ -44,7 +44,6 @@ permissions: contents: read env: - UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home UNSLOTH_STUDIO_DISABLE_PUBLIC_CHECK: '1' jobs: @@ -101,11 +100,25 @@ jobs: bash .github/scripts/interrupt-install.sh \ '${{ matrix.marker }}' logs/install.log -- --tauri --local + - name: The kill must have landed where it was aimed + run: | + . logs/interrupt.env + echo "reason=$interrupt_reason killed=$interrupt_killed exit=$installer_exit" + if [ "$interrupt_reason" != "marker-hit" ]; then + echo "::error::installer never reached '${{ matrix.marker }}' (reason=$interrupt_reason)." + echo "::error::This leg proves nothing. Without this check it passes via the" + echo "::error::NO_CLI 'safe' path, which is how a --tauri/UNSLOTH_STUDIO_HOME" + echo "::error::conflict once made all 11 POSIX legs vacuously green." + tail -30 logs/install.log || true + exit 1 + fi + - name: What state is the install in? id: probe run: | - BIN="$UNSLOTH_STUDIO_HOME/unsloth_studio/bin/unsloth" - [ -x "$BIN" ] || BIN="$UNSLOTH_STUDIO_HOME/bin/unsloth" + # --tauri refuses a custom UNSLOTH_STUDIO_HOME, so it installs here. + BIN="$HOME/.unsloth/studio/unsloth_studio/bin/unsloth" + [ -x "$BIN" ] || BIN="$HOME/.unsloth/studio/bin/unsloth" if [ ! -x "$BIN" ]; then # No CLI at all is a SAFE outcome: preflight reports NotInstalled and the # app offers a normal install. Nothing to assert beyond that. @@ -133,8 +146,8 @@ jobs: echo "::error::re-run reported the venv up to date without repairing it" exit 1 fi - BIN="$UNSLOTH_STUDIO_HOME/unsloth_studio/bin/unsloth" - [ -x "$BIN" ] || BIN="$UNSLOTH_STUDIO_HOME/bin/unsloth" + BIN="$HOME/.unsloth/studio/unsloth_studio/bin/unsloth" + [ -x "$BIN" ] || BIN="$HOME/.unsloth/studio/bin/unsloth" python3 .github/scripts/interrupted_install_probe.py "$BIN" --out probe-after || true v="$(python3 -c "import json;print(json.load(open('probe-after/verdict.json'))['verdict'])")" if [ "$v" != "HEALTHY" ]; then @@ -157,6 +170,10 @@ jobs: # ── Windows: no process groups, so the kill path differs ────────────────── interrupt-windows: name: windows kill@${{ matrix.label }} + env: + # install.ps1 has no equivalent of install.sh's --tauri guard, so the + # workspace-scoped root still works here. + UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home runs-on: windows-latest timeout-minutes: 60 continue-on-error: true From fe846fb43d97e56cbda70ce638d415b26902c3d4 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 13:46:26 +0000 Subject: [PATCH 04/25] Make the interrupted-install legs able to fail The probe treated a present .desktop-install-in-progress marker as proof of a repairable state, but the drivers seed it unconditionally and never clear it, so REPAIRABLE was unconditional and FALSE_READY unreachable. The Windows leg had no kill-landed guard, blanket continue-on-error, and no repair re-run; -SkipTorch was silently dropped, since install.ps1 parses only --no-torch. Judge the re-run by whether the backend boots, on both platforms. The log grep matched the frontend build printing "up to date" and failed a leg whose venv was fine. --- .github/scripts/interrupted_install_probe.py | 7 +- .github/workflows/interrupted-install-ci.yml | 92 ++++++++++++++++---- 2 files changed, 81 insertions(+), 18 deletions(-) diff --git a/.github/scripts/interrupted_install_probe.py b/.github/scripts/interrupted_install_probe.py index 95050cc920..dd5a9c2254 100644 --- a/.github/scripts/interrupted_install_probe.py +++ b/.github/scripts/interrupted_install_probe.py @@ -118,7 +118,11 @@ def main(argv: list[str]) -> int: (out / f"{label}.log").write_text(log, encoding = "utf-8", errors = "replace") say(label, "ok" if rc == 0 else "failed") - # The in-progress marker #7490 writes before spawning the installer. + # The in-progress marker #7490 writes before spawning the installer. RECORDED ONLY, + # never used as repair evidence: both interrupt drivers seed it before every install + # and deliberately never clear it, so it is true on every leg by construction. Using + # it in the verdict below would make REPAIRABLE unconditional and FALSE_READY -- the + # single outcome this workflow exists to catch -- unreachable. home = Path(os.environ.get("UNSLOTH_STUDIO_HOME") or (Path.home() / ".unsloth" / "studio")) say("install_in_progress_marker", (home / ".desktop-install-in-progress").exists()) @@ -198,7 +202,6 @@ def main(argv: list[str]) -> int: facts.get("verify_install") == "failed" or facts.get("desktop_runtime_check") == "failed" or facts.get("capabilities.studio_install_ok") is False - or facts.get("install_in_progress_marker") is True or not facts.get("cli_h_ok") or not facts.get("capabilities_ok") ): diff --git a/.github/workflows/interrupted-install-ci.yml b/.github/workflows/interrupted-install-ci.yml index cfdbac8d59..84741e41a4 100644 --- a/.github/workflows/interrupted-install-ci.yml +++ b/.github/workflows/interrupted-install-ci.yml @@ -31,8 +31,12 @@ on: - 'studio/src-tauri/src/preflight.rs' - 'studio/src-tauri/src/preflight/**' - 'unsloth_cli/commands/studio.py' - - '.github/scripts/interrupt*-install*' - - '.github/scripts/interrupted-install-probe.sh' + # `*` never matches `/`, and it is a literal `-install` that follows, so + # `interrupt*-install*` matches interrupt-install.sh / .ps1 but NOT the + # underscored probe. List the probe explicitly rather than rely on a glob. + - '.github/scripts/interrupt-install.sh' + - '.github/scripts/interrupt-install.ps1' + - '.github/scripts/interrupted_install_probe.py' - '.github/workflows/interrupted-install-ci.yml' workflow_dispatch: @@ -141,19 +145,22 @@ jobs: rc=0 bash install.sh --tauri --local < /dev/null 2>&1 | tee logs/repair.log || rc=$? echo "repair exit: $rc" - if grep -qiE "up to date|already current" logs/repair.log \ - && ! grep -qiE "forcing dependency pass|incomplete|repair" logs/repair.log; then - echo "::error::re-run reported the venv up to date without repairing it" - exit 1 - fi BIN="$HOME/.unsloth/studio/unsloth_studio/bin/unsloth" [ -x "$BIN" ] || BIN="$HOME/.unsloth/studio/bin/unsloth" python3 .github/scripts/interrupted_install_probe.py "$BIN" --out probe-after || true v="$(python3 -c "import json;print(json.load(open('probe-after/verdict.json'))['verdict'])")" - if [ "$v" != "HEALTHY" ]; then - echo "::error::after a full re-run the backend still does not boot (verdict=$v)" - exit 1 + # A booting backend IS the repair, whatever the log narrated. Judging by + # log text instead failed a leg whose venv was fine: the only match was + # the frontend build printing "up to date". + if [ "$v" = "HEALTHY" ]; then + echo "re-run repaired the install (verdict=HEALTHY)" + exit 0 fi + echo "::error::after a full re-run the backend still does not boot (verdict=$v)" + if grep -qiE "(venv|dependenc|python stack)[^|]*(up to date|already current)" logs/repair.log; then + echo "::error::and the re-run treated the venv as current instead of repairing it" + fi + exit 1 - name: Upload logs if: always() @@ -171,18 +178,21 @@ jobs: interrupt-windows: name: windows kill@${{ matrix.label }} env: - # install.ps1 has no equivalent of install.sh's --tauri guard, so the - # workspace-scoped root still works here. + # These legs run install.ps1 WITHOUT --tauri (install.ps1:189-215 rejects a custom + # root under --tauri exactly like install.sh:100-147), so the workspace-scoped + # root is usable here. UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home runs-on: windows-latest timeout-minutes: 60 - continue-on-error: true strategy: fail-fast: false matrix: include: - - {label: studio-deps, marker: 'studio deps'} - - {label: torch, marker: 'Installing PyTorch'} + # install.ps1 parses `--no-torch` (install.ps1:121); `-SkipTorch` matches no + # case in that switch and is silently dropped. The torch leg must NOT skip + # torch, or its marker never appears. + - {label: studio-deps, marker: 'studio deps', installArgs: '--no-torch --local'} + - {label: torch, marker: 'Installing PyTorch', installArgs: '--local'} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -194,13 +204,32 @@ jobs: run: | pwsh -NoProfile -File .github/scripts/interrupt-install.ps1 ` -Marker '${{ matrix.marker }}' -LogPath logs/install.log ` - -InstallArgs '-SkipTorch --local' -KillAtSeconds 1500 + -InstallArgs '${{ matrix.installArgs }}' -KillAtSeconds 1500 + + - name: The kill must have landed where it was aimed + shell: pwsh + run: | + $vals = @{} + foreach ($line in (Get-Content logs/interrupt.env)) { + $kv = $line -split '=', 2 + if ($kv.Count -eq 2) { $vals[$kv[0]] = $kv[1] } + } + Write-Host "reason=$($vals['interrupt_reason']) killed=$($vals['interrupt_killed']) exit=$($vals['installer_exit'])" + if ($vals['interrupt_reason'] -ne 'marker-hit') { + Write-Host "::error::installer never reached '${{ matrix.marker }}' (reason=$($vals['interrupt_reason']))." + Write-Host '::error::This leg proves nothing: without this check it passes via the' + Write-Host '::error::probe NO_CLI safe path, exactly as the POSIX legs once did.' + Get-Content logs/install.log -Tail 30 -ErrorAction SilentlyContinue + exit 1 + } - name: What state is the install in? + id: probe shell: pwsh run: | $bin = Join-Path $env:UNSLOTH_STUDIO_HOME 'unsloth_studio\Scripts\unsloth.exe' if (-not (Test-Path $bin)) { + "verdict=NO_CLI" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 Write-Host '[probe] no unsloth CLI -> preflight reports NotInstalled (safe)' exit 0 } @@ -210,6 +239,36 @@ jobs: # desktop-runtime-check -- it would have failed the very PRs that add them, # no matter how well they worked. python .github/scripts/interrupted_install_probe.py $bin --out probe + $rc = $LASTEXITCODE + $v = (Get-Content probe/verdict.json -Raw | ConvertFrom-Json).verdict + "verdict=$v" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 + exit $rc + + - name: A re-run must repair, not short-circuit + # Same assertion the POSIX legs make. Without it a Windows leg proves only that + # the break was DETECTED, never that install.ps1's version fast path does not + # short-circuit over it -- which is the half of the bug that strands the user. + if: always() && steps.probe.outputs.verdict != 'NO_CLI' && steps.probe.outputs.verdict != 'HEALTHY' + shell: pwsh + run: | + pwsh -NoProfile -NonInteractive -File install.ps1 ${{ matrix.installArgs }} *>&1 | + Tee-Object -FilePath logs/repair.log + $bin = Join-Path $env:UNSLOTH_STUDIO_HOME 'unsloth_studio\Scripts\unsloth.exe' + python .github/scripts/interrupted_install_probe.py $bin --out probe-after + $v = (Get-Content probe-after/verdict.json -Raw | ConvertFrom-Json).verdict + # A booting backend IS the repair, whatever the log narrated. Judging by + # log text instead failed a POSIX leg whose venv was fine: the only match + # was the frontend build printing "up to date". + if ($v -eq 'HEALTHY') { + Write-Host 're-run repaired the install (verdict=HEALTHY)' + exit 0 + } + Write-Host "::error::after a full re-run the backend still does not boot (verdict=$v)" + $log = Get-Content logs/repair.log -Raw -ErrorAction SilentlyContinue + if ($log -match '(?i)(venv|dependenc|python stack)[^|]*(up to date|already current)') { + Write-Host '::error::and the re-run treated the venv as current instead of repairing it' + } + exit 1 - name: Upload logs if: always() @@ -219,5 +278,6 @@ jobs: path: | logs/ probe/ + probe-after/ retention-days: 7 if-no-files-found: warn From 987d9996881325afb1372de3cfa657f7c990ab60 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 13:57:24 +0000 Subject: [PATCH 05/25] Drop the interrupt cell that could never be interrupted install.sh --local sets skip_base, so install_python_stack returns before any "base packages" label is printed. The kill had nothing to land on and the installer ran to completion, reaching [TAURI:DONE] in 62s. --- .github/workflows/interrupted-install-ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/interrupted-install-ci.yml b/.github/workflows/interrupted-install-ci.yml index 84741e41a4..7f09174e17 100644 --- a/.github/workflows/interrupted-install-ci.yml +++ b/.github/workflows/interrupted-install-ci.yml @@ -70,7 +70,10 @@ jobs: - {os: macos-14, label: setup, marker: '\[TAURI:STEP\] Running Unsloth setup', experimental: false} # Other dependency-pass steps around the named one. - {os: macos-14, label: pip-bootstrap, marker: 'pip bootstrap', experimental: false} - - {os: macos-14, label: base-packages, marker: 'base packages', experimental: false} + # No base-packages cell: install.sh --local sets skip_base, so + # install_python_stack returns before any "base packages" label is + # printed and the kill can never land. It ran to completion instead, + # proving nothing. - {os: macos-14, label: unsloth-extras, marker: 'unsloth extras', experimental: true} - {os: macos-14, label: data-designer, marker: 'data designer deps', experimental: true} # Linux: same teardown path, different package manager and process semantics. From 3100e90453762091999a0a410746fd06ec51c381 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 15:14:06 +0000 Subject: [PATCH 06/25] Fail the leg when the installer finished instead of being killed The driver set reason=marker-hit before the post-marker sleep and never rechecked, so a step whose work was already cached could run to completion inside that beat and still be recorded as an interruption. The landing assertion tests reason != marker-hit, so a fully completed install passed green having interrupted nothing. Reproduced with a stub that exits during the delay: reported marker-hit / killed=true / exit=0 next to "install finished fully". Set the reason after the sleep, on both drivers. Also trigger on _studio_deps.py and install_manifest.py, where the two decisions the probe asserts on are actually implemented. --- .github/scripts/interrupt-install.ps1 | 5 ++++- .github/scripts/interrupt-install.sh | 10 +++++++++- .github/workflows/interrupted-install-ci.yml | 6 ++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/scripts/interrupt-install.ps1 b/.github/scripts/interrupt-install.ps1 index 038c6a794a..0cdafed048 100644 --- a/.github/scripts/interrupt-install.ps1 +++ b/.github/scripts/interrupt-install.ps1 @@ -65,8 +65,11 @@ for ($i = 0; $i -lt $KillAtSeconds; $i++) { if ($Marker) { $hit = Select-String -Path $LogPath -Pattern $Marker -SimpleMatch:$false -ErrorAction SilentlyContinue if ($hit) { - $reason = 'marker-hit' Start-Sleep -Seconds $KillAfterMarkerSeconds + # The installer can finish inside the delay; recording marker-hit before it + # let a COMPLETED install satisfy the landing assertion and probe HEALTHY. + if ($proc.HasExited) { $reason = 'exited-during-marker-delay'; break } + $reason = 'marker-hit' $killed = $true break } diff --git a/.github/scripts/interrupt-install.sh b/.github/scripts/interrupt-install.sh index 9f3fa3a49d..e805b11d90 100755 --- a/.github/scripts/interrupt-install.sh +++ b/.github/scripts/interrupt-install.sh @@ -56,10 +56,18 @@ for i in $(seq 1 "$KILL_AT_SECONDS"); do break fi if [ -n "$MARKER" ] && grep -qE "$MARKER" "$LOG" 2>/dev/null; then - reason="marker-hit" # Let it get a beat into the step, so the kill lands mid-work rather than on the # boundary where the step has not started touching the venv yet. sleep "${KILL_AFTER_MARKER_SECONDS:-3}" + # ...but a late step whose work is already cached can FINISH inside that beat. + # Recording marker-hit before the sleep handed the landing assertion a COMPLETED + # install: the signal reached no process, the probe read HEALTHY, and the leg + # passed green having interrupted nothing. Set the reason after, not before. + if ! kill -0 "$PID" 2>/dev/null; then + reason="exited-during-marker-delay" + break + fi + reason="marker-hit" killed=true break fi diff --git a/.github/workflows/interrupted-install-ci.yml b/.github/workflows/interrupted-install-ci.yml index 7f09174e17..319cf4ba4e 100644 --- a/.github/workflows/interrupted-install-ci.yml +++ b/.github/workflows/interrupted-install-ci.yml @@ -31,6 +31,12 @@ on: - 'studio/src-tauri/src/preflight.rs' - 'studio/src-tauri/src/preflight/**' - 'unsloth_cli/commands/studio.py' + # studio_install_ok and verify-install, the two decisions the probe asserts + # on, are implemented here rather than in commands/studio.py, so a change + # that made install_state() accept a missing manifest would otherwise merge + # without a single leg running. + - 'unsloth_cli/_studio_deps.py' + - 'studio/install_manifest.py' # `*` never matches `/`, and it is a literal `-install` that follows, so # `interrupt*-install*` matches interrupt-install.sh / .ps1 but NOT the # underscored probe. List the probe explicitly rather than rely on a glob. From 80fc65dfe3b5b7645ce01ed7f6a54ab503393852 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 19:24:53 +0000 Subject: [PATCH 07/25] Kill the group, and stop the probe blocking on a full pipe The escalation was gated on the leader still being alive, so a leader that exits promptly on SIGTERM while a uv or python descendant ignores it skipped the SIGKILL entirely, and wait reaped only the leader. Proven with a descendant that traps TERM: pre-fix its heartbeat keeps ticking while the probe would be running, post-fix it stops. Signal the group unconditionally and drain it after the reap, since an unreaped leader is still a member of its own group. The probe started the backend on stdout=PIPE and read nothing until after the poll loop, so a backend logging more than the pipe buffer during import blocked before binding. Measured 65536 bytes here; a child emitting 200 KB never reaches its bind line, which would make backend_ok false for a healthy install. Write straight to the artefact file. Also trigger on studio/backend/requirements/**, where structlog is declared. --- .github/scripts/interrupt-install.sh | 25 ++++++++++++++++---- .github/scripts/interrupted_install_probe.py | 17 +++++++------ .github/workflows/interrupted-install-ci.yml | 5 ++++ 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/.github/scripts/interrupt-install.sh b/.github/scripts/interrupt-install.sh index e805b11d90..c6cfe9b0f8 100755 --- a/.github/scripts/interrupt-install.sh +++ b/.github/scripts/interrupt-install.sh @@ -86,14 +86,31 @@ if [ "$killed" = "true" ]; then kill -0 "$PID" 2>/dev/null || break sleep 1 done - if kill -0 "$PID" 2>/dev/null; then - echo "[interrupt] group survived SIGTERM; SIGKILL" - kill -KILL -- -"$PID" 2>/dev/null || kill -KILL "$PID" 2>/dev/null || true - fi + # Unconditional, and to the GROUP. The leader can exit on SIGTERM while a uv or + # python descendant ignores it or is mid-shutdown; `kill -0 "$PID"` then reported + # the leader gone, this escalation was skipped, and `wait` reaped only the leader, + # leaving that descendant free to finish the dependency pass while the probe ran. + # Signalling an already-empty group is a no-op. + echo "[interrupt] SIGKILL to process group -$PID" + kill -KILL -- -"$PID" 2>/dev/null || kill -KILL "$PID" 2>/dev/null || true fi wait "$PID" 2>/dev/null rc=$? + +# Only after the reap: an unreaped leader is still a member of its own group, so +# polling the group before `wait` would report it alive forever. Do not let the +# probe start while an installer process is still running. +if [ "$killed" = "true" ]; then + for _ in $(seq 1 "$KILL_GRACE"); do + kill -0 -- -"$PID" 2>/dev/null || break + kill -KILL -- -"$PID" 2>/dev/null || true + sleep 1 + done + if kill -0 -- -"$PID" 2>/dev/null; then + echo "::warning::processes from installer group -$PID outlived SIGKILL" + fi +fi echo "[interrupt] installer exit=$rc reason=$reason killed=$killed" echo "[interrupt] last log lines:" tail -15 "$LOG" || true diff --git a/.github/scripts/interrupted_install_probe.py b/.github/scripts/interrupted_install_probe.py index dd5a9c2254..e7a27c98b9 100644 --- a/.github/scripts/interrupted_install_probe.py +++ b/.github/scripts/interrupted_install_probe.py @@ -135,9 +135,16 @@ def main(argv: list[str]) -> int: popen_kw["start_new_session"] = True else: popen_kw["creationflags"] = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) + # Straight to the artefact file, never a PIPE: nothing reads that pipe until after + # the polling loop, so a backend whose imports emit more than the OS pipe buffer + # (64 KiB on Linux and macOS, a single page by default on Windows) blocks on write + # BEFORE it binds the port. backend_ok, which this whole verdict pivots on, would + # then be false for a perfectly good install. + blog_path = out / "backend.log" + blog_fh = blog_path.open("w", encoding = "utf-8", errors = "replace") proc = subprocess.Popen( [binp, "studio", "--api-only", "-H", "127.0.0.1", "-p", str(port)], - stdout = subprocess.PIPE, + stdout = blog_fh, stderr = subprocess.STDOUT, text = True, **popen_kw, @@ -180,12 +187,8 @@ def main(argv: list[str]) -> int: proc.kill() reap() - try: - blog = proc.communicate(timeout = 30)[0] or "" - except subprocess.TimeoutExpired: - proc.kill() - blog = proc.communicate()[0] or "" - (out / "backend.log").write_text(blog, encoding = "utf-8", errors = "replace") + blog_fh.close() + blog = blog_path.read_text(encoding = "utf-8", errors = "replace") say("backend_ok", backend_ok) missing = "" diff --git a/.github/workflows/interrupted-install-ci.yml b/.github/workflows/interrupted-install-ci.yml index 319cf4ba4e..358141f456 100644 --- a/.github/workflows/interrupted-install-ci.yml +++ b/.github/workflows/interrupted-install-ci.yml @@ -37,6 +37,11 @@ on: # without a single leg running. - 'unsloth_cli/_studio_deps.py' - 'studio/install_manifest.py' + # The requirement files are the phases. studio.txt is where structlog is + # declared, the package whose absence IS the reported false-ready bug, and + # the single-env files drive the later steps, so moving a package between + # them changes what every interrupted state looks like. + - 'studio/backend/requirements/**' # `*` never matches `/`, and it is a literal `-install` that follows, so # `interrupt*-install*` matches interrupt-install.sh / .ps1 but NOT the # underscored probe. List the probe explicitly rather than rely on a glob. From 3e7fc344bbd2d09016f782e285f6fa8fb0315f92 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 28 Jul 2026 20:01:10 +0000 Subject: [PATCH 08/25] Make the NO_CLI legs assert repair, and fix the Windows straggler sweep Two of the interrupted-install legs were passing without testing anything. The re-run assertion skipped verdict=NO_CLI, but a kill at "venv" or "torch" lands before install.sh ever prints "Installing Unsloth" (:2125, :3667, :3961), so those legs can only ever produce NO_CLI. Three non-gating-exempt cells (macos-14 kill@venv, macos-14 kill@torch, ubuntu-latest kill@torch) therefore asserted nothing beyond a marker appearing in a log. NO_CLI is now included: a re-run must produce a booting backend regardless of how little the first run managed to install. Each re-run step grows an existence check first, because the probe exits without writing verdict.json when the binary is absent and the json.load would crash rather than report. The Windows straggler sweep matched nothing at all. UNSLOTH_STUDIO_HOME arrives as D:\a\r\r/.studio-home, since the workflow joins ${{ github.workspace }} with a forward slash, while Process.Path is all backslashes, so the literal -like missed even the venv's own python.exe. uv is never under the studio home in any case: install.ps1 takes it from winget or astral.sh. Normalise the separators, match uv by name (the runner is ephemeral and runs no other uv), and skip the home comparison entirely when the variable is empty, which would otherwise turn the pattern into "**" and kill every python on the runner. --- .github/scripts/interrupt-install.ps1 | 21 +++++++++++---- .github/workflows/interrupted-install-ci.yml | 28 +++++++++++++++----- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/.github/scripts/interrupt-install.ps1 b/.github/scripts/interrupt-install.ps1 index 0cdafed048..6860ad0107 100644 --- a/.github/scripts/interrupt-install.ps1 +++ b/.github/scripts/interrupt-install.ps1 @@ -81,11 +81,22 @@ if (-not $killed -and -not $proc.HasExited) { if (-not $reason) { $reason = 'dea if ($killed) { Write-Host "[interrupt] killing process tree of $($proc.Id) ($reason)" Stop-Tree $proc.Id - # Any straggler uv/python that reparented away from the installer. - foreach ($name in 'uv', 'python') { - Get-Process -Name $name -ErrorAction SilentlyContinue | - Where-Object { $_.Path -and $_.Path -like "*$env:UNSLOTH_STUDIO_HOME*" } | - ForEach-Object { try { Stop-Process -Id $_.Id -Force } catch { } } + # Any straggler uv/python that reparented away from the installer. The old sweep + # matched nothing: UNSLOTH_STUDIO_HOME arrives as `D:\a\r\r/.studio-home` + # (github.workspace joined with a forward slash) while Process.Path is all + # backslashes, so the literal -like missed even the venv's own python, and uv is + # never under the studio home anyway (install.ps1 takes it from winget or + # astral.sh). Normalise the separators, and take uv by name since the runner is + # ephemeral and runs no other uv. + $homeNorm = if ([string]::IsNullOrWhiteSpace($env:UNSLOTH_STUDIO_HOME)) { $null } + else { ($env:UNSLOTH_STUDIO_HOME -replace '/', '\').TrimEnd('\') } + foreach ($p in @(Get-Process -Name 'uv', 'python', 'pythonw' -ErrorAction SilentlyContinue)) { + $path = $null + try { $path = $p.Path } catch { } + $inHome = $homeNorm -and $path -and ($path -like "$homeNorm\*") + if ($p.ProcessName -eq 'uv' -or $inHome) { + try { Stop-Process -Id $p.Id -Force; Write-Host "[interrupt] swept $($p.ProcessName) pid=$($p.Id)" } catch { } + } } } diff --git a/.github/workflows/interrupted-install-ci.yml b/.github/workflows/interrupted-install-ci.yml index 358141f456..c59b31fdd2 100644 --- a/.github/workflows/interrupted-install-ci.yml +++ b/.github/workflows/interrupted-install-ci.yml @@ -151,9 +151,12 @@ jobs: exit "$rc" - name: A re-run must repair, not short-circuit - # Only meaningful when the install is broken but present. The bug's second half - # is that `install.sh` sees a "current" version and no-ops over a broken venv. - if: always() && steps.probe.outputs.verdict != 'NO_CLI' && steps.probe.outputs.verdict != 'HEALTHY' + # NO_CLI included. A kill at venv or torch lands before "Installing Unsloth" + # (install.sh:2125 / :3667 / :3961), so those legs always take NO_CLI, and + # skipping the re-run left three non-experimental legs asserting nothing but + # that a marker appeared. The bug's second half is that `install.sh` sees a + # "current" version and no-ops over a broken venv. + if: always() && steps.probe.outputs.verdict != 'HEALTHY' run: | set -o pipefail rc=0 @@ -161,6 +164,13 @@ jobs: echo "repair exit: $rc" BIN="$HOME/.unsloth/studio/unsloth_studio/bin/unsloth" [ -x "$BIN" ] || BIN="$HOME/.unsloth/studio/bin/unsloth" + # The probe exits without writing verdict.json when the bin is missing, so + # check here or the json.load below crashes instead of reporting. + if [ ! -x "$BIN" ]; then + echo "::error::after a full re-run there is still no unsloth CLI at $BIN" + tail -30 logs/repair.log || true + exit 1 + fi python3 .github/scripts/interrupted_install_probe.py "$BIN" --out probe-after || true v="$(python3 -c "import json;print(json.load(open('probe-after/verdict.json'))['verdict'])")" # A booting backend IS the repair, whatever the log narrated. Judging by @@ -259,15 +269,21 @@ jobs: exit $rc - name: A re-run must repair, not short-circuit - # Same assertion the POSIX legs make. Without it a Windows leg proves only that + # Same assertion the POSIX legs make, NO_CLI included: a leg that left no CLI + # otherwise asserts nothing, and without this a Windows leg proves only that # the break was DETECTED, never that install.ps1's version fast path does not - # short-circuit over it -- which is the half of the bug that strands the user. - if: always() && steps.probe.outputs.verdict != 'NO_CLI' && steps.probe.outputs.verdict != 'HEALTHY' + # short-circuit over it, which is the half of the bug that strands the user. + if: always() && steps.probe.outputs.verdict != 'HEALTHY' shell: pwsh run: | pwsh -NoProfile -NonInteractive -File install.ps1 ${{ matrix.installArgs }} *>&1 | Tee-Object -FilePath logs/repair.log $bin = Join-Path $env:UNSLOTH_STUDIO_HOME 'unsloth_studio\Scripts\unsloth.exe' + if (-not (Test-Path $bin)) { + Write-Host "::error::after a full re-run there is still no unsloth CLI at $bin" + Get-Content logs/repair.log -Tail 30 -ErrorAction SilentlyContinue + exit 1 + } python .github/scripts/interrupted_install_probe.py $bin --out probe-after $v = (Get-Content probe-after/verdict.json -Raw | ConvertFrom-Json).verdict # A booting backend IS the repair, whatever the log narrated. Judging by From 5c65d06b0f10f6fc89bc2ea88985b43c4e9a96cf Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 28 Jul 2026 20:11:35 +0000 Subject: [PATCH 09/25] Run the Windows legs as the desktop does, and judge repair by what preflight reads The Windows matrix set a workspace-scoped UNSLOTH_STUDIO_HOME, which forces install.ps1 down the shell-install path: install.ps1:189-215 rejects a custom root under --tauri, so those legs ran with UNSLOTH_TAURI_MODE=0, the frontend build on and no bundled-file overlay, while the desktop always spawns the installer as --tauri with the variable scrubbed (install.rs:202 and :356). The torch leg could not even reach its marker: "Installing PyTorch" is printed only by Write-TauriLog (install.ps1:2440), so it was killed at the deadline. Both legs now run --tauri --local at the default root, and the probe and re-run resolve the CLI under %USERPROFILE%\.unsloth\studio. The probe counted `studio verify-install` and `studio desktop-runtime-check` failures as proof the app can repair, but preflight/managed.rs runs only `-h` and `studio desktop-capabilities --json` (:357) and reads studio_install_ok from that payload (:445); neither deeper command is invoked anywhere under studio/src-tauri. A leg where capabilities regressed to ready while only those standalone commands saw the damage would have passed green with the app stuck on ManagedReady, which is the exact false negative this workflow exists to catch. They are still run and recorded in verdict.json, just no longer repair evidence. An interrupted install can leave the console script in place while its venv interpreter is gone. The probes go through run(), which catches OSError, but the backend spawn did not, so the probe aborted before writing verdict.json and both workflows died on the json.load instead of reporting. That state is now recorded as backend_spawn_error and lands on REPAIRABLE, which is what `-h` failing already implies. On win32 the CLI re-spawns the server as a child and waits on it (unsloth_cli/commands/studio.py:1543), and CREATE_NEW_PROCESS_GROUP does not make terminate() reach descendants, so the reap left a server holding the venv open while the repair step reinstalled into files Windows had locked. Use taskkill /F /T for the tree. The straggler sweep now falls back to the default studio root, since under --tauri there is no UNSLOTH_STUDIO_HOME to match on. --- .github/scripts/interrupt-install.ps1 | 10 ++-- .github/scripts/interrupted_install_probe.py | 53 ++++++++++++++------ .github/workflows/interrupted-install-ci.yml | 22 ++++---- 3 files changed, 59 insertions(+), 26 deletions(-) diff --git a/.github/scripts/interrupt-install.ps1 b/.github/scripts/interrupt-install.ps1 index 6860ad0107..92f6019c31 100644 --- a/.github/scripts/interrupt-install.ps1 +++ b/.github/scripts/interrupt-install.ps1 @@ -12,7 +12,7 @@ # # Usage: # pwsh -File .github/scripts/interrupt-install.ps1 -Marker 'studio deps' ` -# -LogPath logs/install.log -InstallArgs '-SkipTorch' +# -LogPath logs/install.log -InstallArgs '--tauri --no-torch --local' [CmdletBinding()] param( [string]$Marker = '', @@ -88,8 +88,12 @@ if ($killed) { # never under the studio home anyway (install.ps1 takes it from winget or # astral.sh). Normalise the separators, and take uv by name since the runner is # ephemeral and runs no other uv. - $homeNorm = if ([string]::IsNullOrWhiteSpace($env:UNSLOTH_STUDIO_HOME)) { $null } - else { ($env:UNSLOTH_STUDIO_HOME -replace '/', '\').TrimEnd('\') } + # Under --tauri there is no UNSLOTH_STUDIO_HOME, so fall back to the root + # install.ps1 uses then, or the sweep would only ever see uv. + $studioRoot = if ([string]::IsNullOrWhiteSpace($env:UNSLOTH_STUDIO_HOME)) { Join-Path $HOME '.unsloth\studio' } + else { $env:UNSLOTH_STUDIO_HOME } + $homeNorm = if ([string]::IsNullOrWhiteSpace($studioRoot)) { $null } + else { ($studioRoot -replace '/', '\').TrimEnd('\') } foreach ($p in @(Get-Process -Name 'uv', 'python', 'pythonw' -ErrorAction SilentlyContinue)) { $path = $null try { $path = $p.Path } catch { } diff --git a/.github/scripts/interrupted_install_probe.py b/.github/scripts/interrupted_install_probe.py index e7a27c98b9..6c2015facd 100644 --- a/.github/scripts/interrupted_install_probe.py +++ b/.github/scripts/interrupted_install_probe.py @@ -22,7 +22,8 @@ ManagedReady with can_auto_repair=false and the backend dies on `import structlo Verdicts: HEALTHY the backend boots -- the interruption did no lasting harm - REPAIRABLE the backend is broken AND something reports it, so the app can repair + REPAIRABLE the backend is broken AND a probe the DESKTOP consumes reports it, + so the app can offer a repair FALSE_READY the backend is broken and every probe says ready -> THE BUG Exit: 0 for HEALTHY/REPAIRABLE/NO_CLI, 1 for FALSE_READY, 2 for a usage error. @@ -107,6 +108,12 @@ def main(argv: list[str]) -> int: say("capabilities.studio_install_ok", install_ok) # ── the deeper probes the fix PRs add ──────────────────────────────────── + # RECORDED, but NOT repair evidence: preflight/managed.rs runs only `-h` and + # `studio desktop-capabilities --json` (managed.rs:357) and reads + # studio_install_ok from that payload (managed.rs:445). It never invokes these + # two commands, so counting them would let a leg pass while the real app still + # reports ManagedReady over a torn install -- the exact false negative this + # workflow exists to catch. for label, args in ( ("verify_install", ["studio", "verify-install"]), ("desktop_runtime_check", ["studio", "desktop-runtime-check"]), @@ -142,16 +149,25 @@ def main(argv: list[str]) -> int: # then be false for a perfectly good install. blog_path = out / "backend.log" blog_fh = blog_path.open("w", encoding = "utf-8", errors = "replace") - proc = subprocess.Popen( - [binp, "studio", "--api-only", "-H", "127.0.0.1", "-p", str(port)], - stdout = blog_fh, - stderr = subprocess.STDOUT, - text = True, - **popen_kw, - ) + # An interrupted install can leave the console script in place while its venv + # interpreter is gone: the earlier probes then report failure through run()'s + # OSError catch, but an unguarded spawn here raises instead, so no verdict.json + # is written and both workflows die on the json.load rather than reporting. An + # unlaunchable CLI is a broken backend that `-h` already flags -> REPAIRABLE. + proc = None + try: + proc = subprocess.Popen( + [binp, "studio", "--api-only", "-H", "127.0.0.1", "-p", str(port)], + stdout = blog_fh, + stderr = subprocess.STDOUT, + text = True, + **popen_kw, + ) + except OSError as e: + say("backend_spawn_error", f"{type(e).__name__}: {e}") backend_ok = False deadline = time.time() + a.boot_timeout - while time.time() < deadline: + while proc is not None and time.time() < deadline: if proc.poll() is not None: break for path in ("/api/health", "/healthz"): @@ -167,6 +183,8 @@ def main(argv: list[str]) -> int: time.sleep(1) def reap() -> None: + if proc is None: + return if os.name == "posix": import signal for sig in (signal.SIGTERM, signal.SIGKILL): @@ -180,11 +198,20 @@ def main(argv: list[str]) -> int: except subprocess.TimeoutExpired: continue else: - proc.terminate() + # On win32 the CLI re-spawns the server as a CHILD and waits on it + # (unsloth_cli/commands/studio.py:1543); CREATE_NEW_PROCESS_GROUP does not + # make terminate() reach descendants, so killing the wrapper alone leaves a + # server holding the venv open and the repair step reinstalls into files + # Windows has locked. taskkill /T takes the tree. + run(["taskkill", "/F", "/T", "/PID", str(proc.pid)], timeout = 30) try: proc.wait(timeout = 10) except subprocess.TimeoutExpired: - proc.kill() + proc.terminate() + try: + proc.wait(timeout = 10) + except subprocess.TimeoutExpired: + proc.kill() reap() blog_fh.close() @@ -202,9 +229,7 @@ def main(argv: list[str]) -> int: if backend_ok: verdict = "HEALTHY" elif ( - facts.get("verify_install") == "failed" - or facts.get("desktop_runtime_check") == "failed" - or facts.get("capabilities.studio_install_ok") is False + facts.get("capabilities.studio_install_ok") is False or not facts.get("cli_h_ok") or not facts.get("capabilities_ok") ): diff --git a/.github/workflows/interrupted-install-ci.yml b/.github/workflows/interrupted-install-ci.yml index c59b31fdd2..7d8d3438b4 100644 --- a/.github/workflows/interrupted-install-ci.yml +++ b/.github/workflows/interrupted-install-ci.yml @@ -201,11 +201,14 @@ jobs: # ── Windows: no process groups, so the kill path differs ────────────────── interrupt-windows: name: windows kill@${{ matrix.label }} - env: - # These legs run install.ps1 WITHOUT --tauri (install.ps1:189-215 rejects a custom - # root under --tauri exactly like install.sh:100-147), so the workspace-scoped - # root is usable here. - UNSLOTH_STUDIO_HOME: ${{ github.workspace }}/.studio-home + # No UNSLOTH_STUDIO_HOME here. The desktop app always invokes the installer as + # `--tauri [--local]` and scrubs the variable first (install.rs:202 and :356), and + # install.ps1:189-215 rejects a custom root under --tauri, so a workspace-scoped + # root forced these legs down the shell-install path instead: UNSLOTH_TAURI_MODE=0, + # frontend build enabled, different root resolution, no bundled-file overlay. Worse, + # "Installing PyTorch" is only ever printed by Write-TauriLog (install.ps1:2440), so + # without --tauri the torch leg's marker could not appear at all. The runner is + # ephemeral, so the default root is safe to install into. runs-on: windows-latest timeout-minutes: 60 strategy: @@ -215,8 +218,8 @@ jobs: # install.ps1 parses `--no-torch` (install.ps1:121); `-SkipTorch` matches no # case in that switch and is silently dropped. The torch leg must NOT skip # torch, or its marker never appears. - - {label: studio-deps, marker: 'studio deps', installArgs: '--no-torch --local'} - - {label: torch, marker: 'Installing PyTorch', installArgs: '--local'} + - {label: studio-deps, marker: 'studio deps', installArgs: '--tauri --no-torch --local'} + - {label: torch, marker: 'Installing PyTorch', installArgs: '--tauri --local'} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -251,7 +254,8 @@ jobs: id: probe shell: pwsh run: | - $bin = Join-Path $env:UNSLOTH_STUDIO_HOME 'unsloth_studio\Scripts\unsloth.exe' + # --tauri refuses a custom root, so this is where install.ps1:254-262 puts it. + $bin = Join-Path $env:USERPROFILE '.unsloth\studio\unsloth_studio\Scripts\unsloth.exe' if (-not (Test-Path $bin)) { "verdict=NO_CLI" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 Write-Host '[probe] no unsloth CLI -> preflight reports NotInstalled (safe)' @@ -278,7 +282,7 @@ jobs: run: | pwsh -NoProfile -NonInteractive -File install.ps1 ${{ matrix.installArgs }} *>&1 | Tee-Object -FilePath logs/repair.log - $bin = Join-Path $env:UNSLOTH_STUDIO_HOME 'unsloth_studio\Scripts\unsloth.exe' + $bin = Join-Path $env:USERPROFILE '.unsloth\studio\unsloth_studio\Scripts\unsloth.exe' if (-not (Test-Path $bin)) { Write-Host "::error::after a full re-run there is still no unsloth CLI at $bin" Get-Content logs/repair.log -Tail 30 -ErrorAction SilentlyContinue From d87c998a8494827b9ac410c6aa09154703ac4075 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 22:01:16 +0000 Subject: [PATCH 10/25] Judge the install the way preflight does, and reap the whole probe group Read desktop-capabilities the way the desktop reads it. preflight/managed.rs pipes stdout and sends stderr to /dev/null (managed.rs:358), then hands the whole stdout buffer to serde_json (managed.rs:414). The probe concatenated both streams and scanned to the first brace, so a single diagnostic line on stderr made json.loads raise on the trailing text, studio_install_ok stayed "absent", and a broken backend was reported FALSE_READY over an install the real app parses, sees as incomplete, and offers to repair. That fails a valid recovery change for a reason that exists only in the probe. stdout and stderr are now captured separately and stdout is parsed strictly; a payload that does not parse counts as repair evidence, matching the Stale the desktop reports when the capability probe returns nothing (managed.rs:521). A booting backend alone is not a finished install. The manifest is written last (install_python_stack.py:3255), so a kill after "studio deps" but before it, the data-designer leg, leaves a venv whose backend boots while desktop-capabilities still reports studio_install_ok=false and preflight reports Stale (managed.rs:445). Calling that HEALTHY skipped the re-run step, so the leg asserted nothing beyond a marker appearing and never exercised the version fast path that is supposed to clear an incomplete install, which is the half of the bug that strands the user. HEALTHY now requires both. Escalate to the process group after reaping the probe's backend. reap() returned as soon as proc.wait() succeeded, and the leader exits promptly on SIGTERM while a uvicorn worker does not, so the SIGKILL iteration was skipped and that worker kept the port and the venv open while the repair step reinstalled underneath it. It also read os.getpgid(proc.pid) after the reap, which raises. The pgid is now captured up front and SIGKILL always goes to the group, the same escalation interrupt-install.sh:94 makes. A heartbeat experiment left the group alive with the old sequence and empty with the new one. Trigger the workflow on pyproject.toml. Every leg installs the checkout with --local, so that file decides the unsloth console script and the core dependencies the probe leans on: -h and desktop-capabilities only survive a torn install because typer/click/rich are declared there. No other install workflow interrupts the installer, so such a change would otherwise merge without a single leg running. --- .github/scripts/interrupted_install_probe.py | 109 ++++++++++++++----- .github/workflows/interrupted-install-ci.yml | 13 +++ 2 files changed, 94 insertions(+), 28 deletions(-) diff --git a/.github/scripts/interrupted_install_probe.py b/.github/scripts/interrupted_install_probe.py index 6c2015facd..59d76f9af0 100644 --- a/.github/scripts/interrupted_install_probe.py +++ b/.github/scripts/interrupted_install_probe.py @@ -21,7 +21,8 @@ probes `unsloth -h` (preflight/managed.rs:419) and `studio desktop-capabilities` ManagedReady with can_auto_repair=false and the backend dies on `import structlog`. Verdicts: - HEALTHY the backend boots -- the interruption did no lasting harm + HEALTHY the backend boots AND desktop-capabilities reports the install + complete -- i.e. preflight would report ManagedReady and be right REPAIRABLE the backend is broken AND a probe the DESKTOP consumes reports it, so the app can offer a repair FALSE_READY the backend is broken and every probe says ready -> THE BUG @@ -43,18 +44,26 @@ import urllib.request from pathlib import Path -def run(cmd: list[str], timeout: int = 120) -> tuple[int, str]: +def run(cmd: list[str], timeout: int = 120) -> tuple[int, str, str]: + """Returns (rc, stdout, stderr). Kept SEPARATE: preflight/managed.rs pipes stdout + and sends stderr to /dev/null (managed.rs:358), so anything the probe folds into + stdout is text the desktop never sees.""" try: p = subprocess.run(cmd, capture_output = True, text = True, timeout = timeout) - return p.returncode, (p.stdout or "") + (p.stderr or "") + return p.returncode, p.stdout or "", p.stderr or "" except (subprocess.TimeoutExpired, OSError) as e: - return 127, f"{type(e).__name__}: {e}" + return 127, "", f"{type(e).__name__}: {e}" + + +def merged(rc_out_err: tuple[int, str, str]) -> str: + """Both streams, for artefact logs only -- never for parsing.""" + return rc_out_err[1] + rc_out_err[2] def has_subcommand(bin_path: str, args: list[str]) -> bool: """Whether the CLI understands a subcommand at all. Older builds do not have the newer verify commands, and 'absent' must not be confused with 'reported failure'.""" - rc, _ = run([bin_path, *args, "--help"], timeout = 60) + rc, _, _ = run([bin_path, *args, "--help"], timeout = 60) return rc == 0 @@ -86,27 +95,48 @@ def main(argv: list[str]) -> int: print(f"[probe] {k:28} = {v}") # ── the two probes Tauri preflight actually runs ───────────────────────── - rc, log = run([binp, "-h"], timeout = 180) - (out / "cli-h.log").write_text(log, encoding = "utf-8", errors = "replace") - say("cli_h_ok", rc == 0) + r = run([binp, "-h"], timeout = 180) + (out / "cli-h.log").write_text(merged(r), encoding = "utf-8", errors = "replace") + say("cli_h_ok", r[0] == 0) - rc, caps_raw = run([binp, "studio", "desktop-capabilities", "--json"], timeout = 180) - (out / "desktop-capabilities.json").write_text(caps_raw, encoding = "utf-8", errors = "replace") - say("capabilities_ok", rc == 0) + caps_rc, caps_out, caps_err = run([binp, "studio", "desktop-capabilities", "--json"], timeout = 180) + (out / "desktop-capabilities.json").write_text(caps_out, encoding = "utf-8", errors = "replace") + (out / "desktop-capabilities.stderr.log").write_text(caps_err, encoding = "utf-8", errors = "replace") + say("capabilities_ok", caps_rc == 0) + # Parse EXACTLY as the desktop does: managed.rs:414 hands the whole stdout buffer + # to serde_json, which rejects any leading or trailing non-JSON, and stderr was + # already discarded at managed.rs:358. Folding stderr in and then scanning to the + # first brace made one warning line on stderr enough for json.loads to raise on + # the trailing text, leaving studio_install_ok "absent" and reporting FALSE_READY + # over an install the real app parses, sees as incomplete, and offers to repair. + # # studio_install_ok is added by the install-manifest work; absent on older trees, - # which is different from present-and-false. + # which is different from present-and-false. A payload that does not parse at all + # is neither: the desktop gets None back and reports Stale + # ("desktop_capability_probe_failed", managed.rs:521), so it is repair evidence. install_ok: object = "absent" try: - # The CLI may print a banner before the JSON, so start at the first brace. - brace = caps_raw.find("{") - if brace >= 0: - v = json.loads(caps_raw[brace:]).get("studio_install_ok") + parsed = json.loads(caps_out) + if isinstance(parsed, dict): + v = parsed.get("studio_install_ok") install_ok = "absent" if v is None else bool(v) - except (json.JSONDecodeError, AttributeError): - pass + else: + install_ok = "unparseable" + except json.JSONDecodeError: + install_ok = "unparseable" say("capabilities.studio_install_ok", install_ok) + # The desktop's own conclusion: Ready only when the payload parses AND + # studio_install_ok is true (managed.rs:445). "absent" stays undecided so an + # older tree, which cannot answer, is judged on the backend alone. + caps_ready: object = "absent" + if caps_rc != 0 or install_ok is False or install_ok == "unparseable": + caps_ready = False + elif install_ok is True: + caps_ready = True + say("desktop_would_call_install_ok", caps_ready) + # ── the deeper probes the fix PRs add ──────────────────────────────────── # RECORDED, but NOT repair evidence: preflight/managed.rs runs only `-h` and # `studio desktop-capabilities --json` (managed.rs:357) and reads @@ -121,9 +151,9 @@ def main(argv: list[str]) -> int: if not has_subcommand(binp, args): say(label, "absent") continue - rc, log = run([binp, *args], timeout = 300) - (out / f"{label}.log").write_text(log, encoding = "utf-8", errors = "replace") - say(label, "ok" if rc == 0 else "failed") + r = run([binp, *args], timeout = 300) + (out / f"{label}.log").write_text(merged(r), encoding = "utf-8", errors = "replace") + say(label, "ok" if r[0] == 0 else "failed") # The in-progress marker #7490 writes before spawning the installer. RECORDED ONLY, # never used as repair evidence: both interrupt drivers seed it before every install @@ -187,16 +217,33 @@ def main(argv: list[str]) -> int: return if os.name == "posix": import signal + # start_new_session made this child its own group leader, so pgid == pid. + # Read it BEFORE the reap: once the leader is waited on, os.getpgid() + # raises and the escalation would target nothing. + try: + pgid = os.getpgid(proc.pid) + except OSError: + pgid = proc.pid for sig in (signal.SIGTERM, signal.SIGKILL): try: - os.killpg(os.getpgid(proc.pid), sig) - except (ProcessLookupError, PermissionError, OSError): + os.killpg(pgid, sig) + except OSError: pass try: proc.wait(timeout = 10) - return + break except subprocess.TimeoutExpired: continue + # Unconditional, and to the GROUP -- the same escalation + # interrupt-install.sh:94 makes, for the same reason. The leader exits + # promptly on SIGTERM while a uvicorn worker does not, so returning as + # soon as proc.wait() succeeded skipped the SIGKILL entirely and left + # that worker holding the port and the venv open while the repair step + # reinstalled underneath it. Signalling an empty group is a no-op. + try: + os.killpg(pgid, signal.SIGKILL) + except OSError: + pass else: # On win32 the CLI re-spawns the server as a CHILD and waits on it # (unsloth_cli/commands/studio.py:1543); CREATE_NEW_PROCESS_GROUP does not @@ -226,12 +273,18 @@ def main(argv: list[str]) -> int: say("backend_error", missing) # ── verdict ────────────────────────────────────────────────────────────── - if backend_ok: + # A booting backend is not enough to call the install finished. The manifest is + # written LAST (install_python_stack.py:3255), so a kill after "studio deps" but + # before it -- the data-designer leg -- leaves a venv whose backend boots while + # desktop-capabilities still says studio_install_ok=false, and preflight reports + # Stale (managed.rs:445) rather than Ready. Calling that HEALTHY skipped the + # re-run step, so the leg asserted nothing beyond a marker appearing and never + # exercised the fast path that is supposed to clear an incomplete install. + if backend_ok and caps_ready is not False: verdict = "HEALTHY" elif ( - facts.get("capabilities.studio_install_ok") is False + caps_ready is False or not facts.get("cli_h_ok") - or not facts.get("capabilities_ok") ): verdict = "REPAIRABLE" else: @@ -250,7 +303,7 @@ def main(argv: list[str]) -> int: ) return 1 if verdict == "REPAIRABLE": - print("[probe] broken install is detectable -> the desktop app can auto-repair") + print("[probe] incomplete install is detectable -> the desktop app can auto-repair") return 0 diff --git a/.github/workflows/interrupted-install-ci.yml b/.github/workflows/interrupted-install-ci.yml index 7d8d3438b4..94de2249a9 100644 --- a/.github/workflows/interrupted-install-ci.yml +++ b/.github/workflows/interrupted-install-ci.yml @@ -31,6 +31,14 @@ on: - 'studio/src-tauri/src/preflight.rs' - 'studio/src-tauri/src/preflight/**' - 'unsloth_cli/commands/studio.py' + # Every leg installs the checkout with `--local`, so this file decides the + # `unsloth` console script and the core dependencies the probe leans on: + # `-h` and `desktop-capabilities` only survive a torn install because + # typer/click/rich are declared here. Moving one of those to an extra + # changes what every interrupted venv looks like, and no other install + # workflow interrupts the installer, so such a PR would otherwise merge + # without a single leg running. + - 'pyproject.toml' # studio_install_ok and verify-install, the two decisions the probe asserts # on, are implemented here rather than in commands/studio.py, so a change # that made install_state() accept a missing manifest would otherwise merge @@ -156,6 +164,11 @@ jobs: # skipping the re-run left three non-experimental legs asserting nothing but # that a marker appeared. The bug's second half is that `install.sh` sees a # "current" version and no-ops over a broken venv. + # + # HEALTHY means the backend boots AND desktop-capabilities reports the + # install complete, so a leg killed after "studio deps" but before the + # manifest is written last (install_python_stack.py:3255) -- data-designer -- + # arrives here rather than skipping the one assertion that matters for it. if: always() && steps.probe.outputs.verdict != 'HEALTHY' run: | set -o pipefail From 18b7de14e1c30222bf4b561a2b1ffb8c49b1c753 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 22:02:00 +0000 Subject: [PATCH 11/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/scripts/interrupted_install_probe.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/scripts/interrupted_install_probe.py b/.github/scripts/interrupted_install_probe.py index 59d76f9af0..8a1c3c5098 100644 --- a/.github/scripts/interrupted_install_probe.py +++ b/.github/scripts/interrupted_install_probe.py @@ -99,9 +99,13 @@ def main(argv: list[str]) -> int: (out / "cli-h.log").write_text(merged(r), encoding = "utf-8", errors = "replace") say("cli_h_ok", r[0] == 0) - caps_rc, caps_out, caps_err = run([binp, "studio", "desktop-capabilities", "--json"], timeout = 180) + caps_rc, caps_out, caps_err = run( + [binp, "studio", "desktop-capabilities", "--json"], timeout = 180 + ) (out / "desktop-capabilities.json").write_text(caps_out, encoding = "utf-8", errors = "replace") - (out / "desktop-capabilities.stderr.log").write_text(caps_err, encoding = "utf-8", errors = "replace") + (out / "desktop-capabilities.stderr.log").write_text( + caps_err, encoding = "utf-8", errors = "replace" + ) say("capabilities_ok", caps_rc == 0) # Parse EXACTLY as the desktop does: managed.rs:414 hands the whole stdout buffer @@ -217,6 +221,7 @@ def main(argv: list[str]) -> int: return if os.name == "posix": import signal + # start_new_session made this child its own group leader, so pgid == pid. # Read it BEFORE the reap: once the leader is waited on, os.getpgid() # raises and the escalation would target nothing. @@ -282,10 +287,7 @@ def main(argv: list[str]) -> int: # exercised the fast path that is supposed to clear an incomplete install. if backend_ok and caps_ready is not False: verdict = "HEALTHY" - elif ( - caps_ready is False - or not facts.get("cli_h_ok") - ): + elif caps_ready is False or not facts.get("cli_h_ok"): verdict = "REPAIRABLE" else: verdict = "FALSE_READY" From 1dfa31ad04e5874d4599ffaffa44005c6539da59 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 23:18:48 +0000 Subject: [PATCH 12/25] Judge an absent capability field and a dead -h the way preflight does The probe left studio_install_ok=absent undecided and judged those installs on whether the backend booted. preflight/managed.rs:445 tests studio_install_ok != Some(true), so an absent field is Stale exactly like a false one; a CLI too old to carry it is already rejected one check earlier on desktop_manageability_version. The gap mattered in both directions: a payload that stopped carrying the field reported HEALTHY on every booting leg and skipped the re-run assertion this workflow exists to make, and a torn venv with a working -h was failed as FALSE_READY even though the app would have offered repair. unsloth_cli/commands/studio.py is in this workflow's path filter precisely to catch that class of change, so it must not be the thing that silences it. The verdict also consulted cli_h_ok only in the repairable arm, so a CLI that cannot print help was called HEALTHY whenever the backend happened to boot. probe_managed_bin runs -h first and returns Stale cli_unusable before it ever reaches the capability probe (managed.rs:465-478), so that install goes to repair in the real app and the leg must assert it here. --- .github/scripts/interrupted_install_probe.py | 38 ++++++++++++-------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/.github/scripts/interrupted_install_probe.py b/.github/scripts/interrupted_install_probe.py index 8a1c3c5098..f05b466fa3 100644 --- a/.github/scripts/interrupted_install_probe.py +++ b/.github/scripts/interrupted_install_probe.py @@ -115,10 +115,11 @@ def main(argv: list[str]) -> int: # the trailing text, leaving studio_install_ok "absent" and reporting FALSE_READY # over an install the real app parses, sees as incomplete, and offers to repair. # - # studio_install_ok is added by the install-manifest work; absent on older trees, - # which is different from present-and-false. A payload that does not parse at all - # is neither: the desktop gets None back and reports Stale - # ("desktop_capability_probe_failed", managed.rs:521), so it is repair evidence. + # studio_install_ok is added by the install-manifest work, so it is absent on older + # trees; that is recorded separately from present-and-false only to make the + # artefact readable, because the desktop treats both as Stale. A payload that does + # not parse at all is a third case with the same outcome: the desktop gets None + # back and reports Stale ("desktop_capability_probe_failed", managed.rs:521). install_ok: object = "absent" try: parsed = json.loads(caps_out) @@ -131,14 +132,17 @@ def main(argv: list[str]) -> int: install_ok = "unparseable" say("capabilities.studio_install_ok", install_ok) - # The desktop's own conclusion: Ready only when the payload parses AND - # studio_install_ok is true (managed.rs:445). "absent" stays undecided so an - # older tree, which cannot answer, is judged on the backend alone. - caps_ready: object = "absent" - if caps_rc != 0 or install_ok is False or install_ok == "unparseable": - caps_ready = False - elif install_ok is True: - caps_ready = True + # The desktop's own conclusion: Ready only when the probe exits 0, the payload + # parses, AND studio_install_ok is true. The predicate is `!= Some(true)` + # (managed.rs:445), so an ABSENT field is Stale exactly like a false one -- a CLI + # too old to answer is already rejected one check earlier on + # desktop_manageability_version. Leaving "absent" undecided judged those installs + # on the backend alone, so a payload that stopped carrying the field reported + # HEALTHY on every booting leg and skipped the repair assertion this workflow + # exists to make, while the real app showed Stale and offered repair. That is the + # regression `unsloth_cli/commands/studio.py` is in this workflow's path filter to + # catch, so it must never be the thing that silences it. + caps_ready = caps_rc == 0 and install_ok is True say("desktop_would_call_install_ok", caps_ready) # ── the deeper probes the fix PRs add ──────────────────────────────────── @@ -285,9 +289,15 @@ def main(argv: list[str]) -> int: # Stale (managed.rs:445) rather than Ready. Calling that HEALTHY skipped the # re-run step, so the leg asserted nothing beyond a marker appearing and never # exercised the fast path that is supposed to clear an incomplete install. - if backend_ok and caps_ready is not False: + # + # `-h` gates the whole thing for the same reason: probe_managed_bin runs it FIRST + # and returns Stale "cli_unusable" without ever reaching the capability probe + # (managed.rs:465-478). Consulting cli_h_ok only in the repairable arm below let a + # CLI that cannot even print help be called HEALTHY as long as the backend booted, + # which skipped the re-run step for an install the app itself sends to repair. + if backend_ok and caps_ready and facts.get("cli_h_ok"): verdict = "HEALTHY" - elif caps_ready is False or not facts.get("cli_h_ok"): + elif not caps_ready or not facts.get("cli_h_ok"): verdict = "REPAIRABLE" else: verdict = "FALSE_READY" From 1a8ad4ae0662b6f00cef87a05d5baed8f86bf102 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Tue, 28 Jul 2026 23:49:18 +0000 Subject: [PATCH 13/25] Judge the probes on the desktop's deadline, and interrupt the host it uses Preflight gives each managed probe ten seconds and nothing more: managed.rs:337 wraps `unsloth -h` and managed.rs:390 wraps `studio desktop-capabilities --json` in a tokio timeout, kills the child on expiry, and returns Stale as "cli_unusable" or "desktop_capability_probe_failed". The probe allowed three minutes, so a venv torn badly enough that its CLI only answers after half a minute of retries was recorded HEALTHY here while the real app shows it as repairable. That skips the re-run assertion the leg exists to make, which is the same false-HEALTHY hole the studio_install_ok and -h gating already closed. Both calls now use the desktop's ten seconds, and the elapsed time is recorded so a leg that flips for timing reasons says so in the artefact. On Windows the installer child now runs where the desktop runs it. install.rs 325-339 spawns the bundled install.ps1 as powershell.exe with -NoLogo -NoProfile -NonInteractive -WindowStyle Hidden -ExecutionPolicy Bypass -File, so Windows PowerShell 5.1 is the only host a real desktop install ever uses. The interrupted run and the repair re-run both used pwsh 7, and every other Windows job in .github runs install.ps1 under pwsh too, so the installer's behaviour on 5.1 was covered by nothing: .NET Framework instead of .NET, OEM console encoding instead of UTF-8, and different native-command and OSArchitecture reporting are all real sources of divergence. A workflow whose point is to reproduce what the app does cannot run a different interpreter than the app does. The driver itself stays under pwsh; only the installer child and the repair invocation change. --- .github/scripts/interrupt-install.ps1 | 21 +++++++++++++++++--- .github/scripts/interrupted_install_probe.py | 19 ++++++++++++++++-- .github/workflows/interrupted-install-ci.yml | 7 ++++++- 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/.github/scripts/interrupt-install.ps1 b/.github/scripts/interrupt-install.ps1 index 92f6019c31..917cef32cd 100644 --- a/.github/scripts/interrupt-install.ps1 +++ b/.github/scripts/interrupt-install.ps1 @@ -41,10 +41,25 @@ foreach ($dir in @($env:UNSLOTH_STUDIO_HOME, (Join-Path $HOME '.unsloth\studio') } catch { Write-Host "[interrupt] could not seed install marker in ${dir}: $_" } } -# Run the installer in its own pwsh so stdout can be redirected to the log while we poll. -$argList = @('-NoProfile', '-NonInteractive', '-File', 'install.ps1') +# Run the installer in its own host so stdout can be redirected to the log while we +# poll. That host is WINDOWS PowerShell, not pwsh: the desktop app spawns the bundled +# install.ps1 as `powershell.exe -NoLogo -NoProfile -NonInteractive -WindowStyle Hidden +# -ExecutionPolicy Bypass -File` (install.rs:325-339), so 5.1 with those flags is the +# only host a real desktop install ever uses. Every other Windows job in .github runs +# install.ps1 under the runner's pwsh 7, which leaves the installer's behaviour on 5.1 +# -- .NET Framework rather than .NET, OEM/ANSI console encoding rather than UTF-8, +# different native-command and OSArchitecture reporting -- covered by nothing. An +# interruption test that runs a different interpreter than the app cannot claim to +# reproduce what the app does. The driver itself stays under pwsh; only the installer +# child and the repair re-run change. +$argList = @( + '-NoLogo', '-NoProfile', '-NonInteractive', + '-WindowStyle', 'Hidden', + '-ExecutionPolicy', 'Bypass', + '-File', 'install.ps1' +) if ($InstallArgs) { $argList += $InstallArgs.Split(' ') } -$proc = Start-Process -FilePath 'pwsh' -ArgumentList $argList ` +$proc = Start-Process -FilePath 'powershell.exe' -ArgumentList $argList ` -RedirectStandardOutput $LogPath -RedirectStandardError "$LogPath.err" ` -PassThru -NoNewWindow Write-Host "[interrupt] installer pid=$($proc.Id) marker='$Marker' deadline=${KillAtSeconds}s" diff --git a/.github/scripts/interrupted_install_probe.py b/.github/scripts/interrupted_install_probe.py index f05b466fa3..7699ade872 100644 --- a/.github/scripts/interrupted_install_probe.py +++ b/.github/scripts/interrupted_install_probe.py @@ -95,18 +95,33 @@ def main(argv: list[str]) -> int: print(f"[probe] {k:28} = {v}") # ── the two probes Tauri preflight actually runs ───────────────────────── - r = run([binp, "-h"], timeout = 180) + # Both under the DESKTOP's deadline, not a generous CI one. preflight wraps each + # call in a 10 second tokio timeout (managed.rs:337 for `-h`, managed.rs:390 for + # desktop-capabilities) and on expiry kills the child and reports Stale -- + # "cli_unusable" or "desktop_capability_probe_failed" (managed.rs:471, :521). A + # torn venv whose CLI still answers, but only after 30 seconds of import retries, + # is therefore an install the app sends to repair; waiting three minutes for it + # here would call the same install HEALTHY and skip the re-run assertion. run() + # reports a timeout as a non-zero rc, which lands in the same REPAIRABLE arm the + # desktop's Stale maps to. + PREFLIGHT_TIMEOUT = 10 + + t0 = time.time() + r = run([binp, "-h"], timeout = PREFLIGHT_TIMEOUT) (out / "cli-h.log").write_text(merged(r), encoding = "utf-8", errors = "replace") say("cli_h_ok", r[0] == 0) + say("cli_h_seconds", round(time.time() - t0, 2)) + t0 = time.time() caps_rc, caps_out, caps_err = run( - [binp, "studio", "desktop-capabilities", "--json"], timeout = 180 + [binp, "studio", "desktop-capabilities", "--json"], timeout = PREFLIGHT_TIMEOUT ) (out / "desktop-capabilities.json").write_text(caps_out, encoding = "utf-8", errors = "replace") (out / "desktop-capabilities.stderr.log").write_text( caps_err, encoding = "utf-8", errors = "replace" ) say("capabilities_ok", caps_rc == 0) + say("capabilities_seconds", round(time.time() - t0, 2)) # Parse EXACTLY as the desktop does: managed.rs:414 hands the whole stdout buffer # to serde_json, which rejects any leading or trailing non-JSON, and stderr was diff --git a/.github/workflows/interrupted-install-ci.yml b/.github/workflows/interrupted-install-ci.yml index 94de2249a9..f73094b89a 100644 --- a/.github/workflows/interrupted-install-ci.yml +++ b/.github/workflows/interrupted-install-ci.yml @@ -293,7 +293,12 @@ jobs: if: always() && steps.probe.outputs.verdict != 'HEALTHY' shell: pwsh run: | - pwsh -NoProfile -NonInteractive -File install.ps1 ${{ matrix.installArgs }} *>&1 | + # powershell.exe with install.rs:325-339's flags, matching the interrupted + # run: the desktop repairs by re-running the same bundled script in Windows + # PowerShell 5.1, so a repair that only works under pwsh 7 would pass here + # and still strand the user. + powershell.exe -NoLogo -NoProfile -NonInteractive -WindowStyle Hidden ` + -ExecutionPolicy Bypass -File install.ps1 ${{ matrix.installArgs }} *>&1 | Tee-Object -FilePath logs/repair.log $bin = Join-Path $env:USERPROFILE '.unsloth\studio\unsloth_studio\Scripts\unsloth.exe' if (-not (Test-Path $bin)) { From a9ab8aead0ab7a90e80147ca6698542bc2529223 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Wed, 29 Jul 2026 02:27:13 +0000 Subject: [PATCH 14/25] Tighten the interrupted-install comments --- .github/scripts/interrupt-install.ps1 | 58 +++---- .github/scripts/interrupt-install.sh | 54 +++---- .github/scripts/interrupted_install_probe.py | 131 +++++++--------- .github/workflows/interrupted-install-ci.yml | 155 +++++++++---------- 4 files changed, 175 insertions(+), 223 deletions(-) diff --git a/.github/scripts/interrupt-install.ps1 b/.github/scripts/interrupt-install.ps1 index 917cef32cd..fd763eaedb 100644 --- a/.github/scripts/interrupt-install.ps1 +++ b/.github/scripts/interrupt-install.ps1 @@ -4,11 +4,9 @@ # Windows counterpart of interrupt-install.sh: run install.ps1 and kill it partway # through, reproducing a user quitting the desktop app mid-install. # -# Windows has no process groups, which is why the app carries windows_job.rs -- a Job -# Object is what makes "kill the installer and everything it spawned" work there. This -# script kills the whole process TREE for the same reason: killing only the leader -# leaves uv/python children to finish the dependency pass, and the interruption would -# prove nothing. +# Windows has no process groups, which is why the app carries windows_job.rs. This script +# kills the whole process TREE for the same reason: killing only the leader leaves +# uv/python children to finish the dependency pass, proving nothing. # # Usage: # pwsh -File .github/scripts/interrupt-install.ps1 -Marker 'studio deps' ` @@ -26,13 +24,11 @@ $ErrorActionPreference = 'Continue' New-Item -ItemType Directory -Force -Path (Split-Path -Parent $LogPath) | Out-Null Set-Content -Path $LogPath -Value '' -Encoding utf8 -# Stand in for the desktop app, which creates this before spawning the installer and -# clears it only on a terminal outcome (install.rs). We kill install.ps1 directly -# rather than driving the real app, so without this the marker #7490 relies on is -# absent for a reason that has nothing to do with #7490 -- which is exactly what the -# Windows legs reported. Both locations because the Rust side hardcodes -# ~/.unsloth/studio while CI overrides UNSLOTH_STUDIO_HOME. Never cleared: being -# killed is the whole point. +# Stand in for the desktop app, which writes this before spawning the installer +# (install.rs). We kill install.ps1 directly, so without it the marker #7490 relies on is +# absent for a reason unrelated to #7490 -- exactly what the Windows legs reported. Both +# locations because the Rust side hardcodes ~/.unsloth/studio while CI overrides +# UNSLOTH_STUDIO_HOME. Never cleared: being killed is the whole point. foreach ($dir in @($env:UNSLOTH_STUDIO_HOME, (Join-Path $HOME '.unsloth\studio'))) { if ([string]::IsNullOrWhiteSpace($dir)) { continue } try { @@ -41,17 +37,13 @@ foreach ($dir in @($env:UNSLOTH_STUDIO_HOME, (Join-Path $HOME '.unsloth\studio') } catch { Write-Host "[interrupt] could not seed install marker in ${dir}: $_" } } -# Run the installer in its own host so stdout can be redirected to the log while we -# poll. That host is WINDOWS PowerShell, not pwsh: the desktop app spawns the bundled -# install.ps1 as `powershell.exe -NoLogo -NoProfile -NonInteractive -WindowStyle Hidden -# -ExecutionPolicy Bypass -File` (install.rs:325-339), so 5.1 with those flags is the -# only host a real desktop install ever uses. Every other Windows job in .github runs -# install.ps1 under the runner's pwsh 7, which leaves the installer's behaviour on 5.1 -# -- .NET Framework rather than .NET, OEM/ANSI console encoding rather than UTF-8, -# different native-command and OSArchitecture reporting -- covered by nothing. An -# interruption test that runs a different interpreter than the app cannot claim to -# reproduce what the app does. The driver itself stays under pwsh; only the installer -# child and the repair re-run change. +# Its own host, so stdout can be redirected to the log while we poll. That host is +# WINDOWS PowerShell 5.1, not pwsh, with install.rs:325-339's exact flags: that is the +# only host a real desktop install ever uses, and every other Windows job in .github runs +# install.ps1 under pwsh 7, leaving 5.1 behaviour (.NET Framework, OEM/ANSI console +# encoding, different native-command and OSArchitecture reporting) covered by nothing. +# The driver itself stays under pwsh; only the installer child and the repair re-run +# change. $argList = @( '-NoLogo', '-NoProfile', '-NonInteractive', '-WindowStyle', 'Hidden', @@ -65,8 +57,8 @@ $proc = Start-Process -FilePath 'powershell.exe' -ArgumentList $argList ` Write-Host "[interrupt] installer pid=$($proc.Id) marker='$Marker' deadline=${KillAtSeconds}s" function Stop-Tree([int]$RootId) { - # Depth-first: children before parents, so a parent cannot respawn a child we already - # killed. CIM gives us the parent link Windows does not expose via process groups. + # Depth-first, so a parent cannot respawn a child we already killed. CIM gives the + # parent link Windows does not expose via process groups. $kids = @(Get-CimInstance Win32_Process -Filter "ParentProcessId=$RootId" -ErrorAction SilentlyContinue) foreach ($k in $kids) { Stop-Tree ([int]$k.ProcessId) } try { Stop-Process -Id $RootId -Force -ErrorAction Stop; Write-Host "[interrupt] killed pid=$RootId" } @@ -96,15 +88,13 @@ if (-not $killed -and -not $proc.HasExited) { if (-not $reason) { $reason = 'dea if ($killed) { Write-Host "[interrupt] killing process tree of $($proc.Id) ($reason)" Stop-Tree $proc.Id - # Any straggler uv/python that reparented away from the installer. The old sweep - # matched nothing: UNSLOTH_STUDIO_HOME arrives as `D:\a\r\r/.studio-home` - # (github.workspace joined with a forward slash) while Process.Path is all - # backslashes, so the literal -like missed even the venv's own python, and uv is - # never under the studio home anyway (install.ps1 takes it from winget or - # astral.sh). Normalise the separators, and take uv by name since the runner is - # ephemeral and runs no other uv. - # Under --tauri there is no UNSLOTH_STUDIO_HOME, so fall back to the root - # install.ps1 uses then, or the sweep would only ever see uv. + # Any straggler uv/python that reparented away from the installer. The old sweep matched + # nothing: UNSLOTH_STUDIO_HOME arrives as `D:\a\r\r/.studio-home` (github.workspace + # joined with a forward slash) while Process.Path is all backslashes, so the literal + # -like missed even the venv's own python. Hence the separator normalisation, and uv by + # name (it lives outside the studio home, and the ephemeral runner has no other uv). + # Under --tauri there is no UNSLOTH_STUDIO_HOME, so fall back to the root install.ps1 + # uses then, or the sweep would only ever see uv. $studioRoot = if ([string]::IsNullOrWhiteSpace($env:UNSLOTH_STUDIO_HOME)) { Join-Path $HOME '.unsloth\studio' } else { $env:UNSLOTH_STUDIO_HOME } $homeNorm = if ([string]::IsNullOrWhiteSpace($studioRoot)) { $null } diff --git a/.github/scripts/interrupt-install.sh b/.github/scripts/interrupt-install.sh index c6cfe9b0f8..c24382c56b 100755 --- a/.github/scripts/interrupt-install.sh +++ b/.github/scripts/interrupt-install.sh @@ -2,13 +2,10 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. # -# Run install.sh and SIGTERM it partway through, reproducing what the desktop app does -# when the user quits mid-install: main.rs cleanup_child_processes() -> -# install::stop_install() -> kill the installer PROCESS GROUP (install.rs:798-807). -# -# Killing only the leader would leave `uv`/`python` children running and finishing the -# dep pass, so the interruption has to target the group -- otherwise the test quietly -# proves nothing. +# Run install.sh and SIGTERM it partway through, reproducing a user quitting the desktop +# app mid-install: main.rs cleanup_child_processes() -> install::stop_install() -> kill +# the installer PROCESS GROUP (install.rs:798-807). It must be the group: killing only +# the leader leaves `uv`/`python` children to finish the dep pass, proving nothing. # # Usage: bash .github/scripts/interrupt-install.sh "" "" [-- install args] # regex to wait for in the install log before killing, e.g. "studio deps" @@ -28,12 +25,11 @@ KILL_GRACE="${KILL_GRACE:-10}" mkdir -p "$(dirname "$LOG")" : > "$LOG" -# Stand in for the desktop app, which creates this before spawning the installer and -# clears it only on a terminal outcome (install.rs). We kill the installer directly -# rather than driving the real app, so without this the marker #7490 relies on is -# absent for a reason that has nothing to do with #7490. Written to both locations -# because the Rust side hardcodes ~/.unsloth/studio while CI overrides -# UNSLOTH_STUDIO_HOME. Deliberately never cleared: being killed is the whole point. +# Stand in for the desktop app, which writes this before spawning the installer +# (install.rs). We kill the installer directly, so without it the marker #7490 relies on +# is absent for a reason unrelated to #7490. Both locations because the Rust side +# hardcodes ~/.unsloth/studio while CI overrides UNSLOTH_STUDIO_HOME. Never cleared: +# being killed is the whole point. for _marker_dir in "${UNSLOTH_STUDIO_HOME:-}" "$HOME/.unsloth/studio"; do [ -n "$_marker_dir" ] || continue mkdir -p "$_marker_dir" 2>/dev/null || continue @@ -41,7 +37,7 @@ for _marker_dir in "${UNSLOTH_STUDIO_HOME:-}" "$HOME/.unsloth/studio"; do done # Job control puts the child in its own process group, so $! is the pgid leader and -# `kill -- -$!` reaches every descendant -- matching the Rust side. +# `kill -- -$!` reaches every descendant, matching the Rust side. set -m bash install.sh "$@" > "$LOG" 2>&1 & PID=$! @@ -56,13 +52,12 @@ for i in $(seq 1 "$KILL_AT_SECONDS"); do break fi if [ -n "$MARKER" ] && grep -qE "$MARKER" "$LOG" 2>/dev/null; then - # Let it get a beat into the step, so the kill lands mid-work rather than on the - # boundary where the step has not started touching the venv yet. + # A beat into the step, so the kill lands mid-work rather than on the boundary + # before the step has touched the venv. sleep "${KILL_AFTER_MARKER_SECONDS:-3}" - # ...but a late step whose work is already cached can FINISH inside that beat. - # Recording marker-hit before the sleep handed the landing assertion a COMPLETED - # install: the signal reached no process, the probe read HEALTHY, and the leg - # passed green having interrupted nothing. Set the reason after, not before. + # ...but a cached step can FINISH inside that beat. Recording marker-hit before the + # sleep handed the landing assertion a COMPLETED install: the signal reached no + # process and the leg passed green having interrupted nothing. if ! kill -0 "$PID" 2>/dev/null; then reason="exited-during-marker-delay" break @@ -86,11 +81,10 @@ if [ "$killed" = "true" ]; then kill -0 "$PID" 2>/dev/null || break sleep 1 done - # Unconditional, and to the GROUP. The leader can exit on SIGTERM while a uv or - # python descendant ignores it or is mid-shutdown; `kill -0 "$PID"` then reported - # the leader gone, this escalation was skipped, and `wait` reaped only the leader, - # leaving that descendant free to finish the dependency pass while the probe ran. - # Signalling an already-empty group is a no-op. + # Unconditional, and to the GROUP. The leader can exit on SIGTERM while a uv or python + # descendant does not; gating on `kill -0 "$PID"` skipped this escalation and left that + # descendant free to finish the dependency pass while the probe ran. Signalling an + # already-empty group is a no-op. echo "[interrupt] SIGKILL to process group -$PID" kill -KILL -- -"$PID" 2>/dev/null || kill -KILL "$PID" 2>/dev/null || true fi @@ -98,9 +92,9 @@ fi wait "$PID" 2>/dev/null rc=$? -# Only after the reap: an unreaped leader is still a member of its own group, so -# polling the group before `wait` would report it alive forever. Do not let the -# probe start while an installer process is still running. +# Only after the reap: an unreaped leader is still a member of its own group, so polling +# the group before `wait` would report it alive forever. The probe must not start while +# an installer process is still running. if [ "$killed" = "true" ]; then for _ in $(seq 1 "$KILL_GRACE"); do kill -0 -- -"$PID" 2>/dev/null || break @@ -115,8 +109,8 @@ echo "[interrupt] installer exit=$rc reason=$reason killed=$killed" echo "[interrupt] last log lines:" tail -15 "$LOG" || true -# Report how far it got, so a leg that never reached the target step is visible as such -# rather than passing for the wrong reason. +# Report how far it got, so a leg that never reached the target step is visible rather +# than passing for the wrong reason. if [ -n "$MARKER" ] && ! grep -qE "$MARKER" "$LOG" 2>/dev/null; then echo "::warning::marker '$MARKER' never appeared -- this leg killed at the deadline, not at the intended step" fi diff --git a/.github/scripts/interrupted_install_probe.py b/.github/scripts/interrupted_install_probe.py index 7699ade872..c85d0397dc 100644 --- a/.github/scripts/interrupted_install_probe.py +++ b/.github/scripts/interrupted_install_probe.py @@ -95,15 +95,11 @@ def main(argv: list[str]) -> int: print(f"[probe] {k:28} = {v}") # ── the two probes Tauri preflight actually runs ───────────────────────── - # Both under the DESKTOP's deadline, not a generous CI one. preflight wraps each - # call in a 10 second tokio timeout (managed.rs:337 for `-h`, managed.rs:390 for - # desktop-capabilities) and on expiry kills the child and reports Stale -- - # "cli_unusable" or "desktop_capability_probe_failed" (managed.rs:471, :521). A - # torn venv whose CLI still answers, but only after 30 seconds of import retries, - # is therefore an install the app sends to repair; waiting three minutes for it - # here would call the same install HEALTHY and skip the re-run assertion. run() - # reports a timeout as a non-zero rc, which lands in the same REPAIRABLE arm the - # desktop's Stale maps to. + # The DESKTOP's deadline, not a generous CI one: preflight times each call out after + # 10s (managed.rs:337 for `-h`, :390 for desktop-capabilities) and reports Stale + # (managed.rs:471, :521). A longer timeout here would call a slow torn venv HEALTHY + # and skip the re-run assertion. run() reports a timeout as a non-zero rc, landing in + # the same REPAIRABLE arm as Stale. PREFLIGHT_TIMEOUT = 10 t0 = time.time() @@ -123,18 +119,13 @@ def main(argv: list[str]) -> int: say("capabilities_ok", caps_rc == 0) say("capabilities_seconds", round(time.time() - t0, 2)) - # Parse EXACTLY as the desktop does: managed.rs:414 hands the whole stdout buffer - # to serde_json, which rejects any leading or trailing non-JSON, and stderr was - # already discarded at managed.rs:358. Folding stderr in and then scanning to the - # first brace made one warning line on stderr enough for json.loads to raise on - # the trailing text, leaving studio_install_ok "absent" and reporting FALSE_READY - # over an install the real app parses, sees as incomplete, and offers to repair. - # - # studio_install_ok is added by the install-manifest work, so it is absent on older - # trees; that is recorded separately from present-and-false only to make the - # artefact readable, because the desktop treats both as Stale. A payload that does - # not parse at all is a third case with the same outcome: the desktop gets None - # back and reports Stale ("desktop_capability_probe_failed", managed.rs:521). + # Parse EXACTLY as the desktop does: managed.rs:414 hands the whole stdout buffer to + # serde_json, which rejects leading or trailing non-JSON, and stderr was already + # discarded at managed.rs:358. Folding stderr in made one warning line enough to fail + # the parse and report FALSE_READY over an install the real app offers to repair. + # "absent" (studio_install_ok predates the install-manifest work) and "unparseable" + # are split apart only for a readable artefact: the desktop reports Stale for both + # ("desktop_capability_probe_failed", managed.rs:521). install_ok: object = "absent" try: parsed = json.loads(caps_out) @@ -147,26 +138,21 @@ def main(argv: list[str]) -> int: install_ok = "unparseable" say("capabilities.studio_install_ok", install_ok) - # The desktop's own conclusion: Ready only when the probe exits 0, the payload - # parses, AND studio_install_ok is true. The predicate is `!= Some(true)` - # (managed.rs:445), so an ABSENT field is Stale exactly like a false one -- a CLI - # too old to answer is already rejected one check earlier on - # desktop_manageability_version. Leaving "absent" undecided judged those installs - # on the backend alone, so a payload that stopped carrying the field reported - # HEALTHY on every booting leg and skipped the repair assertion this workflow - # exists to make, while the real app showed Stale and offered repair. That is the - # regression `unsloth_cli/commands/studio.py` is in this workflow's path filter to - # catch, so it must never be the thing that silences it. + # The desktop's own conclusion: Ready only on rc 0 + a parsed payload + a true + # studio_install_ok. The predicate is `!= Some(true)` (managed.rs:445), so an ABSENT + # field is Stale exactly like a false one; a CLI too old to answer is already + # rejected one check earlier on desktop_manageability_version. Leaving "absent" + # undecided reported HEALTHY on every booting leg and skipped the repair assertion + # this workflow exists to make -- the regression `unsloth_cli/commands/studio.py` + # sits in the path filter to catch, so it must never be what silences it. caps_ready = caps_rc == 0 and install_ok is True say("desktop_would_call_install_ok", caps_ready) # ── the deeper probes the fix PRs add ──────────────────────────────────── - # RECORDED, but NOT repair evidence: preflight/managed.rs runs only `-h` and - # `studio desktop-capabilities --json` (managed.rs:357) and reads - # studio_install_ok from that payload (managed.rs:445). It never invokes these - # two commands, so counting them would let a leg pass while the real app still - # reports ManagedReady over a torn install -- the exact false negative this - # workflow exists to catch. + # RECORDED, but NOT repair evidence: preflight runs only `-h` and + # `studio desktop-capabilities --json` (managed.rs:357, :445) and never these two, so + # counting them would let a leg pass while the real app still reports ManagedReady + # over a torn install -- the false negative this workflow exists to catch. for label, args in ( ("verify_install", ["studio", "verify-install"]), ("desktop_runtime_check", ["studio", "desktop-runtime-check"]), @@ -178,35 +164,32 @@ def main(argv: list[str]) -> int: (out / f"{label}.log").write_text(merged(r), encoding = "utf-8", errors = "replace") say(label, "ok" if r[0] == 0 else "failed") - # The in-progress marker #7490 writes before spawning the installer. RECORDED ONLY, - # never used as repair evidence: both interrupt drivers seed it before every install - # and deliberately never clear it, so it is true on every leg by construction. Using - # it in the verdict below would make REPAIRABLE unconditional and FALSE_READY -- the - # single outcome this workflow exists to catch -- unreachable. + # The in-progress marker #7490 writes before spawning the installer. RECORDED ONLY: + # both interrupt drivers seed it and deliberately never clear it, so it is true on + # every leg by construction, and using it in the verdict below would make REPAIRABLE + # unconditional and FALSE_READY -- the one outcome this catches -- unreachable. home = Path(os.environ.get("UNSLOTH_STUDIO_HOME") or (Path.home() / ".unsloth" / "studio")) say("install_in_progress_marker", (home / ".desktop-install-in-progress").exists()) # ── ground truth: does the backend actually boot? ──────────────────────── - # Own the whole process tree: the CLI spawns uvicorn/python children, and - # terminating only the parent leaves them holding the port, so the next leg's - # probe would hang. Same reason the interrupt driver kills the group. + # Own the whole process tree: the CLI spawns uvicorn/python children that would keep + # holding the port and hang the next leg's probe. Same reason the driver kills the + # group. popen_kw: dict = {} if os.name == "posix": popen_kw["start_new_session"] = True else: popen_kw["creationflags"] = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) - # Straight to the artefact file, never a PIPE: nothing reads that pipe until after - # the polling loop, so a backend whose imports emit more than the OS pipe buffer - # (64 KiB on Linux and macOS, a single page by default on Windows) blocks on write - # BEFORE it binds the port. backend_ok, which this whole verdict pivots on, would - # then be false for a perfectly good install. + # Straight to the artefact file, never a PIPE: nothing drains a pipe until after the + # polling loop, so a backend whose imports outrun the OS buffer (64 KiB on Linux and + # macOS, one page on Windows) blocks on write BEFORE binding the port, and backend_ok + # -- what this verdict pivots on -- would be false for a perfectly good install. blog_path = out / "backend.log" blog_fh = blog_path.open("w", encoding = "utf-8", errors = "replace") - # An interrupted install can leave the console script in place while its venv - # interpreter is gone: the earlier probes then report failure through run()'s - # OSError catch, but an unguarded spawn here raises instead, so no verdict.json - # is written and both workflows die on the json.load rather than reporting. An - # unlaunchable CLI is a broken backend that `-h` already flags -> REPAIRABLE. + # An interrupted install can leave the console script in place with its venv + # interpreter gone. run() catches that as OSError, but an unguarded spawn here would + # raise, so no verdict.json is written and both workflows die on the json.load rather + # than reporting. An unlaunchable CLI is a broken backend `-h` already flags. proc = None try: proc = subprocess.Popen( @@ -241,9 +224,9 @@ def main(argv: list[str]) -> int: if os.name == "posix": import signal - # start_new_session made this child its own group leader, so pgid == pid. - # Read it BEFORE the reap: once the leader is waited on, os.getpgid() - # raises and the escalation would target nothing. + # start_new_session made this child its own group leader. Read the pgid + # BEFORE the reap: once the leader is waited on, os.getpgid() raises and the + # escalation would target nothing. try: pgid = os.getpgid(proc.pid) except OSError: @@ -259,9 +242,8 @@ def main(argv: list[str]) -> int: except subprocess.TimeoutExpired: continue # Unconditional, and to the GROUP -- the same escalation - # interrupt-install.sh:94 makes, for the same reason. The leader exits - # promptly on SIGTERM while a uvicorn worker does not, so returning as - # soon as proc.wait() succeeded skipped the SIGKILL entirely and left + # interrupt-install.sh:94 makes. The leader exits promptly on SIGTERM while a + # uvicorn worker does not, so returning as soon as proc.wait() succeeded left # that worker holding the port and the venv open while the repair step # reinstalled underneath it. Signalling an empty group is a no-op. try: @@ -270,10 +252,10 @@ def main(argv: list[str]) -> int: pass else: # On win32 the CLI re-spawns the server as a CHILD and waits on it - # (unsloth_cli/commands/studio.py:1543); CREATE_NEW_PROCESS_GROUP does not + # (unsloth_cli/commands/studio.py:1543), and CREATE_NEW_PROCESS_GROUP does not # make terminate() reach descendants, so killing the wrapper alone leaves a - # server holding the venv open and the repair step reinstalls into files - # Windows has locked. taskkill /T takes the tree. + # server holding the venv open and the repair reinstalls into locked files. + # taskkill /T takes the tree. run(["taskkill", "/F", "/T", "/PID", str(proc.pid)], timeout = 30) try: proc.wait(timeout = 10) @@ -297,19 +279,16 @@ def main(argv: list[str]) -> int: say("backend_error", missing) # ── verdict ────────────────────────────────────────────────────────────── - # A booting backend is not enough to call the install finished. The manifest is - # written LAST (install_python_stack.py:3255), so a kill after "studio deps" but - # before it -- the data-designer leg -- leaves a venv whose backend boots while - # desktop-capabilities still says studio_install_ok=false, and preflight reports - # Stale (managed.rs:445) rather than Ready. Calling that HEALTHY skipped the - # re-run step, so the leg asserted nothing beyond a marker appearing and never - # exercised the fast path that is supposed to clear an incomplete install. + # A booting backend is not enough. The manifest is written LAST + # (install_python_stack.py:3255), so the data-designer leg boots while + # desktop-capabilities still says studio_install_ok=false and preflight reports Stale + # (managed.rs:445). Calling that HEALTHY skipped the re-run step, leaving the leg + # asserting nothing beyond a marker appearing. # - # `-h` gates the whole thing for the same reason: probe_managed_bin runs it FIRST - # and returns Stale "cli_unusable" without ever reaching the capability probe - # (managed.rs:465-478). Consulting cli_h_ok only in the repairable arm below let a - # CLI that cannot even print help be called HEALTHY as long as the backend booted, - # which skipped the re-run step for an install the app itself sends to repair. + # `-h` gates it for the same reason: probe_managed_bin runs it FIRST and returns + # Stale "cli_unusable" without reaching the capability probe (managed.rs:465-478), so + # consulting cli_h_ok only in the repairable arm called a CLI that cannot print help + # HEALTHY whenever the backend booted. if backend_ok and caps_ready and facts.get("cli_h_ok"): verdict = "HEALTHY" elif not caps_ready or not facts.get("cli_h_ok"): diff --git a/.github/workflows/interrupted-install-ci.yml b/.github/workflows/interrupted-install-ci.yml index f73094b89a..6c3c3b58f5 100644 --- a/.github/workflows/interrupted-install-ci.yml +++ b/.github/workflows/interrupted-install-ci.yml @@ -3,19 +3,17 @@ # Proves an INTERRUPTED install can never masquerade as a healthy one. # -# Reported failure: a user quits the desktop app while it is installing. The app kills -# the installer process group (main.rs cleanup_child_processes -> install.rs:798-807), -# which lands mid "studio deps" -- the step that installs -# studio/backend/requirements/studio.txt, where structlog is declared. On relaunch, -# preflight probes `unsloth -h` and `studio desktop-capabilities --json`; both succeed -# because the CLI's own deps (typer/click/rich) are core, so the app reports -# ManagedReady with can_auto_repair=false. The backend then dies on -# `import structlog` and the user is permanently stuck on "Server stopped -# unexpectedly". +# Reported failure: quitting the app mid-install kills the installer process group +# (main.rs cleanup_child_processes -> install.rs:798-807), landing mid "studio deps" -- +# the step installing studio/backend/requirements/studio.txt, where structlog is +# declared. On relaunch preflight probes `unsloth -h` and `studio desktop-capabilities +# --json`; both succeed because the CLI's own deps (typer/click/rich) are core, so the +# app reports ManagedReady with can_auto_repair=false while the backend dies on +# `import structlog` and the user is stuck on "Server stopped unexpectedly". # -# Nothing in CI covered this: no job has ever interrupted an install. This workflow -# kills the installer at each interesting phase and asserts the result is either -# genuinely healthy or explicitly repairable -- never silently ready. +# No CI job had ever interrupted an install. This one kills the installer at each phase +# and asserts the result is genuinely healthy or explicitly repairable, never silently +# ready. name: Interrupted install recovery @@ -32,27 +30,23 @@ on: - 'studio/src-tauri/src/preflight/**' - 'unsloth_cli/commands/studio.py' # Every leg installs the checkout with `--local`, so this file decides the - # `unsloth` console script and the core dependencies the probe leans on: - # `-h` and `desktop-capabilities` only survive a torn install because - # typer/click/rich are declared here. Moving one of those to an extra - # changes what every interrupted venv looks like, and no other install - # workflow interrupts the installer, so such a PR would otherwise merge - # without a single leg running. + # `unsloth` console script and the core deps the probe leans on: `-h` and + # `desktop-capabilities` only survive a torn install because typer/click/rich + # are declared here. Moving one to an extra changes what every interrupted + # venv looks like, and no other install workflow interrupts the installer. - 'pyproject.toml' # studio_install_ok and verify-install, the two decisions the probe asserts - # on, are implemented here rather than in commands/studio.py, so a change - # that made install_state() accept a missing manifest would otherwise merge - # without a single leg running. + # on, live here rather than in commands/studio.py, so a change making + # install_state() accept a missing manifest would otherwise merge unrun. - 'unsloth_cli/_studio_deps.py' - 'studio/install_manifest.py' - # The requirement files are the phases. studio.txt is where structlog is - # declared, the package whose absence IS the reported false-ready bug, and - # the single-env files drive the later steps, so moving a package between - # them changes what every interrupted state looks like. + # The requirement files are the phases: studio.txt declares structlog, whose + # absence IS the reported false-ready bug, so moving a package between them + # changes what every interrupted state looks like. - 'studio/backend/requirements/**' - # `*` never matches `/`, and it is a literal `-install` that follows, so - # `interrupt*-install*` matches interrupt-install.sh / .ps1 but NOT the - # underscored probe. List the probe explicitly rather than rely on a glob. + # `*` never matches `/` and a literal `-install` follows, so + # `interrupt*-install*` would match the .sh / .ps1 but NOT the underscored + # probe. List all three explicitly rather than rely on a glob. - '.github/scripts/interrupt-install.sh' - '.github/scripts/interrupt-install.ps1' - '.github/scripts/interrupted_install_probe.py' @@ -89,10 +83,9 @@ jobs: - {os: macos-14, label: setup, marker: '\[TAURI:STEP\] Running Unsloth setup', experimental: false} # Other dependency-pass steps around the named one. - {os: macos-14, label: pip-bootstrap, marker: 'pip bootstrap', experimental: false} - # No base-packages cell: install.sh --local sets skip_base, so - # install_python_stack returns before any "base packages" label is - # printed and the kill can never land. It ran to completion instead, - # proving nothing. + # No base-packages cell: --local sets skip_base, so install_python_stack + # returns before any "base packages" label prints and the kill can never + # land. That leg ran to completion instead, proving nothing. - {os: macos-14, label: unsloth-extras, marker: 'unsloth extras', experimental: true} - {os: macos-14, label: data-designer, marker: 'data designer deps', experimental: true} # Linux: same teardown path, different package manager and process semantics. @@ -115,14 +108,13 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} KILL_AT_SECONDS: '1500' run: | - # --local is load-bearing, not a convenience. Without it install.sh:3996 - # resolves `unsloth>=2026.7.5` from PyPI, so the venv gets the PUBLISHED CLI - # and none of the branch's unsloth_cli changes are present. Every probe of - # `studio verify-install` / `studio desktop-runtime-check` then reports - # "absent" no matter what the branch does, which makes the whole lane - # incapable of observing the fix it exists to test. --local overlays the - # checkout editable (install.sh:3990) before `studio setup` runs the dep - # pass, so a kill at "studio deps" leaves the branch's CLI installed. + # --local is load-bearing. Without it install.sh:3996 resolves + # `unsloth>=2026.7.5` from PyPI, so the venv gets the PUBLISHED CLI, every + # probe of `studio verify-install` / `desktop-runtime-check` reports "absent" + # whatever the branch does, and the lane cannot observe the fix it tests. + # --local overlays the checkout editable (install.sh:3990) before `studio + # setup` runs the dep pass, so a kill at "studio deps" leaves the branch's + # CLI installed. bash .github/scripts/interrupt-install.sh \ '${{ matrix.marker }}' logs/install.log -- --tauri --local @@ -146,8 +138,8 @@ jobs: BIN="$HOME/.unsloth/studio/unsloth_studio/bin/unsloth" [ -x "$BIN" ] || BIN="$HOME/.unsloth/studio/bin/unsloth" if [ ! -x "$BIN" ]; then - # No CLI at all is a SAFE outcome: preflight reports NotInstalled and the - # app offers a normal install. Nothing to assert beyond that. + # No CLI at all is SAFE: preflight reports NotInstalled and the app offers + # a normal install. echo "verdict=NO_CLI" >> "$GITHUB_OUTPUT" echo "[probe] no unsloth CLI installed -> preflight reports NotInstalled (safe)" exit 0 @@ -159,16 +151,16 @@ jobs: exit "$rc" - name: A re-run must repair, not short-circuit - # NO_CLI included. A kill at venv or torch lands before "Installing Unsloth" - # (install.sh:2125 / :3667 / :3961), so those legs always take NO_CLI, and - # skipping the re-run left three non-experimental legs asserting nothing but - # that a marker appeared. The bug's second half is that `install.sh` sees a - # "current" version and no-ops over a broken venv. + # NO_CLI included: a kill at venv or torch lands before "Installing Unsloth" + # (install.sh:2125 / :3667 / :3961), so those legs always take NO_CLI and + # skipping the re-run left three non-experimental legs asserting only that a + # marker appeared. The bug's second half is `install.sh` seeing a "current" + # version and no-opping over a broken venv. # - # HEALTHY means the backend boots AND desktop-capabilities reports the - # install complete, so a leg killed after "studio deps" but before the - # manifest is written last (install_python_stack.py:3255) -- data-designer -- - # arrives here rather than skipping the one assertion that matters for it. + # HEALTHY needs the backend booting AND the install reported complete, so the + # data-designer leg (killed after "studio deps" but before the manifest is + # written last, install_python_stack.py:3255) arrives here rather than skipping + # the one assertion that matters for it. if: always() && steps.probe.outputs.verdict != 'HEALTHY' run: | set -o pipefail @@ -177,8 +169,8 @@ jobs: echo "repair exit: $rc" BIN="$HOME/.unsloth/studio/unsloth_studio/bin/unsloth" [ -x "$BIN" ] || BIN="$HOME/.unsloth/studio/bin/unsloth" - # The probe exits without writing verdict.json when the bin is missing, so - # check here or the json.load below crashes instead of reporting. + # The probe writes no verdict.json when the bin is missing, so check here or + # the json.load below crashes instead of reporting. if [ ! -x "$BIN" ]; then echo "::error::after a full re-run there is still no unsloth CLI at $BIN" tail -30 logs/repair.log || true @@ -186,9 +178,9 @@ jobs: fi python3 .github/scripts/interrupted_install_probe.py "$BIN" --out probe-after || true v="$(python3 -c "import json;print(json.load(open('probe-after/verdict.json'))['verdict'])")" - # A booting backend IS the repair, whatever the log narrated. Judging by - # log text instead failed a leg whose venv was fine: the only match was - # the frontend build printing "up to date". + # A booting backend IS the repair, whatever the log narrated. Judging by log + # text failed a leg whose venv was fine: the only match was the frontend + # build printing "up to date". if [ "$v" = "HEALTHY" ]; then echo "re-run repaired the install (verdict=HEALTHY)" exit 0 @@ -214,23 +206,22 @@ jobs: # ── Windows: no process groups, so the kill path differs ────────────────── interrupt-windows: name: windows kill@${{ matrix.label }} - # No UNSLOTH_STUDIO_HOME here. The desktop app always invokes the installer as - # `--tauri [--local]` and scrubs the variable first (install.rs:202 and :356), and - # install.ps1:189-215 rejects a custom root under --tauri, so a workspace-scoped - # root forced these legs down the shell-install path instead: UNSLOTH_TAURI_MODE=0, - # frontend build enabled, different root resolution, no bundled-file overlay. Worse, - # "Installing PyTorch" is only ever printed by Write-TauriLog (install.ps1:2440), so - # without --tauri the torch leg's marker could not appear at all. The runner is - # ephemeral, so the default root is safe to install into. + # No UNSLOTH_STUDIO_HOME here. The app scrubs it before invoking the installer as + # `--tauri [--local]` (install.rs:202, :356) and install.ps1:189-215 rejects a custom + # root under --tauri, so a workspace-scoped root forced these legs down the + # shell-install path: UNSLOTH_TAURI_MODE=0, frontend build on, different root + # resolution, no bundled-file overlay. Worse, "Installing PyTorch" is only printed by + # Write-TauriLog (install.ps1:2440), so the torch leg's marker could never appear. + # The runner is ephemeral, so the default root is safe to install into. runs-on: windows-latest timeout-minutes: 60 strategy: fail-fast: false matrix: include: - # install.ps1 parses `--no-torch` (install.ps1:121); `-SkipTorch` matches no - # case in that switch and is silently dropped. The torch leg must NOT skip - # torch, or its marker never appears. + # install.ps1:121 parses `--no-torch`; `-SkipTorch` matches no case there and + # is silently dropped. The torch leg must NOT skip torch, or its marker never + # appears. - {label: studio-deps, marker: 'studio deps', installArgs: '--tauri --no-torch --local'} - {label: torch, marker: 'Installing PyTorch', installArgs: '--tauri --local'} @@ -274,11 +265,10 @@ jobs: Write-Host '[probe] no unsloth CLI -> preflight reports NotInstalled (safe)' exit 0 } - # The SAME probe the other platforms run. This step used to be a bespoke - # inline version that only checked `-h` and `desktop-capabilities`, so it - # could not observe studio_install_ok / verify-install / - # desktop-runtime-check -- it would have failed the very PRs that add them, - # no matter how well they worked. + # The SAME probe the other platforms run. The bespoke inline version this + # replaced only checked `-h` and `desktop-capabilities`, so it could not + # observe studio_install_ok / verify-install / desktop-runtime-check and + # would have failed the very PRs that add them. python .github/scripts/interrupted_install_probe.py $bin --out probe $rc = $LASTEXITCODE $v = (Get-Content probe/verdict.json -Raw | ConvertFrom-Json).verdict @@ -286,17 +276,16 @@ jobs: exit $rc - name: A re-run must repair, not short-circuit - # Same assertion the POSIX legs make, NO_CLI included: a leg that left no CLI - # otherwise asserts nothing, and without this a Windows leg proves only that - # the break was DETECTED, never that install.ps1's version fast path does not - # short-circuit over it, which is the half of the bug that strands the user. + # Same assertion the POSIX legs make, NO_CLI included: without it a Windows leg + # proves only that the break was DETECTED, never that install.ps1's version fast + # path does not short-circuit over it -- the half of the bug that strands the + # user. if: always() && steps.probe.outputs.verdict != 'HEALTHY' shell: pwsh run: | - # powershell.exe with install.rs:325-339's flags, matching the interrupted - # run: the desktop repairs by re-running the same bundled script in Windows - # PowerShell 5.1, so a repair that only works under pwsh 7 would pass here - # and still strand the user. + # powershell.exe with install.rs:325-339's flags, matching the interrupted run: + # the desktop repairs under Windows PowerShell 5.1, so a repair that only works + # under pwsh 7 would pass here and still strand the user. powershell.exe -NoLogo -NoProfile -NonInteractive -WindowStyle Hidden ` -ExecutionPolicy Bypass -File install.ps1 ${{ matrix.installArgs }} *>&1 | Tee-Object -FilePath logs/repair.log @@ -308,9 +297,9 @@ jobs: } python .github/scripts/interrupted_install_probe.py $bin --out probe-after $v = (Get-Content probe-after/verdict.json -Raw | ConvertFrom-Json).verdict - # A booting backend IS the repair, whatever the log narrated. Judging by - # log text instead failed a POSIX leg whose venv was fine: the only match - # was the frontend build printing "up to date". + # A booting backend IS the repair, whatever the log narrated. Judging by log + # text failed a POSIX leg whose venv was fine: the only match was the frontend + # build printing "up to date". if ($v -eq 'HEALTHY') { Write-Host 're-run repaired the install (verdict=HEALTHY)' exit 0 From 3e4ba7743656ea86a0d9551e883284e54f87ac9e Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Wed, 29 Jul 2026 02:28:44 +0000 Subject: [PATCH 15/25] Tighten the probe docstrings --- .github/scripts/interrupted_install_probe.py | 30 +++++++++----------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/.github/scripts/interrupted_install_probe.py b/.github/scripts/interrupted_install_probe.py index c85d0397dc..9544cc02f5 100644 --- a/.github/scripts/interrupted_install_probe.py +++ b/.github/scripts/interrupted_install_probe.py @@ -6,23 +6,21 @@ resulting venv as healthy -- reproducing the Tauri preflight probes so the regression is testable without building the app. -One implementation for all three platforms. There were briefly two (a shell probe and -an inline PowerShell one), and they diverged: the PowerShell version only ran `-h` and -`desktop-capabilities`, so it could not see the `studio_install_ok`, `verify-install` -or `desktop-runtime-check` signals that the fix PRs introduce -- it would have -reported those PRs as failing no matter how well they worked. A probe that cannot -observe the fix is worse than no probe, hence a single shared one. - The reported bug: quitting the app during the dependency pass SIGTERMs the installer (install.rs stop_install). Landing in the "studio deps" step drops studio/backend/requirements/studio.txt, where structlog is declared. Preflight then -probes `unsloth -h` (preflight/managed.rs:419) and `studio desktop-capabilities` -(managed.rs:318); both SUCCEED because typer/click/rich are core, so the app reports -ManagedReady with can_auto_repair=false and the backend dies on `import structlog`. +probes `unsloth -h` (managed.rs:419) and `studio desktop-capabilities` (managed.rs:318); +both SUCCEED because typer/click/rich are core, so the app reports ManagedReady with +can_auto_repair=false and the backend dies on `import structlog`. + +ONE implementation for all three platforms. The Windows leg was briefly a bespoke inline +PowerShell probe that ran only `-h` and `desktop-capabilities`, so it could not observe +`studio_install_ok`, `verify-install` or `desktop-runtime-check` and would have failed +the very PRs that add them. A probe that cannot see the fix is worse than no probe. Verdicts: HEALTHY the backend boots AND desktop-capabilities reports the install - complete -- i.e. preflight would report ManagedReady and be right + complete -- preflight would report ManagedReady and be right REPAIRABLE the backend is broken AND a probe the DESKTOP consumes reports it, so the app can offer a repair FALSE_READY the backend is broken and every probe says ready -> THE BUG @@ -45,9 +43,9 @@ from pathlib import Path def run(cmd: list[str], timeout: int = 120) -> tuple[int, str, str]: - """Returns (rc, stdout, stderr). Kept SEPARATE: preflight/managed.rs pipes stdout - and sends stderr to /dev/null (managed.rs:358), so anything the probe folds into - stdout is text the desktop never sees.""" + """Returns (rc, stdout, stderr). Kept SEPARATE: preflight pipes stdout and sends + stderr to /dev/null (managed.rs:358), so anything folded into stdout here is text + the desktop never sees.""" try: p = subprocess.run(cmd, capture_output = True, text = True, timeout = timeout) return p.returncode, p.stdout or "", p.stderr or "" @@ -61,8 +59,8 @@ def merged(rc_out_err: tuple[int, str, str]) -> str: def has_subcommand(bin_path: str, args: list[str]) -> bool: - """Whether the CLI understands a subcommand at all. Older builds do not have the - newer verify commands, and 'absent' must not be confused with 'reported failure'.""" + """Whether the CLI understands a subcommand at all: older builds lack the newer + verify commands, and 'absent' must not be read as 'reported failure'.""" rc, _, _ = run([bin_path, *args, "--help"], timeout = 60) return rc == 0 From 52f6b66f5295c2c675ec818ada483aec0c75b30b Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Wed, 29 Jul 2026 02:46:48 +0000 Subject: [PATCH 16/25] Fail the leg when the installer completed inside the kill window --- .github/workflows/interrupted-install-ci.yml | 24 ++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/interrupted-install-ci.yml b/.github/workflows/interrupted-install-ci.yml index 6c3c3b58f5..6b0a9d87ad 100644 --- a/.github/workflows/interrupted-install-ci.yml +++ b/.github/workflows/interrupted-install-ci.yml @@ -130,6 +130,19 @@ jobs: tail -30 logs/install.log || true exit 1 fi + # reason alone is not proof: the driver re-checks liveness after the marker + # delay, but the installer can still finish in the window between that check + # and the signal, leaving reason=marker-hit over a COMPLETED install. The probe + # then reads HEALTHY and the re-run assertion below is skipped, so the leg goes + # green having interrupted nothing. The exit status separates the two: SIGTERM + # takes install.sh's trap to 143 (install.sh:716) and SIGKILL to 137, while only + # an install that ran to completion exits 0. + if [ "$installer_exit" = "0" ]; then + echo "::error::installer exited 0 -- it COMPLETED inside the kill window, so" + echo "::error::nothing was interrupted and this leg asserts nothing." + tail -30 logs/install.log || true + exit 1 + fi - name: What state is the install in? id: probe @@ -253,6 +266,17 @@ jobs: Get-Content logs/install.log -Tail 30 -ErrorAction SilentlyContinue exit 1 } + # Same guard as the POSIX leg: the driver re-checks HasExited after the marker + # delay, but the installer can still finish between that check and Stop-Tree, + # leaving reason=marker-hit over a COMPLETED install that then probes HEALTHY + # and skips the re-run assertion. Stop-Process -Force terminates with a non-zero + # code, so only an install that ran to completion reports 0. + if ($vals['installer_exit'] -eq '0') { + Write-Host '::error::installer exited 0 -- it COMPLETED inside the kill window, so' + Write-Host '::error::nothing was interrupted and this leg asserts nothing.' + Get-Content logs/install.log -Tail 30 -ErrorAction SilentlyContinue + exit 1 + } - name: What state is the install in? id: probe From 8035be12a35d77f03fcb0a4693924aadf4a983a5 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 04:01:24 +0000 Subject: [PATCH 17/25] Land the kill in the marked step, and reject non-boolean capabilities Two holes found from the staging run's own logs. The venv leg never interrupted the venv step. Creating the venv takes ~0.1s, so by the time the 1s poll noticed its line the installer was already in "Installing PyTorch", and the flat 3s beat sent the signal there: staging run 30419729244 shows both step lines in the tail and a kill 4s in. That made the leg a duplicate of the torch leg while its label claimed otherwise. Both drivers now poll in half-second slices, cut the beat short the moment a later [TAURI:STEP] line appears, and print the step the signal actually landed in, warning when it is not the marked one. Sub-step markers such as "studio deps" print no step line, so they keep the whole beat and never warn. studio_install_ok is Option (managed.rs:43), so serde rejects a non-boolean and the whole payload fails to deserialize, which the desktop reports as Stale. bool() read a JSON string "false" as True, so the probe called a torn install ready. Only a literal JSON true counts now. --- .github/scripts/interrupt-install.ps1 | 30 +++++++++++++++++-- .github/scripts/interrupt-install.sh | 31 +++++++++++++++++--- .github/scripts/interrupted_install_probe.py | 17 ++++++++--- 3 files changed, 67 insertions(+), 11 deletions(-) diff --git a/.github/scripts/interrupt-install.ps1 b/.github/scripts/interrupt-install.ps1 index fd763eaedb..7200c1f5f7 100644 --- a/.github/scripts/interrupt-install.ps1 +++ b/.github/scripts/interrupt-install.ps1 @@ -65,14 +65,28 @@ function Stop-Tree([int]$RootId) { catch { } } +function Get-StepCount([string]$Path) { + @(Select-String -Path $Path -Pattern '^\[TAURI:STEP\]' -ErrorAction SilentlyContinue).Count +} + $killed = $false $reason = '' -for ($i = 0; $i -lt $KillAtSeconds; $i++) { +# Half-second slices: a step lasting under a second is over by the time a 1s poll notices +# its line, and the beat below then cannot help. +for ($i = 0; $i -lt ($KillAtSeconds * 2); $i++) { if ($proc.HasExited) { $reason = 'exited-before-marker'; break } if ($Marker) { $hit = Select-String -Path $LogPath -Pattern $Marker -SimpleMatch:$false -ErrorAction SilentlyContinue if ($hit) { - Start-Sleep -Seconds $KillAfterMarkerSeconds + # Same beat as the POSIX driver, waited in slices and cut short once a later + # [TAURI:STEP] line appears, so a fast step finishing inside the beat does not send + # the signal into the step after it. + $stepsAtMarker = Get-StepCount $LogPath + for ($j = 0; $j -lt ($KillAfterMarkerSeconds * 5); $j++) { + Start-Sleep -Milliseconds 200 + if ($proc.HasExited) { break } + if ((Get-StepCount $LogPath) -ne $stepsAtMarker) { break } + } # The installer can finish inside the delay; recording marker-hit before it # let a COMPLETED install satisfy the landing assertion and probe HEALTHY. if ($proc.HasExited) { $reason = 'exited-during-marker-delay'; break } @@ -81,7 +95,7 @@ for ($i = 0; $i -lt $KillAtSeconds; $i++) { break } } - Start-Sleep -Seconds 1 + Start-Sleep -Milliseconds 500 } if (-not $killed -and -not $proc.HasExited) { if (-not $reason) { $reason = 'deadline' }; $killed = $true } @@ -118,6 +132,16 @@ Get-Content $LogPath -Tail 15 -ErrorAction SilentlyContinue if ($Marker -and -not (Select-String -Path $LogPath -Pattern $Marker -ErrorAction SilentlyContinue)) { Write-Host "::warning::marker '$Marker' never appeared -- killed at the deadline, not the intended step" } +# Which step the signal actually landed in. A sub-second step can be over before any log +# poll notices its line, and the leg then silently kills the NEXT step while its matrix +# label still claims the marked one. Sub-step markers ('studio deps') print no +# [TAURI:STEP] line of their own, so the test skips them rather than warning every leg. +$steps = @(Select-String -Path $LogPath -Pattern '^\[TAURI:STEP\]' -ErrorAction SilentlyContinue) +$lastStep = if ($steps.Count) { $steps[-1].Line } else { '' } +Write-Host "[interrupt] step at kill: $lastStep" +if ($Marker -and ($steps | Where-Object { $_.Line -match $Marker }) -and $lastStep -notmatch $Marker) { + Write-Host "::warning::killed in '$lastStep', not the marked step -- that step was already over" +} @( "interrupt_reason=$reason" "interrupt_killed=$killed" diff --git a/.github/scripts/interrupt-install.sh b/.github/scripts/interrupt-install.sh index c24382c56b..276f5593cc 100755 --- a/.github/scripts/interrupt-install.sh +++ b/.github/scripts/interrupt-install.sh @@ -46,15 +46,26 @@ echo "[interrupt] installer pid/pgid=$PID marker='${MARKER}' deadline=${KILL_AT_ killed=false reason="" -for i in $(seq 1 "$KILL_AT_SECONDS"); do +# Half-second slices, not one-second: a step that lasts under a second is over by the time +# a 1s poll notices its line, and the beat below then cannot help. +for i in $(seq 1 $(( KILL_AT_SECONDS * 2 ))); do if ! kill -0 "$PID" 2>/dev/null; then reason="exited-before-marker" break fi if [ -n "$MARKER" ] && grep -qE "$MARKER" "$LOG" 2>/dev/null; then # A beat into the step, so the kill lands mid-work rather than on the boundary - # before the step has touched the venv. - sleep "${KILL_AFTER_MARKER_SECONDS:-3}" + # before the step has touched the venv. Waited in slices and cut short the moment a + # later [TAURI:STEP] line appears: creating the venv takes ~0.1s, so a flat 3s sleep + # sent the venv leg's signal into "Installing PyTorch" and made it a duplicate of the + # torch leg. Sub-step markers ("studio deps") print no step line of their own, so + # they still get the whole beat. + _steps_at_marker="$(grep -cE '^\[TAURI:STEP\]' "$LOG" 2>/dev/null || true)" + for _ in $(seq 1 $(( ${KILL_AFTER_MARKER_SECONDS:-3} * 5 ))); do + sleep 0.2 + kill -0 "$PID" 2>/dev/null || break + [ "$(grep -cE '^\[TAURI:STEP\]' "$LOG" 2>/dev/null || true)" = "$_steps_at_marker" ] || break + done # ...but a cached step can FINISH inside that beat. Recording marker-hit before the # sleep handed the landing assertion a COMPLETED install: the signal reached no # process and the leg passed green having interrupted nothing. @@ -66,7 +77,7 @@ for i in $(seq 1 "$KILL_AT_SECONDS"); do killed=true break fi - sleep 1 + sleep 0.5 done if [ "$killed" != "true" ] && kill -0 "$PID" 2>/dev/null; then @@ -114,6 +125,18 @@ tail -15 "$LOG" || true if [ -n "$MARKER" ] && ! grep -qE "$MARKER" "$LOG" 2>/dev/null; then echo "::warning::marker '$MARKER' never appeared -- this leg killed at the deadline, not at the intended step" fi +# Which step the signal actually landed in. A sub-second step (creating the venv takes +# ~0.1s) can be over before any log poll notices its line, and the leg then silently kills +# the NEXT step while its matrix label still claims the marked one. Only steps say where +# they landed: sub-step markers ("studio deps") print no [TAURI:STEP] line of their own, so +# the test skips them rather than warning on every leg. +_last_step="$(grep -E '^\[TAURI:STEP\]' "$LOG" 2>/dev/null | tail -1)" +echo "[interrupt] step at kill: $_last_step" +if [ -n "$MARKER" ] && + grep -E '^\[TAURI:STEP\]' "$LOG" 2>/dev/null | grep -qE "$MARKER" && + ! printf '%s\n' "$_last_step" | grep -qE "$MARKER"; then + echo "::warning::killed in '$_last_step', not the marked step -- that step was already over" +fi { echo "interrupt_reason=$reason" echo "interrupt_killed=$killed" diff --git a/.github/scripts/interrupted_install_probe.py b/.github/scripts/interrupted_install_probe.py index 9544cc02f5..1926512d08 100644 --- a/.github/scripts/interrupted_install_probe.py +++ b/.github/scripts/interrupted_install_probe.py @@ -124,14 +124,23 @@ def main(argv: list[str]) -> int: # "absent" (studio_install_ok predates the install-manifest work) and "unparseable" # are split apart only for a readable artefact: the desktop reports Stale for both # ("desktop_capability_probe_failed", managed.rs:521). + # The field is Option (managed.rs:43), so serde rejects a non-boolean and the + # WHOLE payload fails to deserialize -> Stale. bool() instead read a JSON string + # "false" as True and reported HEALTHY over a torn install, skipping the repair + # assertion. Only a literal JSON true counts. install_ok: object = "absent" try: parsed = json.loads(caps_out) - if isinstance(parsed, dict): - v = parsed.get("studio_install_ok") - install_ok = "absent" if v is None else bool(v) - else: + if not isinstance(parsed, dict): install_ok = "unparseable" + else: + v = parsed.get("studio_install_ok") + if v is None: + install_ok = "absent" + elif isinstance(v, bool): + install_ok = v + else: + install_ok = "non-boolean" except json.JSONDecodeError: install_ok = "unparseable" say("capabilities.studio_install_ok", install_ok) From 05e417d966b02e899e010b83312919a0f56090a0 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 04:06:06 +0000 Subject: [PATCH 18/25] Tighten the interrupt driver comments --- .github/scripts/interrupt-install.sh | 68 ++++++++++++---------------- 1 file changed, 28 insertions(+), 40 deletions(-) diff --git a/.github/scripts/interrupt-install.sh b/.github/scripts/interrupt-install.sh index 276f5593cc..772481ccd0 100755 --- a/.github/scripts/interrupt-install.sh +++ b/.github/scripts/interrupt-install.sh @@ -3,16 +3,14 @@ # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. # # Run install.sh and SIGTERM it partway through, reproducing a user quitting the desktop -# app mid-install: main.rs cleanup_child_processes() -> install::stop_install() -> kill -# the installer PROCESS GROUP (install.rs:798-807). It must be the group: killing only -# the leader leaves `uv`/`python` children to finish the dep pass, proving nothing. +# app mid-install: main.rs cleanup_child_processes() -> install::stop_install() kills the +# installer PROCESS GROUP (install.rs:798-807). Group, not leader: `uv`/`python` children +# would otherwise finish the dep pass and the leg would prove nothing. # # Usage: bash .github/scripts/interrupt-install.sh "" "" [-- install args] -# regex to wait for in the install log before killing, e.g. "studio deps" -# or "\[TAURI:STEP\] Installing PyTorch". Use "" to kill after --at-seconds. -# Env: -# KILL_AT_SECONDS hard deadline; kill even if the marker never appears (default 900) -# KILL_GRACE seconds to wait for the group to die before SIGKILL (default 10) +# log regex to wait for before killing, e.g. "studio deps". "" kills at the +# deadline. +# Env: KILL_AT_SECONDS deadline (default 900), KILL_GRACE grace before SIGKILL (default 10) set -uo pipefail MARKER="${1:-}" @@ -25,19 +23,18 @@ KILL_GRACE="${KILL_GRACE:-10}" mkdir -p "$(dirname "$LOG")" : > "$LOG" -# Stand in for the desktop app, which writes this before spawning the installer -# (install.rs). We kill the installer directly, so without it the marker #7490 relies on -# is absent for a reason unrelated to #7490. Both locations because the Rust side -# hardcodes ~/.unsloth/studio while CI overrides UNSLOTH_STUDIO_HOME. Never cleared: -# being killed is the whole point. +# The desktop writes this before spawning the installer (install.rs); we kill the +# installer directly, so without it #7490's marker is absent for an unrelated reason. Both +# roots: Rust hardcodes ~/.unsloth/studio, CI overrides UNSLOTH_STUDIO_HOME. Never +# cleared, being killed is the point. for _marker_dir in "${UNSLOTH_STUDIO_HOME:-}" "$HOME/.unsloth/studio"; do [ -n "$_marker_dir" ] || continue mkdir -p "$_marker_dir" 2>/dev/null || continue : > "$_marker_dir/.desktop-install-in-progress" 2>/dev/null || true done -# Job control puts the child in its own process group, so $! is the pgid leader and -# `kill -- -$!` reaches every descendant, matching the Rust side. +# Job control puts the child in its own group, so $! is the pgid and `kill -- -$!` +# reaches every descendant, as the Rust side does. set -m bash install.sh "$@" > "$LOG" 2>&1 & PID=$! @@ -46,29 +43,25 @@ echo "[interrupt] installer pid/pgid=$PID marker='${MARKER}' deadline=${KILL_AT_ killed=false reason="" -# Half-second slices, not one-second: a step that lasts under a second is over by the time -# a 1s poll notices its line, and the beat below then cannot help. +# Half-second slices: a sub-second step is over before a 1s poll sees its line. for i in $(seq 1 $(( KILL_AT_SECONDS * 2 ))); do if ! kill -0 "$PID" 2>/dev/null; then reason="exited-before-marker" break fi if [ -n "$MARKER" ] && grep -qE "$MARKER" "$LOG" 2>/dev/null; then - # A beat into the step, so the kill lands mid-work rather than on the boundary - # before the step has touched the venv. Waited in slices and cut short the moment a - # later [TAURI:STEP] line appears: creating the venv takes ~0.1s, so a flat 3s sleep - # sent the venv leg's signal into "Installing PyTorch" and made it a duplicate of the - # torch leg. Sub-step markers ("studio deps") print no step line of their own, so - # they still get the whole beat. + # A beat into the step so the kill lands mid-work, cut short the moment a later + # [TAURI:STEP] line appears: the venv takes ~0.1s, so a flat 3s sleep put the venv + # leg's signal in "Installing PyTorch", a duplicate of the torch leg. Sub-step + # markers ("studio deps") print no step line, so they keep the whole beat. _steps_at_marker="$(grep -cE '^\[TAURI:STEP\]' "$LOG" 2>/dev/null || true)" for _ in $(seq 1 $(( ${KILL_AFTER_MARKER_SECONDS:-3} * 5 ))); do sleep 0.2 kill -0 "$PID" 2>/dev/null || break [ "$(grep -cE '^\[TAURI:STEP\]' "$LOG" 2>/dev/null || true)" = "$_steps_at_marker" ] || break done - # ...but a cached step can FINISH inside that beat. Recording marker-hit before the - # sleep handed the landing assertion a COMPLETED install: the signal reached no - # process and the leg passed green having interrupted nothing. + # ...but a cached step can FINISH inside the beat. Recording marker-hit before it + # handed the landing assertion a COMPLETED install that interrupted nothing. if ! kill -0 "$PID" 2>/dev/null; then reason="exited-during-marker-delay" break @@ -92,10 +85,9 @@ if [ "$killed" = "true" ]; then kill -0 "$PID" 2>/dev/null || break sleep 1 done - # Unconditional, and to the GROUP. The leader can exit on SIGTERM while a uv or python - # descendant does not; gating on `kill -0 "$PID"` skipped this escalation and left that - # descendant free to finish the dependency pass while the probe ran. Signalling an - # already-empty group is a no-op. + # Unconditional, and to the GROUP: the leader exits on SIGTERM while a uv or python + # descendant does not, and gating on `kill -0 "$PID"` left that child finishing the dep + # pass under the probe. Signalling an empty group is a no-op. echo "[interrupt] SIGKILL to process group -$PID" kill -KILL -- -"$PID" 2>/dev/null || kill -KILL "$PID" 2>/dev/null || true fi @@ -103,9 +95,8 @@ fi wait "$PID" 2>/dev/null rc=$? -# Only after the reap: an unreaped leader is still a member of its own group, so polling -# the group before `wait` would report it alive forever. The probe must not start while -# an installer process is still running. +# Only after the reap: an unreaped leader is still a member of its own group, so this +# poll would report it alive forever. No installer may still run when the probe starts. if [ "$killed" = "true" ]; then for _ in $(seq 1 "$KILL_GRACE"); do kill -0 -- -"$PID" 2>/dev/null || break @@ -120,16 +111,13 @@ echo "[interrupt] installer exit=$rc reason=$reason killed=$killed" echo "[interrupt] last log lines:" tail -15 "$LOG" || true -# Report how far it got, so a leg that never reached the target step is visible rather -# than passing for the wrong reason. +# A leg that never reached the target step must be visible, not quietly green. if [ -n "$MARKER" ] && ! grep -qE "$MARKER" "$LOG" 2>/dev/null; then echo "::warning::marker '$MARKER' never appeared -- this leg killed at the deadline, not at the intended step" fi -# Which step the signal actually landed in. A sub-second step (creating the venv takes -# ~0.1s) can be over before any log poll notices its line, and the leg then silently kills -# the NEXT step while its matrix label still claims the marked one. Only steps say where -# they landed: sub-step markers ("studio deps") print no [TAURI:STEP] line of their own, so -# the test skips them rather than warning on every leg. +# Where the signal actually landed. A sub-second step can end before any poll sees its +# line, and the leg then kills the NEXT step while its label claims otherwise. Sub-step +# markers print no [TAURI:STEP] line, so the test skips them instead of always warning. _last_step="$(grep -E '^\[TAURI:STEP\]' "$LOG" 2>/dev/null | tail -1)" echo "[interrupt] step at kill: $_last_step" if [ -n "$MARKER" ] && From 1892b130ce23e6d973f49d9b4fe49c69fce2ce9c Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 04:44:54 +0000 Subject: [PATCH 19/25] Fail the leg when the signal landed after the marked step The cut-short added last round only helps when the marked step is still the last [TAURI:STEP] line at the moment the poll notices it. Creating the venv takes ~0.1s (staging run 30419729244: 03:31:07.371 -> 07.478), less than the 0.5s poll, so the next step's line is usually already in the log when the marker matches, the step count never changes during the beat, and the full 3s elapses inside "Installing PyTorch". Reproduced with a stub installer against the driver at head: kill at 4.11s, step at kill "Installing PyTorch". The leg then duplicates the torch leg while its matrix label claims the venv step, and passed green on nothing but a ::warning::. Both drivers now skip the beat entirely when the marked step is already over, so the kill goes out at once instead of deeper into the next step, and both record interrupt_step_mismatch in interrupt.env. The landing assertion fails on it: a warning that cannot fail the leg proves nothing. Sub-step markers ("studio deps", "pip bootstrap") print no step line of their own and stay exempt, as before. The venv leg becomes experimental. Its step is shorter than any log poll can resolve, so it must not block the PR on a race it cannot win, and it still probes the earliest torn state whenever it does land. --- .github/scripts/interrupt-install.ps1 | 41 +++++++++++++++----- .github/scripts/interrupt-install.sh | 36 ++++++++++++----- .github/workflows/interrupted-install-ci.yml | 28 ++++++++++++- 3 files changed, 86 insertions(+), 19 deletions(-) diff --git a/.github/scripts/interrupt-install.ps1 b/.github/scripts/interrupt-install.ps1 index 7200c1f5f7..1d8ac91ae1 100644 --- a/.github/scripts/interrupt-install.ps1 +++ b/.github/scripts/interrupt-install.ps1 @@ -69,6 +69,22 @@ function Get-StepCount([string]$Path) { @(Select-String -Path $Path -Pattern '^\[TAURI:STEP\]' -ErrorAction SilentlyContinue).Count } +function Get-LastStep([string]$Path) { + $steps = @(Select-String -Path $Path -Pattern '^\[TAURI:STEP\]' -ErrorAction SilentlyContinue) + if ($steps.Count) { return $steps[-1].Line } + return '' +} + +# True when the marker names a [TAURI:STEP] line that is no longer the last one: the step +# ended before the poll noticed it. Sub-step markers ('studio deps') print no step line of +# their own, so they are never judged here. +function Test-MarkedStepOver { + if (-not $Marker) { return $false } + $steps = @(Select-String -Path $LogPath -Pattern '^\[TAURI:STEP\]' -ErrorAction SilentlyContinue) + if (-not ($steps | Where-Object { $_.Line -match $Marker })) { return $false } + return ((Get-LastStep $LogPath) -notmatch $Marker) +} + $killed = $false $reason = '' # Half-second slices: a step lasting under a second is over by the time a 1s poll notices @@ -80,12 +96,16 @@ for ($i = 0; $i -lt ($KillAtSeconds * 2); $i++) { if ($hit) { # Same beat as the POSIX driver, waited in slices and cut short once a later # [TAURI:STEP] line appears, so a fast step finishing inside the beat does not send - # the signal into the step after it. - $stepsAtMarker = Get-StepCount $LogPath - for ($j = 0; $j -lt ($KillAfterMarkerSeconds * 5); $j++) { - Start-Sleep -Milliseconds 200 - if ($proc.HasExited) { break } - if ((Get-StepCount $LogPath) -ne $stepsAtMarker) { break } + # the signal into the step after it. Skipped when the step is ALREADY over: the + # cut-short cannot help once the next step's line is in the log before the first + # sample, and beating on would push the signal deeper into the following step. + if (-not (Test-MarkedStepOver)) { + $stepsAtMarker = Get-StepCount $LogPath + for ($j = 0; $j -lt ($KillAfterMarkerSeconds * 5); $j++) { + Start-Sleep -Milliseconds 200 + if ($proc.HasExited) { break } + if ((Get-StepCount $LogPath) -ne $stepsAtMarker) { break } + } } # The installer can finish inside the delay; recording marker-hit before it # let a COMPLETED install satisfy the landing assertion and probe HEALTHY. @@ -136,15 +156,18 @@ if ($Marker -and -not (Select-String -Path $LogPath -Pattern $Marker -ErrorActio # poll notices its line, and the leg then silently kills the NEXT step while its matrix # label still claims the marked one. Sub-step markers ('studio deps') print no # [TAURI:STEP] line of their own, so the test skips them rather than warning every leg. -$steps = @(Select-String -Path $LogPath -Pattern '^\[TAURI:STEP\]' -ErrorAction SilentlyContinue) -$lastStep = if ($steps.Count) { $steps[-1].Line } else { '' } +$lastStep = Get-LastStep $LogPath Write-Host "[interrupt] step at kill: $lastStep" -if ($Marker -and ($steps | Where-Object { $_.Line -match $Marker }) -and $lastStep -notmatch $Marker) { +$mismatch = Test-MarkedStepOver +if ($mismatch) { Write-Host "::warning::killed in '$lastStep', not the marked step -- that step was already over" } +# Lower-cased so the workflow can compare it the same way on every platform, and only +# simple values: the POSIX side sources this file. @( "interrupt_reason=$reason" "interrupt_killed=$killed" "installer_exit=$rc" + "interrupt_step_mismatch=$(if ($mismatch) { 'true' } else { 'false' })" ) | Set-Content -Path (Join-Path (Split-Path -Parent $LogPath) 'interrupt.env') -Encoding utf8 exit 0 diff --git a/.github/scripts/interrupt-install.sh b/.github/scripts/interrupt-install.sh index 772481ccd0..f175758540 100755 --- a/.github/scripts/interrupt-install.sh +++ b/.github/scripts/interrupt-install.sh @@ -41,6 +41,16 @@ PID=$! set +m echo "[interrupt] installer pid/pgid=$PID marker='${MARKER}' deadline=${KILL_AT_SECONDS}s" +# True when the marker names a [TAURI:STEP] line that is no longer the last one: the step +# ended before the poll noticed it. Sub-step markers ("studio deps") print no step line of +# their own, so they are never judged here. +marked_step_over() { + [ -n "$MARKER" ] || return 1 + grep -E '^\[TAURI:STEP\]' "$LOG" 2>/dev/null | grep -qE "$MARKER" || return 1 + grep -E '^\[TAURI:STEP\]' "$LOG" 2>/dev/null | tail -1 | grep -qE "$MARKER" && return 1 + return 0 +} + killed=false reason="" # Half-second slices: a sub-second step is over before a 1s poll sees its line. @@ -54,12 +64,18 @@ for i in $(seq 1 $(( KILL_AT_SECONDS * 2 ))); do # [TAURI:STEP] line appears: the venv takes ~0.1s, so a flat 3s sleep put the venv # leg's signal in "Installing PyTorch", a duplicate of the torch leg. Sub-step # markers ("studio deps") print no step line, so they keep the whole beat. - _steps_at_marker="$(grep -cE '^\[TAURI:STEP\]' "$LOG" 2>/dev/null || true)" - for _ in $(seq 1 $(( ${KILL_AFTER_MARKER_SECONDS:-3} * 5 ))); do - sleep 0.2 - kill -0 "$PID" 2>/dev/null || break - [ "$(grep -cE '^\[TAURI:STEP\]' "$LOG" 2>/dev/null || true)" = "$_steps_at_marker" ] || break - done + # Skipped entirely when the step is ALREADY over: cutting the beat short cannot help + # once the next step's line is in the log before the first sample, which is the normal + # case for a 0.1s step, and beating on would only push the signal deeper into the step + # after it. Kill now and let the mismatch check below report where it landed. + if ! marked_step_over; then + _steps_at_marker="$(grep -cE '^\[TAURI:STEP\]' "$LOG" 2>/dev/null || true)" + for _ in $(seq 1 $(( ${KILL_AFTER_MARKER_SECONDS:-3} * 5 ))); do + sleep 0.2 + kill -0 "$PID" 2>/dev/null || break + [ "$(grep -cE '^\[TAURI:STEP\]' "$LOG" 2>/dev/null || true)" = "$_steps_at_marker" ] || break + done + fi # ...but a cached step can FINISH inside the beat. Recording marker-hit before it # handed the landing assertion a COMPLETED install that interrupted nothing. if ! kill -0 "$PID" 2>/dev/null; then @@ -120,14 +136,16 @@ fi # markers print no [TAURI:STEP] line, so the test skips them instead of always warning. _last_step="$(grep -E '^\[TAURI:STEP\]' "$LOG" 2>/dev/null | tail -1)" echo "[interrupt] step at kill: $_last_step" -if [ -n "$MARKER" ] && - grep -E '^\[TAURI:STEP\]' "$LOG" 2>/dev/null | grep -qE "$MARKER" && - ! printf '%s\n' "$_last_step" | grep -qE "$MARKER"; then +mismatch=false +if marked_step_over; then + mismatch=true echo "::warning::killed in '$_last_step', not the marked step -- that step was already over" fi +# Only simple values: the workflow sources this file, so the step text stays out of it. { echo "interrupt_reason=$reason" echo "interrupt_killed=$killed" echo "installer_exit=$rc" + echo "interrupt_step_mismatch=$mismatch" } > "$(dirname "$LOG")/interrupt.env" exit 0 diff --git a/.github/workflows/interrupted-install-ci.yml b/.github/workflows/interrupted-install-ci.yml index 6b0a9d87ad..a09d44ebe7 100644 --- a/.github/workflows/interrupted-install-ci.yml +++ b/.github/workflows/interrupted-install-ci.yml @@ -77,7 +77,13 @@ jobs: # The exact reported case: killed during the step that installs structlog. - {os: macos-14, label: studio-deps, marker: 'studio deps', experimental: false} # Coarse phases, earliest to latest -- each leaves a different partial venv. - - {os: macos-14, label: venv, marker: '\[TAURI:STEP\] Creating virtual environment', experimental: false} + # venv is experimental because the step is not reliably interruptible: with a + # warm uv cache creating it takes ~0.1s (staging run 30419729244 logged + # 03:31:07.371 -> 07.478 between its step line and "Installing PyTorch"), less + # than the log poll, so the kill often lands in the next step. The landing check + # now FAILS that instead of warning, and this leg must not block the PR on a race + # no log poll can win. It still probes the earliest torn state whenever it lands. + - {os: macos-14, label: venv, marker: '\[TAURI:STEP\] Creating virtual environment', experimental: true} - {os: macos-14, label: torch, marker: '\[TAURI:STEP\] Installing PyTorch', experimental: false} - {os: macos-14, label: unsloth, marker: '\[TAURI:STEP\] Installing Unsloth', experimental: false} - {os: macos-14, label: setup, marker: '\[TAURI:STEP\] Running Unsloth setup', experimental: false} @@ -143,6 +149,17 @@ jobs: tail -30 logs/install.log || true exit 1 fi + # ...and the signal has to have landed in the step this leg is named after. A + # step whose line is no longer the last one was already over when the poll saw + # it, so the kill hit the NEXT step and the leg silently duplicates another one + # while its label claims otherwise. A warning here proves nothing, so it fails. + # Sub-step markers ("studio deps") print no [TAURI:STEP] line and are exempt. + if [ "$interrupt_step_mismatch" = "true" ]; then + echo "::error::the signal landed after '${{ matrix.marker }}' finished, so this" + echo "::error::leg interrupted a later step than the one it is named for." + grep -E '^\[TAURI:STEP\]' logs/install.log || true + exit 1 + fi - name: What state is the install in? id: probe @@ -277,6 +294,15 @@ jobs: Get-Content logs/install.log -Tail 30 -ErrorAction SilentlyContinue exit 1 } + # Same landing check as the POSIX leg: a step that was already over when the poll + # saw its line means the kill hit the NEXT step, so the leg duplicates another + # one while its label claims otherwise. + if ($vals['interrupt_step_mismatch'] -eq 'true') { + Write-Host "::error::the signal landed after '${{ matrix.marker }}' finished, so this" + Write-Host '::error::leg interrupted a later step than the one it is named for.' + Select-String -Path logs/install.log -Pattern '^\[TAURI:STEP\]' -ErrorAction SilentlyContinue + exit 1 + } - name: What state is the install in? id: probe From cf5d7affad7f4d2767865add0e52d7a7a420feb7 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 04:52:17 +0000 Subject: [PATCH 20/25] Tighten the interrupted-install workflow and driver comments --- .github/scripts/interrupt-install.ps1 | 55 +++---- .github/scripts/interrupt-install.sh | 20 +-- .github/scripts/interrupted_install_probe.py | 75 ++++----- .github/workflows/interrupted-install-ci.yml | 157 ++++++++----------- 4 files changed, 131 insertions(+), 176 deletions(-) diff --git a/.github/scripts/interrupt-install.ps1 b/.github/scripts/interrupt-install.ps1 index 1d8ac91ae1..05e3dc237c 100644 --- a/.github/scripts/interrupt-install.ps1 +++ b/.github/scripts/interrupt-install.ps1 @@ -4,9 +4,8 @@ # Windows counterpart of interrupt-install.sh: run install.ps1 and kill it partway # through, reproducing a user quitting the desktop app mid-install. # -# Windows has no process groups, which is why the app carries windows_job.rs. This script -# kills the whole process TREE for the same reason: killing only the leader leaves -# uv/python children to finish the dependency pass, proving nothing. +# Windows has no process groups (hence the app's windows_job.rs), so this kills the whole +# process TREE: killing only the leader leaves uv/python children to finish the dep pass. # # Usage: # pwsh -File .github/scripts/interrupt-install.ps1 -Marker 'studio deps' ` @@ -25,10 +24,9 @@ New-Item -ItemType Directory -Force -Path (Split-Path -Parent $LogPath) | Out-Nu Set-Content -Path $LogPath -Value '' -Encoding utf8 # Stand in for the desktop app, which writes this before spawning the installer -# (install.rs). We kill install.ps1 directly, so without it the marker #7490 relies on is -# absent for a reason unrelated to #7490 -- exactly what the Windows legs reported. Both -# locations because the Rust side hardcodes ~/.unsloth/studio while CI overrides -# UNSLOTH_STUDIO_HOME. Never cleared: being killed is the whole point. +# (install.rs). We kill install.ps1 directly, so without it #7490's marker is absent for an +# unrelated reason -- exactly what the Windows legs reported. Both locations: Rust +# hardcodes ~/.unsloth/studio, CI overrides UNSLOTH_STUDIO_HOME. Never cleared by design. foreach ($dir in @($env:UNSLOTH_STUDIO_HOME, (Join-Path $HOME '.unsloth\studio'))) { if ([string]::IsNullOrWhiteSpace($dir)) { continue } try { @@ -37,13 +35,12 @@ foreach ($dir in @($env:UNSLOTH_STUDIO_HOME, (Join-Path $HOME '.unsloth\studio') } catch { Write-Host "[interrupt] could not seed install marker in ${dir}: $_" } } -# Its own host, so stdout can be redirected to the log while we poll. That host is -# WINDOWS PowerShell 5.1, not pwsh, with install.rs:325-339's exact flags: that is the -# only host a real desktop install ever uses, and every other Windows job in .github runs -# install.ps1 under pwsh 7, leaving 5.1 behaviour (.NET Framework, OEM/ANSI console -# encoding, different native-command and OSArchitecture reporting) covered by nothing. -# The driver itself stays under pwsh; only the installer child and the repair re-run -# change. +# Its own host, so stdout can be redirected to the log while we poll. That host is WINDOWS +# PowerShell 5.1, not pwsh, with install.rs:325-339's exact flags: the only host a real +# desktop install ever uses, while every other Windows job in .github runs install.ps1 +# under pwsh 7, leaving 5.1 behaviour (.NET Framework, OEM/ANSI console encoding, different +# native-command and OSArchitecture reporting) covered by nothing. The driver stays under +# pwsh; only the installer child and the repair re-run change. $argList = @( '-NoLogo', '-NoProfile', '-NonInteractive', '-WindowStyle', 'Hidden', @@ -87,18 +84,16 @@ function Test-MarkedStepOver { $killed = $false $reason = '' -# Half-second slices: a step lasting under a second is over by the time a 1s poll notices -# its line, and the beat below then cannot help. +# Half-second slices: a sub-second step is over before a 1s poll sees its line. for ($i = 0; $i -lt ($KillAtSeconds * 2); $i++) { if ($proc.HasExited) { $reason = 'exited-before-marker'; break } if ($Marker) { $hit = Select-String -Path $LogPath -Pattern $Marker -SimpleMatch:$false -ErrorAction SilentlyContinue if ($hit) { - # Same beat as the POSIX driver, waited in slices and cut short once a later - # [TAURI:STEP] line appears, so a fast step finishing inside the beat does not send - # the signal into the step after it. Skipped when the step is ALREADY over: the - # cut-short cannot help once the next step's line is in the log before the first - # sample, and beating on would push the signal deeper into the following step. + # Same beat as the POSIX driver, in slices and cut short once a later [TAURI:STEP] + # line appears, so a fast step does not send the signal into the step after it. + # Skipped when the step is ALREADY over: the cut-short cannot help once the next + # line is logged, and beating on would push the signal deeper into the next step. if (-not (Test-MarkedStepOver)) { $stepsAtMarker = Get-StepCount $LogPath for ($j = 0; $j -lt ($KillAfterMarkerSeconds * 5); $j++) { @@ -123,12 +118,11 @@ if ($killed) { Write-Host "[interrupt] killing process tree of $($proc.Id) ($reason)" Stop-Tree $proc.Id # Any straggler uv/python that reparented away from the installer. The old sweep matched - # nothing: UNSLOTH_STUDIO_HOME arrives as `D:\a\r\r/.studio-home` (github.workspace - # joined with a forward slash) while Process.Path is all backslashes, so the literal - # -like missed even the venv's own python. Hence the separator normalisation, and uv by - # name (it lives outside the studio home, and the ephemeral runner has no other uv). - # Under --tauri there is no UNSLOTH_STUDIO_HOME, so fall back to the root install.ps1 - # uses then, or the sweep would only ever see uv. + # nothing: UNSLOTH_STUDIO_HOME arrives as `D:\a\r\r/.studio-home` (github.workspace joined + # with a forward slash) while Process.Path is all backslashes, so the literal -like missed + # even the venv's own python. Hence the separator normalisation, and uv by name (it lives + # outside the studio home, and the ephemeral runner has no other uv). Under --tauri there + # is no UNSLOTH_STUDIO_HOME, so fall back to install.ps1's root or the sweep only sees uv. $studioRoot = if ([string]::IsNullOrWhiteSpace($env:UNSLOTH_STUDIO_HOME)) { Join-Path $HOME '.unsloth\studio' } else { $env:UNSLOTH_STUDIO_HOME } $homeNorm = if ([string]::IsNullOrWhiteSpace($studioRoot)) { $null } @@ -152,10 +146,9 @@ Get-Content $LogPath -Tail 15 -ErrorAction SilentlyContinue if ($Marker -and -not (Select-String -Path $LogPath -Pattern $Marker -ErrorAction SilentlyContinue)) { Write-Host "::warning::marker '$Marker' never appeared -- killed at the deadline, not the intended step" } -# Which step the signal actually landed in. A sub-second step can be over before any log -# poll notices its line, and the leg then silently kills the NEXT step while its matrix -# label still claims the marked one. Sub-step markers ('studio deps') print no -# [TAURI:STEP] line of their own, so the test skips them rather than warning every leg. +# Where the signal actually landed. A sub-second step can end before any poll sees its +# line, and the leg then kills the NEXT step while its label claims otherwise. Sub-step +# markers print no [TAURI:STEP] line, so the test skips them instead of always warning. $lastStep = Get-LastStep $LogPath Write-Host "[interrupt] step at kill: $lastStep" $mismatch = Test-MarkedStepOver diff --git a/.github/scripts/interrupt-install.sh b/.github/scripts/interrupt-install.sh index f175758540..5a9b052ce5 100755 --- a/.github/scripts/interrupt-install.sh +++ b/.github/scripts/interrupt-install.sh @@ -8,8 +8,7 @@ # would otherwise finish the dep pass and the leg would prove nothing. # # Usage: bash .github/scripts/interrupt-install.sh "" "" [-- install args] -# log regex to wait for before killing, e.g. "studio deps". "" kills at the -# deadline. +# log regex to wait for before killing, e.g. "studio deps"; "" kills at deadline. # Env: KILL_AT_SECONDS deadline (default 900), KILL_GRACE grace before SIGKILL (default 10) set -uo pipefail @@ -23,10 +22,9 @@ KILL_GRACE="${KILL_GRACE:-10}" mkdir -p "$(dirname "$LOG")" : > "$LOG" -# The desktop writes this before spawning the installer (install.rs); we kill the -# installer directly, so without it #7490's marker is absent for an unrelated reason. Both -# roots: Rust hardcodes ~/.unsloth/studio, CI overrides UNSLOTH_STUDIO_HOME. Never -# cleared, being killed is the point. +# The desktop writes this before spawning the installer (install.rs); we kill the installer +# directly, so without it #7490's marker is absent for an unrelated reason. Both roots: Rust +# hardcodes ~/.unsloth/studio, CI overrides UNSLOTH_STUDIO_HOME. Never cleared by design. for _marker_dir in "${UNSLOTH_STUDIO_HOME:-}" "$HOME/.unsloth/studio"; do [ -n "$_marker_dir" ] || continue mkdir -p "$_marker_dir" 2>/dev/null || continue @@ -62,12 +60,10 @@ for i in $(seq 1 $(( KILL_AT_SECONDS * 2 ))); do if [ -n "$MARKER" ] && grep -qE "$MARKER" "$LOG" 2>/dev/null; then # A beat into the step so the kill lands mid-work, cut short the moment a later # [TAURI:STEP] line appears: the venv takes ~0.1s, so a flat 3s sleep put the venv - # leg's signal in "Installing PyTorch", a duplicate of the torch leg. Sub-step - # markers ("studio deps") print no step line, so they keep the whole beat. - # Skipped entirely when the step is ALREADY over: cutting the beat short cannot help - # once the next step's line is in the log before the first sample, which is the normal - # case for a 0.1s step, and beating on would only push the signal deeper into the step - # after it. Kill now and let the mismatch check below report where it landed. + # leg's signal in "Installing PyTorch", a duplicate of the torch leg. Sub-step markers + # ("studio deps") print no step line and keep the whole beat. Skipped when the step is + # ALREADY over: the cut-short cannot help once the next line is logged and beating on + # only pushes the signal deeper, so kill now and let the mismatch check below report. if ! marked_step_over; then _steps_at_marker="$(grep -cE '^\[TAURI:STEP\]' "$LOG" 2>/dev/null || true)" for _ in $(seq 1 $(( ${KILL_AFTER_MARKER_SECONDS:-3} * 5 ))); do diff --git a/.github/scripts/interrupted_install_probe.py b/.github/scripts/interrupted_install_probe.py index 1926512d08..d302a38d2c 100644 --- a/.github/scripts/interrupted_install_probe.py +++ b/.github/scripts/interrupted_install_probe.py @@ -95,9 +95,8 @@ def main(argv: list[str]) -> int: # ── the two probes Tauri preflight actually runs ───────────────────────── # The DESKTOP's deadline, not a generous CI one: preflight times each call out after # 10s (managed.rs:337 for `-h`, :390 for desktop-capabilities) and reports Stale - # (managed.rs:471, :521). A longer timeout here would call a slow torn venv HEALTHY - # and skip the re-run assertion. run() reports a timeout as a non-zero rc, landing in - # the same REPAIRABLE arm as Stale. + # (managed.rs:471, :521). A longer timeout would call a slow torn venv HEALTHY and skip + # the re-run assertion; run() reports a timeout as a non-zero rc, the same REPAIRABLE arm. PREFLIGHT_TIMEOUT = 10 t0 = time.time() @@ -122,12 +121,11 @@ def main(argv: list[str]) -> int: # discarded at managed.rs:358. Folding stderr in made one warning line enough to fail # the parse and report FALSE_READY over an install the real app offers to repair. # "absent" (studio_install_ok predates the install-manifest work) and "unparseable" - # are split apart only for a readable artefact: the desktop reports Stale for both - # ("desktop_capability_probe_failed", managed.rs:521). - # The field is Option (managed.rs:43), so serde rejects a non-boolean and the - # WHOLE payload fails to deserialize -> Stale. bool() instead read a JSON string - # "false" as True and reported HEALTHY over a torn install, skipping the repair - # assertion. Only a literal JSON true counts. + # split only for a readable artefact: the desktop reports Stale for both + # ("desktop_capability_probe_failed", managed.rs:521). The field is Option + # (managed.rs:43), so serde rejects a non-boolean and the WHOLE payload fails to + # deserialize -> Stale; bool() instead read the JSON string "false" as True and + # reported HEALTHY over a torn install. Only a literal JSON true counts. install_ok: object = "absent" try: parsed = json.loads(caps_out) @@ -147,19 +145,17 @@ def main(argv: list[str]) -> int: # The desktop's own conclusion: Ready only on rc 0 + a parsed payload + a true # studio_install_ok. The predicate is `!= Some(true)` (managed.rs:445), so an ABSENT - # field is Stale exactly like a false one; a CLI too old to answer is already - # rejected one check earlier on desktop_manageability_version. Leaving "absent" - # undecided reported HEALTHY on every booting leg and skipped the repair assertion - # this workflow exists to make -- the regression `unsloth_cli/commands/studio.py` - # sits in the path filter to catch, so it must never be what silences it. + # field is Stale exactly like a false one; a CLI too old to answer is already rejected + # one check earlier on desktop_manageability_version. Leaving "absent" undecided + # reported HEALTHY on every booting leg and skipped the repair assertion -- the very + # regression `unsloth_cli/commands/studio.py` sits in the path filter to catch. caps_ready = caps_rc == 0 and install_ok is True say("desktop_would_call_install_ok", caps_ready) # ── the deeper probes the fix PRs add ──────────────────────────────────── # RECORDED, but NOT repair evidence: preflight runs only `-h` and - # `studio desktop-capabilities --json` (managed.rs:357, :445) and never these two, so - # counting them would let a leg pass while the real app still reports ManagedReady - # over a torn install -- the false negative this workflow exists to catch. + # `studio desktop-capabilities --json` (managed.rs:357, :445), so counting these would + # let a leg pass while the real app still reports ManagedReady over a torn install. for label, args in ( ("verify_install", ["studio", "verify-install"]), ("desktop_runtime_check", ["studio", "desktop-runtime-check"]), @@ -172,16 +168,14 @@ def main(argv: list[str]) -> int: say(label, "ok" if r[0] == 0 else "failed") # The in-progress marker #7490 writes before spawning the installer. RECORDED ONLY: - # both interrupt drivers seed it and deliberately never clear it, so it is true on - # every leg by construction, and using it in the verdict below would make REPAIRABLE - # unconditional and FALSE_READY -- the one outcome this catches -- unreachable. + # both drivers seed it and never clear it, so it is true on every leg by construction, + # and using it in the verdict would make FALSE_READY, the one outcome, unreachable. home = Path(os.environ.get("UNSLOTH_STUDIO_HOME") or (Path.home() / ".unsloth" / "studio")) say("install_in_progress_marker", (home / ".desktop-install-in-progress").exists()) # ── ground truth: does the backend actually boot? ──────────────────────── - # Own the whole process tree: the CLI spawns uvicorn/python children that would keep - # holding the port and hang the next leg's probe. Same reason the driver kills the - # group. + # Own the whole process tree: the CLI spawns uvicorn/python children that would hold + # the port and hang the next leg's probe. Same reason the driver kills the group. popen_kw: dict = {} if os.name == "posix": popen_kw["start_new_session"] = True @@ -189,14 +183,13 @@ def main(argv: list[str]) -> int: popen_kw["creationflags"] = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) # Straight to the artefact file, never a PIPE: nothing drains a pipe until after the # polling loop, so a backend whose imports outrun the OS buffer (64 KiB on Linux and - # macOS, one page on Windows) blocks on write BEFORE binding the port, and backend_ok - # -- what this verdict pivots on -- would be false for a perfectly good install. + # macOS, one page on Windows) blocks BEFORE binding the port and backend_ok, what this + # verdict pivots on, would be false for a perfectly good install. blog_path = out / "backend.log" blog_fh = blog_path.open("w", encoding = "utf-8", errors = "replace") # An interrupted install can leave the console script in place with its venv - # interpreter gone. run() catches that as OSError, but an unguarded spawn here would - # raise, so no verdict.json is written and both workflows die on the json.load rather - # than reporting. An unlaunchable CLI is a broken backend `-h` already flags. + # interpreter gone. An unguarded spawn would raise, so no verdict.json is written and + # both workflows die on json.load. An unlaunchable CLI is a broken backend `-h` flags. proc = None try: proc = subprocess.Popen( @@ -232,8 +225,8 @@ def main(argv: list[str]) -> int: import signal # start_new_session made this child its own group leader. Read the pgid - # BEFORE the reap: once the leader is waited on, os.getpgid() raises and the - # escalation would target nothing. + # BEFORE the reap: once waited on, os.getpgid() raises and escalation targets + # nothing. try: pgid = os.getpgid(proc.pid) except OSError: @@ -250,9 +243,9 @@ def main(argv: list[str]) -> int: continue # Unconditional, and to the GROUP -- the same escalation # interrupt-install.sh:94 makes. The leader exits promptly on SIGTERM while a - # uvicorn worker does not, so returning as soon as proc.wait() succeeded left - # that worker holding the port and the venv open while the repair step - # reinstalled underneath it. Signalling an empty group is a no-op. + # uvicorn worker does not, so returning once proc.wait() succeeded left that + # worker holding the port and venv while the repair reinstalled underneath. + # Signalling an empty group is a no-op. try: os.killpg(pgid, signal.SIGKILL) except OSError: @@ -260,9 +253,8 @@ def main(argv: list[str]) -> int: else: # On win32 the CLI re-spawns the server as a CHILD and waits on it # (unsloth_cli/commands/studio.py:1543), and CREATE_NEW_PROCESS_GROUP does not - # make terminate() reach descendants, so killing the wrapper alone leaves a - # server holding the venv open and the repair reinstalls into locked files. - # taskkill /T takes the tree. + # make terminate() reach descendants, so killing the wrapper alone leaves the + # venv locked against the repair. taskkill /T takes the tree. run(["taskkill", "/F", "/T", "/PID", str(proc.pid)], timeout = 30) try: proc.wait(timeout = 10) @@ -289,13 +281,10 @@ def main(argv: list[str]) -> int: # A booting backend is not enough. The manifest is written LAST # (install_python_stack.py:3255), so the data-designer leg boots while # desktop-capabilities still says studio_install_ok=false and preflight reports Stale - # (managed.rs:445). Calling that HEALTHY skipped the re-run step, leaving the leg - # asserting nothing beyond a marker appearing. - # - # `-h` gates it for the same reason: probe_managed_bin runs it FIRST and returns - # Stale "cli_unusable" without reaching the capability probe (managed.rs:465-478), so - # consulting cli_h_ok only in the repairable arm called a CLI that cannot print help - # HEALTHY whenever the backend booted. + # (managed.rs:445); calling that HEALTHY skipped the re-run step. + # `-h` gates it for the same reason: probe_managed_bin runs it FIRST and returns Stale + # "cli_unusable" without reaching the capability probe (managed.rs:465-478), so + # consulting cli_h_ok only in the repairable arm called a help-less CLI HEALTHY. if backend_ok and caps_ready and facts.get("cli_h_ok"): verdict = "HEALTHY" elif not caps_ready or not facts.get("cli_h_ok"): diff --git a/.github/workflows/interrupted-install-ci.yml b/.github/workflows/interrupted-install-ci.yml index a09d44ebe7..50bfc75ddf 100644 --- a/.github/workflows/interrupted-install-ci.yml +++ b/.github/workflows/interrupted-install-ci.yml @@ -4,16 +4,15 @@ # Proves an INTERRUPTED install can never masquerade as a healthy one. # # Reported failure: quitting the app mid-install kills the installer process group -# (main.rs cleanup_child_processes -> install.rs:798-807), landing mid "studio deps" -- -# the step installing studio/backend/requirements/studio.txt, where structlog is -# declared. On relaunch preflight probes `unsloth -h` and `studio desktop-capabilities -# --json`; both succeed because the CLI's own deps (typer/click/rich) are core, so the -# app reports ManagedReady with can_auto_repair=false while the backend dies on -# `import structlog` and the user is stuck on "Server stopped unexpectedly". +# (main.rs cleanup_child_processes -> install.rs:798-807) mid "studio deps", the step +# installing studio/backend/requirements/studio.txt where structlog is declared. On +# relaunch preflight probes `unsloth -h` and `studio desktop-capabilities --json`; both +# succeed because the CLI's own deps (typer/click/rich) are core, so the app reports +# ManagedReady with can_auto_repair=false while the backend dies on `import structlog` +# and the user is stuck on "Server stopped unexpectedly". # # No CI job had ever interrupted an install. This one kills the installer at each phase -# and asserts the result is genuinely healthy or explicitly repairable, never silently -# ready. +# and asserts the result is genuinely healthy or explicitly repairable, never silently ready. name: Interrupted install recovery @@ -29,24 +28,21 @@ on: - 'studio/src-tauri/src/preflight.rs' - 'studio/src-tauri/src/preflight/**' - 'unsloth_cli/commands/studio.py' - # Every leg installs the checkout with `--local`, so this file decides the - # `unsloth` console script and the core deps the probe leans on: `-h` and - # `desktop-capabilities` only survive a torn install because typer/click/rich - # are declared here. Moving one to an extra changes what every interrupted - # venv looks like, and no other install workflow interrupts the installer. + # Every leg installs the checkout with `--local`, so this file decides the console + # script and the core deps: `-h` and `desktop-capabilities` only survive a torn + # install because typer/click/rich are declared here, not in an extra. No other + # install workflow interrupts the installer. - 'pyproject.toml' - # studio_install_ok and verify-install, the two decisions the probe asserts - # on, live here rather than in commands/studio.py, so a change making - # install_state() accept a missing manifest would otherwise merge unrun. + # studio_install_ok and verify-install, the decisions the probe asserts on, live + # here, so an install_state() that accepts a missing manifest would merge unrun. - 'unsloth_cli/_studio_deps.py' - 'studio/install_manifest.py' - # The requirement files are the phases: studio.txt declares structlog, whose - # absence IS the reported false-ready bug, so moving a package between them - # changes what every interrupted state looks like. + # The requirement files are the phases: studio.txt declares structlog, whose absence + # IS the reported false-ready bug, so moving a package between them changes what + # every interrupted state looks like. - 'studio/backend/requirements/**' - # `*` never matches `/` and a literal `-install` follows, so - # `interrupt*-install*` would match the .sh / .ps1 but NOT the underscored - # probe. List all three explicitly rather than rely on a glob. + # `interrupt*-install*` would match the .sh / .ps1 but NOT the underscored probe + # (`*` never matches `/`, and a literal `-install` follows), so list all three. - '.github/scripts/interrupt-install.sh' - '.github/scripts/interrupt-install.ps1' - '.github/scripts/interrupted_install_probe.py' @@ -77,21 +73,18 @@ jobs: # The exact reported case: killed during the step that installs structlog. - {os: macos-14, label: studio-deps, marker: 'studio deps', experimental: false} # Coarse phases, earliest to latest -- each leaves a different partial venv. - # venv is experimental because the step is not reliably interruptible: with a - # warm uv cache creating it takes ~0.1s (staging run 30419729244 logged - # 03:31:07.371 -> 07.478 between its step line and "Installing PyTorch"), less - # than the log poll, so the kill often lands in the next step. The landing check - # now FAILS that instead of warning, and this leg must not block the PR on a race - # no log poll can win. It still probes the earliest torn state whenever it lands. + # venv is experimental: with a warm uv cache the step takes ~0.1s (staging run + # 30419729244 logged 03:31:07.371 -> 07.478 between its step line and "Installing + # PyTorch"), under the log poll, so the kill often lands in the next step and the + # landing check FAILS it. It still probes the earliest torn state when it lands. - {os: macos-14, label: venv, marker: '\[TAURI:STEP\] Creating virtual environment', experimental: true} - {os: macos-14, label: torch, marker: '\[TAURI:STEP\] Installing PyTorch', experimental: false} - {os: macos-14, label: unsloth, marker: '\[TAURI:STEP\] Installing Unsloth', experimental: false} - {os: macos-14, label: setup, marker: '\[TAURI:STEP\] Running Unsloth setup', experimental: false} # Other dependency-pass steps around the named one. - {os: macos-14, label: pip-bootstrap, marker: 'pip bootstrap', experimental: false} - # No base-packages cell: --local sets skip_base, so install_python_stack - # returns before any "base packages" label prints and the kill can never - # land. That leg ran to completion instead, proving nothing. + # No base-packages cell: --local sets skip_base, so install_python_stack returns + # before "base packages" ever prints -- that leg ran to completion, proving nothing. - {os: macos-14, label: unsloth-extras, marker: 'unsloth extras', experimental: true} - {os: macos-14, label: data-designer, marker: 'data designer deps', experimental: true} # Linux: same teardown path, different package manager and process semantics. @@ -116,11 +109,10 @@ jobs: run: | # --local is load-bearing. Without it install.sh:3996 resolves # `unsloth>=2026.7.5` from PyPI, so the venv gets the PUBLISHED CLI, every - # probe of `studio verify-install` / `desktop-runtime-check` reports "absent" - # whatever the branch does, and the lane cannot observe the fix it tests. - # --local overlays the checkout editable (install.sh:3990) before `studio - # setup` runs the dep pass, so a kill at "studio deps" leaves the branch's - # CLI installed. + # `verify-install` / `desktop-runtime-check` probe reports "absent" whatever the + # branch does, and the lane cannot observe the fix it tests. --local overlays the + # checkout editable (install.sh:3990) before the dep pass, so a kill at "studio + # deps" leaves the branch's CLI installed. bash .github/scripts/interrupt-install.sh \ '${{ matrix.marker }}' logs/install.log -- --tauri --local @@ -136,23 +128,20 @@ jobs: tail -30 logs/install.log || true exit 1 fi - # reason alone is not proof: the driver re-checks liveness after the marker - # delay, but the installer can still finish in the window between that check - # and the signal, leaving reason=marker-hit over a COMPLETED install. The probe - # then reads HEALTHY and the re-run assertion below is skipped, so the leg goes - # green having interrupted nothing. The exit status separates the two: SIGTERM - # takes install.sh's trap to 143 (install.sh:716) and SIGKILL to 137, while only - # an install that ran to completion exits 0. + # reason alone is not proof: the installer can finish between the driver's + # post-delay liveness check and the signal, leaving reason=marker-hit over a + # COMPLETED install that probes HEALTHY and skips the re-run assertion below. The + # exit status separates them: SIGTERM takes install.sh's trap to 143 + # (install.sh:716) and SIGKILL to 137, while only a completed install exits 0. if [ "$installer_exit" = "0" ]; then echo "::error::installer exited 0 -- it COMPLETED inside the kill window, so" echo "::error::nothing was interrupted and this leg asserts nothing." tail -30 logs/install.log || true exit 1 fi - # ...and the signal has to have landed in the step this leg is named after. A - # step whose line is no longer the last one was already over when the poll saw - # it, so the kill hit the NEXT step and the leg silently duplicates another one - # while its label claims otherwise. A warning here proves nothing, so it fails. + # ...and the signal must land in the step this leg is named after. A step whose + # line is no longer the last was already over when the poll saw it, so the kill + # hit the NEXT step and the leg duplicates another one under a false label. # Sub-step markers ("studio deps") print no [TAURI:STEP] line and are exempt. if [ "$interrupt_step_mismatch" = "true" ]; then echo "::error::the signal landed after '${{ matrix.marker }}' finished, so this" @@ -168,8 +157,7 @@ jobs: BIN="$HOME/.unsloth/studio/unsloth_studio/bin/unsloth" [ -x "$BIN" ] || BIN="$HOME/.unsloth/studio/bin/unsloth" if [ ! -x "$BIN" ]; then - # No CLI at all is SAFE: preflight reports NotInstalled and the app offers - # a normal install. + # No CLI at all is SAFE: preflight reports NotInstalled, the app reinstalls. echo "verdict=NO_CLI" >> "$GITHUB_OUTPUT" echo "[probe] no unsloth CLI installed -> preflight reports NotInstalled (safe)" exit 0 @@ -182,15 +170,12 @@ jobs: - name: A re-run must repair, not short-circuit # NO_CLI included: a kill at venv or torch lands before "Installing Unsloth" - # (install.sh:2125 / :3667 / :3961), so those legs always take NO_CLI and - # skipping the re-run left three non-experimental legs asserting only that a - # marker appeared. The bug's second half is `install.sh` seeing a "current" - # version and no-opping over a broken venv. - # - # HEALTHY needs the backend booting AND the install reported complete, so the - # data-designer leg (killed after "studio deps" but before the manifest is - # written last, install_python_stack.py:3255) arrives here rather than skipping - # the one assertion that matters for it. + # (install.sh:2125 / :3667 / :3961), so those legs always take NO_CLI and skipping + # the re-run left three non-experimental legs asserting only that a marker + # appeared. The bug's second half is `install.sh` seeing a "current" version and + # no-opping over a broken venv. HEALTHY also needs the install reported complete, + # so the data-designer leg (killed before the manifest is written last, + # install_python_stack.py:3255) arrives here instead of skipping that assertion. if: always() && steps.probe.outputs.verdict != 'HEALTHY' run: | set -o pipefail @@ -199,8 +184,7 @@ jobs: echo "repair exit: $rc" BIN="$HOME/.unsloth/studio/unsloth_studio/bin/unsloth" [ -x "$BIN" ] || BIN="$HOME/.unsloth/studio/bin/unsloth" - # The probe writes no verdict.json when the bin is missing, so check here or - # the json.load below crashes instead of reporting. + # No verdict.json when the bin is missing, so check here or json.load crashes. if [ ! -x "$BIN" ]; then echo "::error::after a full re-run there is still no unsloth CLI at $BIN" tail -30 logs/repair.log || true @@ -208,9 +192,8 @@ jobs: fi python3 .github/scripts/interrupted_install_probe.py "$BIN" --out probe-after || true v="$(python3 -c "import json;print(json.load(open('probe-after/verdict.json'))['verdict'])")" - # A booting backend IS the repair, whatever the log narrated. Judging by log - # text failed a leg whose venv was fine: the only match was the frontend - # build printing "up to date". + # A booting backend IS the repair, whatever the log narrated: judging by log + # text failed a leg whose venv was fine, matching only the frontend's "up to date". if [ "$v" = "HEALTHY" ]; then echo "re-run repaired the install (verdict=HEALTHY)" exit 0 @@ -236,22 +219,20 @@ jobs: # ── Windows: no process groups, so the kill path differs ────────────────── interrupt-windows: name: windows kill@${{ matrix.label }} - # No UNSLOTH_STUDIO_HOME here. The app scrubs it before invoking the installer as - # `--tauri [--local]` (install.rs:202, :356) and install.ps1:189-215 rejects a custom - # root under --tauri, so a workspace-scoped root forced these legs down the - # shell-install path: UNSLOTH_TAURI_MODE=0, frontend build on, different root - # resolution, no bundled-file overlay. Worse, "Installing PyTorch" is only printed by - # Write-TauriLog (install.ps1:2440), so the torch leg's marker could never appear. - # The runner is ephemeral, so the default root is safe to install into. + # No UNSLOTH_STUDIO_HOME here. The app scrubs it (install.rs:202, :356) and + # install.ps1:189-215 rejects a custom root under --tauri, so a workspace-scoped root + # forced these legs down the shell-install path: UNSLOTH_TAURI_MODE=0, frontend build + # on, different root resolution, no bundled-file overlay. Worse, "Installing PyTorch" + # is only printed by Write-TauriLog (install.ps1:2440), so the torch leg's marker could + # never appear. The runner is ephemeral, so the default root is safe to install into. runs-on: windows-latest timeout-minutes: 60 strategy: fail-fast: false matrix: include: - # install.ps1:121 parses `--no-torch`; `-SkipTorch` matches no case there and - # is silently dropped. The torch leg must NOT skip torch, or its marker never - # appears. + # install.ps1:121 parses `--no-torch`; `-SkipTorch` matches no case there and is + # silently dropped. The torch leg must NOT skip torch or its marker never appears. - {label: studio-deps, marker: 'studio deps', installArgs: '--tauri --no-torch --local'} - {label: torch, marker: 'Installing PyTorch', installArgs: '--tauri --local'} @@ -283,20 +264,18 @@ jobs: Get-Content logs/install.log -Tail 30 -ErrorAction SilentlyContinue exit 1 } - # Same guard as the POSIX leg: the driver re-checks HasExited after the marker - # delay, but the installer can still finish between that check and Stop-Tree, - # leaving reason=marker-hit over a COMPLETED install that then probes HEALTHY - # and skips the re-run assertion. Stop-Process -Force terminates with a non-zero - # code, so only an install that ran to completion reports 0. + # Same guard as the POSIX leg: the installer can finish between the driver's + # post-delay HasExited check and Stop-Tree, leaving reason=marker-hit over a + # COMPLETED install that probes HEALTHY and skips the re-run assertion. + # Stop-Process -Force is non-zero, so only a completed install reports 0. if ($vals['installer_exit'] -eq '0') { Write-Host '::error::installer exited 0 -- it COMPLETED inside the kill window, so' Write-Host '::error::nothing was interrupted and this leg asserts nothing.' Get-Content logs/install.log -Tail 30 -ErrorAction SilentlyContinue exit 1 } - # Same landing check as the POSIX leg: a step that was already over when the poll - # saw its line means the kill hit the NEXT step, so the leg duplicates another - # one while its label claims otherwise. + # Same landing check as the POSIX leg: a step already over when the poll saw its + # line means the kill hit the NEXT step, so the leg duplicates another one. if ($vals['interrupt_step_mismatch'] -eq 'true') { Write-Host "::error::the signal landed after '${{ matrix.marker }}' finished, so this" Write-Host '::error::leg interrupted a later step than the one it is named for.' @@ -315,10 +294,10 @@ jobs: Write-Host '[probe] no unsloth CLI -> preflight reports NotInstalled (safe)' exit 0 } - # The SAME probe the other platforms run. The bespoke inline version this - # replaced only checked `-h` and `desktop-capabilities`, so it could not - # observe studio_install_ok / verify-install / desktop-runtime-check and - # would have failed the very PRs that add them. + # The SAME probe the other platforms run. The bespoke inline version it replaced + # checked only `-h` and `desktop-capabilities`, so it could not observe + # studio_install_ok / verify-install / desktop-runtime-check and would have + # failed the very PRs that add them. python .github/scripts/interrupted_install_probe.py $bin --out probe $rc = $LASTEXITCODE $v = (Get-Content probe/verdict.json -Raw | ConvertFrom-Json).verdict @@ -328,8 +307,7 @@ jobs: - name: A re-run must repair, not short-circuit # Same assertion the POSIX legs make, NO_CLI included: without it a Windows leg # proves only that the break was DETECTED, never that install.ps1's version fast - # path does not short-circuit over it -- the half of the bug that strands the - # user. + # path does not short-circuit over it, the half of the bug that strands the user. if: always() && steps.probe.outputs.verdict != 'HEALTHY' shell: pwsh run: | @@ -347,9 +325,8 @@ jobs: } python .github/scripts/interrupted_install_probe.py $bin --out probe-after $v = (Get-Content probe-after/verdict.json -Raw | ConvertFrom-Json).verdict - # A booting backend IS the repair, whatever the log narrated. Judging by log - # text failed a POSIX leg whose venv was fine: the only match was the frontend - # build printing "up to date". + # A booting backend IS the repair, whatever the log narrated: judging by log text + # failed a POSIX leg whose venv was fine, matching only the frontend's "up to date". if ($v -eq 'HEALTHY') { Write-Host 're-run repaired the install (verdict=HEALTHY)' exit 0 From 1da8e972c3d902207e86751556e9309a101db68d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 05:10:39 +0000 Subject: [PATCH 21/25] Land the kill in the phase each leg is named for Splitting the log on \r shows that 5 of the 12 legs of staging run 30419729244 interrupted a later phase than their label claims, and the run was fully green. The venv leg's install.log is byte-identical to the torch leg's. So is pip-bootstrap's to unsloth-extras'. Worse, both "studio deps" legs, the cells that reproduce the reported bug, were killed at "7/10 data designer deps" and "12/14 local plugin": their own probe artefacts report backend_ok=true, so structlog was installed and the flagship cell was passing on the manifest gate alone. Two causes. The dependency pass rewrites ONE physical line with \r (install_python_stack.py:2499), so its sub-steps are CR-separated segments and a line-based check could not see one end; the drivers exempted them and warned about nothing. And the flat 3s beat between the marker and the signal is longer than several phases, while every phase label prints BEFORE its work starts, so the beat pushed the signal past the phase instead of into it. Both drivers now split on \r, track the running phase at both levels, and judge a sub-step marker against the running sub-step and a step marker against the running step, so a step is not "over" because the sub-steps beneath it advanced. The beat defaults to 0 and is set per leg, 3s only for torch, unsloth and setup, which run for minutes. The mismatch is recorded in interrupt.env and the landing assertion fails on it, in both languages. Both detectors were replayed against the 12 real logs from 30419729244 and agree with the artefacts on every leg. venv and pip-bootstrap become experimental: their phases are shorter than any log poll can resolve. --- .github/scripts/interrupt-install.ps1 | 81 ++++++++++-------- .github/scripts/interrupt-install.sh | 89 ++++++++++++-------- .github/workflows/interrupted-install-ci.yml | 80 ++++++++++-------- 3 files changed, 149 insertions(+), 101 deletions(-) diff --git a/.github/scripts/interrupt-install.ps1 b/.github/scripts/interrupt-install.ps1 index 05e3dc237c..f5f5eb7bc4 100644 --- a/.github/scripts/interrupt-install.ps1 +++ b/.github/scripts/interrupt-install.ps1 @@ -16,7 +16,7 @@ param( [string]$LogPath = 'logs/install.log', [string]$InstallArgs = '', [int]$KillAtSeconds = 900, - [int]$KillAfterMarkerSeconds = 3 + [int]$KillAfterMarkerSeconds = 0 ) $ErrorActionPreference = 'Continue' @@ -62,45 +62,58 @@ function Stop-Tree([int]$RootId) { catch { } } -function Get-StepCount([string]$Path) { - @(Select-String -Path $Path -Pattern '^\[TAURI:STEP\]' -ErrorAction SilentlyContinue).Count +# A leg can be aimed at either kind of phase the installer prints, and only one of them is +# a line. install.ps1 prints "[TAURI:STEP] " lines, while the dependency pass rewrites +# ONE physical line with \r (install_python_stack.py:2499), so its sub-steps are +# CR-separated SEGMENTS. Splitting on \r is what makes a sub-step's END observable at all. +$SubRe = '\[[=-]+\]\s*\d+/\d+\s' + +function Get-PhaseLines([string]$Path) { + $raw = Get-Content -Path $Path -Raw -ErrorAction SilentlyContinue + if (-not $raw) { return @() } + return @(($raw -replace "`r", "`n") -split "`n") } -function Get-LastStep([string]$Path) { - $steps = @(Select-String -Path $Path -Pattern '^\[TAURI:STEP\]' -ErrorAction SilentlyContinue) - if ($steps.Count) { return $steps[-1].Line } +function Get-LastPhase([string]$Path) { + $p = @(Get-PhaseLines $Path | Where-Object { $_ -match '^\[TAURI:STEP\]' -or $_ -match $SubRe }) + if ($p.Count) { return $p[-1] } return '' } -# True when the marker names a [TAURI:STEP] line that is no longer the last one: the step -# ended before the poll noticed it. Sub-step markers ('studio deps') print no step line of -# their own, so they are never judged here. -function Test-MarkedStepOver { +# True when the phase the marker named is no longer the running one. A sub-step marker is +# judged against the running sub-step, a step marker against the running step -- a step is +# not "over" because the sub-steps beneath it advanced. +function Test-MarkedPhaseOver { if (-not $Marker) { return $false } - $steps = @(Select-String -Path $LogPath -Pattern '^\[TAURI:STEP\]' -ErrorAction SilentlyContinue) - if (-not ($steps | Where-Object { $_.Line -match $Marker })) { return $false } - return ((Get-LastStep $LogPath) -notmatch $Marker) + $lines = @(Get-PhaseLines $LogPath) + $subs = @($lines | Where-Object { $_ -match $SubRe }) + if ($subs | Where-Object { $_ -match $Marker }) { + $last = Get-LastPhase $LogPath + return -not ($last -match $SubRe -and $last -match $Marker) + } + $steps = @($lines | Where-Object { $_ -match '^\[TAURI:STEP\]' }) + if ($steps | Where-Object { $_ -match $Marker }) { + return ($steps[-1] -notmatch $Marker) + } + return $false } $killed = $false $reason = '' -# Half-second slices: a sub-second step is over before a 1s poll sees its line. -for ($i = 0; $i -lt ($KillAtSeconds * 2); $i++) { +# Fifth-of-a-second slices: every phase label prints BEFORE its work starts, so the poll +# delay is the whole distance between the label and the signal. +for ($i = 0; $i -lt ($KillAtSeconds * 5); $i++) { if ($proc.HasExited) { $reason = 'exited-before-marker'; break } if ($Marker) { $hit = Select-String -Path $LogPath -Pattern $Marker -SimpleMatch:$false -ErrorAction SilentlyContinue if ($hit) { - # Same beat as the POSIX driver, in slices and cut short once a later [TAURI:STEP] - # line appears, so a fast step does not send the signal into the step after it. - # Skipped when the step is ALREADY over: the cut-short cannot help once the next - # line is logged, and beating on would push the signal deeper into the next step. - if (-not (Test-MarkedStepOver)) { - $stepsAtMarker = Get-StepCount $LogPath - for ($j = 0; $j -lt ($KillAfterMarkerSeconds * 5); $j++) { - Start-Sleep -Milliseconds 200 - if ($proc.HasExited) { break } - if ((Get-StepCount $LogPath) -ne $stepsAtMarker) { break } - } + # Same as the POSIX driver: no beat by default, because the label prints before the + # work, so killing at detection is already inside the phase while a flat beat sends + # the signal into a LATER phase. The loop stops the moment the marked phase ends. + for ($j = 0; $j -lt ($KillAfterMarkerSeconds * 5); $j++) { + if (Test-MarkedPhaseOver) { break } + Start-Sleep -Milliseconds 200 + if ($proc.HasExited) { break } } # The installer can finish inside the delay; recording marker-hit before it # let a COMPLETED install satisfy the landing assertion and probe HEALTHY. @@ -146,14 +159,14 @@ Get-Content $LogPath -Tail 15 -ErrorAction SilentlyContinue if ($Marker -and -not (Select-String -Path $LogPath -Pattern $Marker -ErrorAction SilentlyContinue)) { Write-Host "::warning::marker '$Marker' never appeared -- killed at the deadline, not the intended step" } -# Where the signal actually landed. A sub-second step can end before any poll sees its -# line, and the leg then kills the NEXT step while its label claims otherwise. Sub-step -# markers print no [TAURI:STEP] line, so the test skips them instead of always warning. -$lastStep = Get-LastStep $LogPath -Write-Host "[interrupt] step at kill: $lastStep" -$mismatch = Test-MarkedStepOver +# Where the signal actually landed. A phase that ended before the poll saw the marker sends +# the kill into a LATER phase, and the leg then duplicates whichever leg owns that phase +# while its own label claims otherwise. +$lastPhase = Get-LastPhase $LogPath +Write-Host "[interrupt] phase at kill: $lastPhase" +$mismatch = Test-MarkedPhaseOver if ($mismatch) { - Write-Host "::warning::killed in '$lastStep', not the marked step -- that step was already over" + Write-Host "::warning::killed in '$lastPhase', not the marked phase -- that phase was already over" } # Lower-cased so the workflow can compare it the same way on every platform, and only # simple values: the POSIX side sources this file. @@ -161,6 +174,6 @@ if ($mismatch) { "interrupt_reason=$reason" "interrupt_killed=$killed" "installer_exit=$rc" - "interrupt_step_mismatch=$(if ($mismatch) { 'true' } else { 'false' })" + "interrupt_phase_mismatch=$(if ($mismatch) { 'true' } else { 'false' })" ) | Set-Content -Path (Join-Path (Split-Path -Parent $LogPath) 'interrupt.env') -Encoding utf8 exit 0 diff --git a/.github/scripts/interrupt-install.sh b/.github/scripts/interrupt-install.sh index 5a9b052ce5..1fd255b037 100755 --- a/.github/scripts/interrupt-install.sh +++ b/.github/scripts/interrupt-install.sh @@ -9,7 +9,8 @@ # # Usage: bash .github/scripts/interrupt-install.sh "" "" [-- install args] # log regex to wait for before killing, e.g. "studio deps"; "" kills at deadline. -# Env: KILL_AT_SECONDS deadline (default 900), KILL_GRACE grace before SIGKILL (default 10) +# Env: KILL_AT_SECONDS deadline (default 900), KILL_GRACE grace before SIGKILL (default 10), +# KILL_AFTER_MARKER_SECONDS beat between the marker and the signal (default 0) set -uo pipefail MARKER="${1:-}" @@ -39,39 +40,59 @@ PID=$! set +m echo "[interrupt] installer pid/pgid=$PID marker='${MARKER}' deadline=${KILL_AT_SECONDS}s" -# True when the marker names a [TAURI:STEP] line that is no longer the last one: the step -# ended before the poll noticed it. Sub-step markers ("studio deps") print no step line of -# their own, so they are never judged here. -marked_step_over() { +# A leg can be aimed at either kind of phase the installer prints, and only one of them is +# a line. install.sh prints "[TAURI:STEP] " lines, while the dependency pass rewrites +# ONE physical line with \r (install_python_stack.py:2499), so its ten sub-steps are +# CR-separated SEGMENTS. Splitting on \r is what makes a sub-step's END observable at all: +# without it the "studio deps" leg of staging run 30419729244 killed at "7/10 data designer +# deps" with backend_ok=true, having installed the structlog it exists to remove. +SUB_RE='\[[=-]+\][[:space:]]*[0-9]+/[0-9]+[[:space:]]' +phase_lines() { tr '\r' '\n' < "$LOG" 2>/dev/null || true; } + +# True when the phase the marker named is no longer the running one. A sub-step marker is +# judged against the running sub-step, a step marker against the running step -- a step is +# not "over" because the sub-steps beneath it advanced. A marker naming neither is not +# judged. Results go through variables, never a `| grep -q`, which can report SIGPIPE +# through pipefail on a long log. +marked_phase_over() { [ -n "$MARKER" ] || return 1 - grep -E '^\[TAURI:STEP\]' "$LOG" 2>/dev/null | grep -qE "$MARKER" || return 1 - grep -E '^\[TAURI:STEP\]' "$LOG" 2>/dev/null | tail -1 | grep -qE "$MARKER" && return 1 - return 0 + local lines steps subs last + lines="$(phase_lines)" + steps="$(printf '%s\n' "$lines" | grep -aE '^\[TAURI:STEP\]')" || true + subs="$(printf '%s\n' "$lines" | grep -aE "$SUB_RE")" || true + if [ -n "$subs" ] && [[ $subs =~ $MARKER ]]; then + last="$(printf '%s\n' "$lines" | grep -aE "^\[TAURI:STEP\]|$SUB_RE" | tail -1)" || true + [[ $last =~ $SUB_RE && $last =~ $MARKER ]] && return 1 + return 0 + fi + if [ -n "$steps" ] && [[ $steps =~ $MARKER ]]; then + last="$(printf '%s\n' "$steps" | tail -1)" + [[ $last =~ $MARKER ]] && return 1 + return 0 + fi + return 1 } killed=false reason="" -# Half-second slices: a sub-second step is over before a 1s poll sees its line. -for i in $(seq 1 $(( KILL_AT_SECONDS * 2 ))); do +# Fifth-of-a-second slices: every phase label prints BEFORE its work starts, so the poll +# delay is the whole distance between the label and the signal. +for i in $(seq 1 $(( KILL_AT_SECONDS * 5 ))); do if ! kill -0 "$PID" 2>/dev/null; then reason="exited-before-marker" break fi if [ -n "$MARKER" ] && grep -qE "$MARKER" "$LOG" 2>/dev/null; then - # A beat into the step so the kill lands mid-work, cut short the moment a later - # [TAURI:STEP] line appears: the venv takes ~0.1s, so a flat 3s sleep put the venv - # leg's signal in "Installing PyTorch", a duplicate of the torch leg. Sub-step markers - # ("studio deps") print no step line and keep the whole beat. Skipped when the step is - # ALREADY over: the cut-short cannot help once the next line is logged and beating on - # only pushes the signal deeper, so kill now and let the mismatch check below report. - if ! marked_step_over; then - _steps_at_marker="$(grep -cE '^\[TAURI:STEP\]' "$LOG" 2>/dev/null || true)" - for _ in $(seq 1 $(( ${KILL_AFTER_MARKER_SECONDS:-3} * 5 ))); do - sleep 0.2 - kill -0 "$PID" 2>/dev/null || break - [ "$(grep -cE '^\[TAURI:STEP\]' "$LOG" 2>/dev/null || true)" = "$_steps_at_marker" ] || break - done - fi + # No beat by default: the label prints before the work, so killing at detection is + # already inside the phase, while a flat 3s beat is what pushed 5 of the 12 legs in + # staging run 30419729244 into a LATER phase (venv -> torch produced a byte-identical + # log to the torch leg). Legs whose phase runs for minutes pass a beat explicitly to + # land mid-work; the loop still stops the moment the marked phase ends. + for _ in $(seq 1 $(( ${KILL_AFTER_MARKER_SECONDS:-0} * 5 ))); do + marked_phase_over && break + sleep 0.2 + kill -0 "$PID" 2>/dev/null || break + done # ...but a cached step can FINISH inside the beat. Recording marker-hit before it # handed the landing assertion a COMPLETED install that interrupted nothing. if ! kill -0 "$PID" 2>/dev/null; then @@ -82,7 +103,7 @@ for i in $(seq 1 $(( KILL_AT_SECONDS * 2 ))); do killed=true break fi - sleep 0.5 + sleep 0.2 done if [ "$killed" != "true" ] && kill -0 "$PID" 2>/dev/null; then @@ -127,21 +148,21 @@ tail -15 "$LOG" || true if [ -n "$MARKER" ] && ! grep -qE "$MARKER" "$LOG" 2>/dev/null; then echo "::warning::marker '$MARKER' never appeared -- this leg killed at the deadline, not at the intended step" fi -# Where the signal actually landed. A sub-second step can end before any poll sees its -# line, and the leg then kills the NEXT step while its label claims otherwise. Sub-step -# markers print no [TAURI:STEP] line, so the test skips them instead of always warning. -_last_step="$(grep -E '^\[TAURI:STEP\]' "$LOG" 2>/dev/null | tail -1)" -echo "[interrupt] step at kill: $_last_step" +# Where the signal actually landed. A phase that ended before the poll saw the marker sends +# the kill into a LATER phase, and the leg then duplicates whichever leg owns that phase +# while its own label claims otherwise. +_last_phase="$(phase_lines | grep -aE "^\[TAURI:STEP\]|$SUB_RE" | tail -1)" || true +echo "[interrupt] phase at kill: $_last_phase" mismatch=false -if marked_step_over; then +if marked_phase_over; then mismatch=true - echo "::warning::killed in '$_last_step', not the marked step -- that step was already over" + echo "::warning::killed in '$_last_phase', not the marked phase -- that phase was already over" fi -# Only simple values: the workflow sources this file, so the step text stays out of it. +# Only simple values: the workflow sources this file, so the phase text stays out of it. { echo "interrupt_reason=$reason" echo "interrupt_killed=$killed" echo "installer_exit=$rc" - echo "interrupt_step_mismatch=$mismatch" + echo "interrupt_phase_mismatch=$mismatch" } > "$(dirname "$LOG")/interrupt.env" exit 0 diff --git a/.github/workflows/interrupted-install-ci.yml b/.github/workflows/interrupted-install-ci.yml index 50bfc75ddf..4567c13b22 100644 --- a/.github/workflows/interrupted-install-ci.yml +++ b/.github/workflows/interrupted-install-ci.yml @@ -70,26 +70,35 @@ jobs: fail-fast: false matrix: include: - # The exact reported case: killed during the step that installs structlog. - - {os: macos-14, label: studio-deps, marker: 'studio deps', experimental: false} + # `beat` is the delay between the marker and the signal, in seconds. It is 0 + # everywhere except the three steps that provably run for minutes: every phase + # label prints BEFORE its work starts, so killing at detection is already inside + # the phase, while any beat longer than the phase lands the signal in the NEXT + # one. A flat 3s beat is what made 5 of the 12 legs of staging run 30419729244 + # interrupt a later phase than their label claims. + # + # The exact reported case: killed during the sub-step that installs structlog. + - {os: macos-14, label: studio-deps, marker: 'studio deps', beat: 0, experimental: false} # Coarse phases, earliest to latest -- each leaves a different partial venv. - # venv is experimental: with a warm uv cache the step takes ~0.1s (staging run - # 30419729244 logged 03:31:07.371 -> 07.478 between its step line and "Installing - # PyTorch"), under the log poll, so the kill often lands in the next step and the - # landing check FAILS it. It still probes the earliest torn state when it lands. - - {os: macos-14, label: venv, marker: '\[TAURI:STEP\] Creating virtual environment', experimental: true} - - {os: macos-14, label: torch, marker: '\[TAURI:STEP\] Installing PyTorch', experimental: false} - - {os: macos-14, label: unsloth, marker: '\[TAURI:STEP\] Installing Unsloth', experimental: false} - - {os: macos-14, label: setup, marker: '\[TAURI:STEP\] Running Unsloth setup', experimental: false} - # Other dependency-pass steps around the named one. - - {os: macos-14, label: pip-bootstrap, marker: 'pip bootstrap', experimental: false} + # venv is experimental: the step takes ~0.1s (30419729244 logged 03:31:07.371 -> + # 07.478 between its line and "Installing PyTorch"), shorter than any log poll, so + # the kill usually lands in the next phase and the landing check FAILS it. It + # still probes the earliest torn state when it does land. + - {os: macos-14, label: venv, marker: '\[TAURI:STEP\] Creating virtual environment', beat: 0, experimental: true} + - {os: macos-14, label: torch, marker: '\[TAURI:STEP\] Installing PyTorch', beat: 3, experimental: false} + - {os: macos-14, label: unsloth, marker: '\[TAURI:STEP\] Installing Unsloth', beat: 3, experimental: false} + - {os: macos-14, label: setup, marker: '\[TAURI:STEP\] Running Unsloth setup', beat: 3, experimental: false} + # Other dependency-pass sub-steps around the named one. pip-bootstrap is + # experimental for the same reason as venv: in 30419729244 it landed in "2/10 + # unsloth extras", producing a log byte-identical to the unsloth-extras leg. + - {os: macos-14, label: pip-bootstrap, marker: 'pip bootstrap', beat: 0, experimental: true} # No base-packages cell: --local sets skip_base, so install_python_stack returns # before "base packages" ever prints -- that leg ran to completion, proving nothing. - - {os: macos-14, label: unsloth-extras, marker: 'unsloth extras', experimental: true} - - {os: macos-14, label: data-designer, marker: 'data designer deps', experimental: true} + - {os: macos-14, label: unsloth-extras, marker: 'unsloth extras', beat: 0, experimental: true} + - {os: macos-14, label: data-designer, marker: 'data designer deps', beat: 0, experimental: true} # Linux: same teardown path, different package manager and process semantics. - - {os: ubuntu-latest, label: studio-deps, marker: 'studio deps', experimental: false} - - {os: ubuntu-latest, label: torch, marker: '\[TAURI:STEP\] Installing PyTorch', experimental: false} + - {os: ubuntu-latest, label: studio-deps, marker: 'studio deps', beat: 0, experimental: false} + - {os: ubuntu-latest, label: torch, marker: '\[TAURI:STEP\] Installing PyTorch', beat: 3, experimental: false} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -106,6 +115,7 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} KILL_AT_SECONDS: '1500' + KILL_AFTER_MARKER_SECONDS: '${{ matrix.beat }}' run: | # --local is load-bearing. Without it install.sh:3996 resolves # `unsloth>=2026.7.5` from PyPI, so the venv gets the PUBLISHED CLI, every @@ -139,14 +149,15 @@ jobs: tail -30 logs/install.log || true exit 1 fi - # ...and the signal must land in the step this leg is named after. A step whose - # line is no longer the last was already over when the poll saw it, so the kill - # hit the NEXT step and the leg duplicates another one under a false label. - # Sub-step markers ("studio deps") print no [TAURI:STEP] line and are exempt. - if [ "$interrupt_step_mismatch" = "true" ]; then - echo "::error::the signal landed after '${{ matrix.marker }}' finished, so this" - echo "::error::leg interrupted a later step than the one it is named for." - grep -E '^\[TAURI:STEP\]' logs/install.log || true + # ...and the signal must land in the phase this leg is NAMED for. Warning-only + # left staging run 30419729244 fully green with the venv leg's install.log + # byte-identical to the torch leg's, and both "studio deps" legs killed past + # structlog (their own probe artefacts report backend_ok=true), so the flagship + # cell never reproduced the bug it is named after. + if [ "$interrupt_phase_mismatch" = "true" ]; then + echo "::error::the kill landed in a LATER phase than '${{ matrix.marker }}', so this" + echo "::error::leg duplicates whichever leg owns that phase and its label lies." + tr '\r' '\n' < logs/install.log | grep -aE '^\[TAURI:STEP\]|\[[=-]+\] *[0-9]+/[0-9]+' || true exit 1 fi @@ -233,8 +244,9 @@ jobs: include: # install.ps1:121 parses `--no-torch`; `-SkipTorch` matches no case there and is # silently dropped. The torch leg must NOT skip torch or its marker never appears. - - {label: studio-deps, marker: 'studio deps', installArgs: '--tauri --no-torch --local'} - - {label: torch, marker: 'Installing PyTorch', installArgs: '--tauri --local'} + # `beat` as in the POSIX job: 0 for the sub-step, 3 for the minutes-long torch step. + - {label: studio-deps, marker: 'studio deps', beat: 0, installArgs: '--tauri --no-torch --local'} + - {label: torch, marker: 'Installing PyTorch', beat: 3, installArgs: '--tauri --local'} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -246,7 +258,8 @@ jobs: run: | pwsh -NoProfile -File .github/scripts/interrupt-install.ps1 ` -Marker '${{ matrix.marker }}' -LogPath logs/install.log ` - -InstallArgs '${{ matrix.installArgs }}' -KillAtSeconds 1500 + -InstallArgs '${{ matrix.installArgs }}' -KillAtSeconds 1500 ` + -KillAfterMarkerSeconds ${{ matrix.beat }} - name: The kill must have landed where it was aimed shell: pwsh @@ -274,12 +287,13 @@ jobs: Get-Content logs/install.log -Tail 30 -ErrorAction SilentlyContinue exit 1 } - # Same landing check as the POSIX leg: a step already over when the poll saw its - # line means the kill hit the NEXT step, so the leg duplicates another one. - if ($vals['interrupt_step_mismatch'] -eq 'true') { - Write-Host "::error::the signal landed after '${{ matrix.marker }}' finished, so this" - Write-Host '::error::leg interrupted a later step than the one it is named for.' - Select-String -Path logs/install.log -Pattern '^\[TAURI:STEP\]' -ErrorAction SilentlyContinue + # Same landing check as the POSIX leg: a phase already over when the poll saw the + # marker means the kill hit a LATER phase, so the leg duplicates another one. + if ($vals['interrupt_phase_mismatch'] -eq 'true') { + Write-Host "::error::the kill landed in a LATER phase than '${{ matrix.marker }}', so this" + Write-Host '::error::leg duplicates whichever leg owns that phase and its label lies.' + ((Get-Content logs/install.log -Raw) -replace "`r", "`n") -split "`n" | + Where-Object { $_ -match '^\[TAURI:STEP\]' -or $_ -match '\[[=-]+\]\s*\d+/\d+\s' } exit 1 } From 005f7a2b1b195387325c66da4da6d4c48beb4ec6 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 05:11:27 +0000 Subject: [PATCH 22/25] Fail the Windows leg when the installer never exited The driver writes installer_exit=running when the installer outlived Stop-Tree and WaitForExit, and 'running' is not '0', so the landing assertion accepted it. A live installer writing into the venv while the probe reads it is not an interrupted install. Only a real integer exit code counts now, checked against 0, 143, 137, -1, running and the empty string. --- .github/workflows/interrupted-install-ci.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/interrupted-install-ci.yml b/.github/workflows/interrupted-install-ci.yml index 4567c13b22..0ea4c03ab7 100644 --- a/.github/workflows/interrupted-install-ci.yml +++ b/.github/workflows/interrupted-install-ci.yml @@ -281,9 +281,12 @@ jobs: # post-delay HasExited check and Stop-Tree, leaving reason=marker-hit over a # COMPLETED install that probes HEALTHY and skips the re-run assertion. # Stop-Process -Force is non-zero, so only a completed install reports 0. - if ($vals['installer_exit'] -eq '0') { - Write-Host '::error::installer exited 0 -- it COMPLETED inside the kill window, so' - Write-Host '::error::nothing was interrupted and this leg asserts nothing.' + # 'running' too, not just 0: the driver reports that when the installer outlived + # Stop-Tree and WaitForExit, and a live installer writing into the venv under the + # probe is not an interrupted install either. Only a real non-zero code counts. + if ($vals['installer_exit'] -eq '0' -or $vals['installer_exit'] -notmatch '^-?\d+$') { + Write-Host "::error::installer exit=$($vals['installer_exit']) -- it completed or never" + Write-Host '::error::died inside the kill window, so this leg asserts nothing.' Get-Content logs/install.log -Tail 30 -ErrorAction SilentlyContinue exit 1 } From e5b40c11cf62bca466b9ac8d954b4931fd8c6750 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 05:47:09 +0000 Subject: [PATCH 23/25] Drop the legs that cannot land, and prove the kill was delivered Two cells never interrupted the phase they were named for. "Creating virtual environment" runs 0.107s (staging 30419729244, 03:31:07.371 -> 07.478) and "1/10 pip bootstrap" is over just as fast, both shorter than any poll that watches the log, so in 30423181897 and 30424366953 the signal landed in "Installing PyTorch" and "2/10 unsloth extras" every time. Each was another leg wearing a false label, so they are gone rather than allowed to fail, and continue-on-error goes with them: a leg permitted to fail asserts nothing. The only coverage lost is a venv caught half-written, which interruption cannot reach at this resolution; the torch leg's signal lands ~3s into a multi-minute download, so it already leaves a complete venv with nothing installed into it. The landing check also accepted an installer that failed on its own. A dependency error between the driver's last liveness check and the signal exits non-zero, which the exit != 0 guard let through as a kill. POSIX now requires 143 or 137, the only statuses a signal produces here and what all ten POSIX legs of 30424366953 reported. Recording whether kill(2) returned 0 would not separate them, since the unreaped leader keeps its group alive. Windows has no such status, so the driver records whether Stop-Process actually terminated the installer: it throws on a process already gone, so the flag is false exactly when there was nothing left to interrupt. --- .github/scripts/interrupt-install.ps1 | 18 ++++- .github/workflows/interrupted-install-ci.yml | 79 ++++++++++++++------ 2 files changed, 72 insertions(+), 25 deletions(-) diff --git a/.github/scripts/interrupt-install.ps1 b/.github/scripts/interrupt-install.ps1 index f5f5eb7bc4..26a7664201 100644 --- a/.github/scripts/interrupt-install.ps1 +++ b/.github/scripts/interrupt-install.ps1 @@ -53,13 +53,24 @@ $proc = Start-Process -FilePath 'powershell.exe' -ArgumentList $argList ` -PassThru -NoNewWindow Write-Host "[interrupt] installer pid=$($proc.Id) marker='$Marker' deadline=${KillAtSeconds}s" +# Proof that the signal was DELIVERED, not merely attempted. The installer can also fail +# on its own between the last HasExited check and Stop-Tree, and a natural failure carries +# a non-zero exit code just like a kill does, so the exit status alone cannot separate the +# two on Windows. Stop-Process throws on a process that has already gone, so this flag is +# false exactly when there was nothing left to interrupt. +$script:rootKilled = $false + function Stop-Tree([int]$RootId) { # Depth-first, so a parent cannot respawn a child we already killed. CIM gives the # parent link Windows does not expose via process groups. $kids = @(Get-CimInstance Win32_Process -Filter "ParentProcessId=$RootId" -ErrorAction SilentlyContinue) foreach ($k in $kids) { Stop-Tree ([int]$k.ProcessId) } - try { Stop-Process -Id $RootId -Force -ErrorAction Stop; Write-Host "[interrupt] killed pid=$RootId" } - catch { } + try { + Stop-Process -Id $RootId -Force -ErrorAction Stop + Write-Host "[interrupt] killed pid=$RootId" + if ($RootId -eq $proc.Id) { $script:rootKilled = $true } + } + catch { if ($RootId -eq $proc.Id) { Write-Host "[interrupt] installer pid=$RootId was already gone: $_" } } } # A leg can be aimed at either kind of phase the installer prints, and only one of them is @@ -152,7 +163,7 @@ if ($killed) { try { $proc.WaitForExit(30000) | Out-Null } catch { } $rc = if ($proc.HasExited) { $proc.ExitCode } else { 'running' } -Write-Host "[interrupt] installer exit=$rc reason=$reason killed=$killed" +Write-Host "[interrupt] installer exit=$rc reason=$reason killed=$killed root_killed=$($script:rootKilled)" Write-Host '[interrupt] last log lines:' Get-Content $LogPath -Tail 15 -ErrorAction SilentlyContinue @@ -173,6 +184,7 @@ if ($mismatch) { @( "interrupt_reason=$reason" "interrupt_killed=$killed" + "interrupt_root_killed=$(if ($script:rootKilled) { 'true' } else { 'false' })" "installer_exit=$rc" "interrupt_phase_mismatch=$(if ($mismatch) { 'true' } else { 'false' })" ) | Set-Content -Path (Join-Path (Split-Path -Parent $LogPath) 'interrupt.env') -Encoding utf8 diff --git a/.github/workflows/interrupted-install-ci.yml b/.github/workflows/interrupted-install-ci.yml index 0ea4c03ab7..df34bef1b8 100644 --- a/.github/workflows/interrupted-install-ci.yml +++ b/.github/workflows/interrupted-install-ci.yml @@ -65,7 +65,6 @@ jobs: name: ${{ matrix.os }} kill@${{ matrix.label }} runs-on: ${{ matrix.os }} timeout-minutes: 60 - continue-on-error: ${{ matrix.experimental }} strategy: fail-fast: false matrix: @@ -77,28 +76,35 @@ jobs: # one. A flat 3s beat is what made 5 of the 12 legs of staging run 30419729244 # interrupt a later phase than their label claims. # + # Every leg is a hard gate. There is no continue-on-error cell: a leg allowed to + # fail is a warning wearing a red icon, and this workflow's entire claim is that + # a killed install cannot report itself healthy. + # # The exact reported case: killed during the sub-step that installs structlog. - - {os: macos-14, label: studio-deps, marker: 'studio deps', beat: 0, experimental: false} + - {os: macos-14, label: studio-deps, marker: 'studio deps', beat: 0} # Coarse phases, earliest to latest -- each leaves a different partial venv. - # venv is experimental: the step takes ~0.1s (30419729244 logged 03:31:07.371 -> - # 07.478 between its line and "Installing PyTorch"), shorter than any log poll, so - # the kill usually lands in the next phase and the landing check FAILS it. It - # still probes the earliest torn state when it does land. - - {os: macos-14, label: venv, marker: '\[TAURI:STEP\] Creating virtual environment', beat: 0, experimental: true} - - {os: macos-14, label: torch, marker: '\[TAURI:STEP\] Installing PyTorch', beat: 3, experimental: false} - - {os: macos-14, label: unsloth, marker: '\[TAURI:STEP\] Installing Unsloth', beat: 3, experimental: false} - - {os: macos-14, label: setup, marker: '\[TAURI:STEP\] Running Unsloth setup', beat: 3, experimental: false} - # Other dependency-pass sub-steps around the named one. pip-bootstrap is - # experimental for the same reason as venv: in 30419729244 it landed in "2/10 - # unsloth extras", producing a log byte-identical to the unsloth-extras leg. - - {os: macos-14, label: pip-bootstrap, marker: 'pip bootstrap', beat: 0, experimental: true} + # No venv cell. "Creating virtual environment" ran 0.107s in staging run + # 30419729244 (03:31:07.371 -> 07.478 to "Installing PyTorch"), shorter than any + # poll that watches the log, so the kill landed in the NEXT phase every time it + # was tried (30423181897 and 30424366953 both). It was the torch leg with a + # different label. Lost with it: a venv caught half-written. That state is not + # reachable by interruption at this resolution, and it is the only thing lost -- + # the torch leg's signal lands ~3s into a multi-minute download, so what it + # leaves behind is already a complete venv with nothing installed into it. + - {os: macos-14, label: torch, marker: '\[TAURI:STEP\] Installing PyTorch', beat: 3} + - {os: macos-14, label: unsloth, marker: '\[TAURI:STEP\] Installing Unsloth', beat: 3} + - {os: macos-14, label: setup, marker: '\[TAURI:STEP\] Running Unsloth setup', beat: 3} + # Other dependency-pass sub-steps around the named one. No pip-bootstrap cell for + # the same reason as venv: "1/10 pip bootstrap" is over before a poll can see it, + # so in both 30419729244 and 30424366953 the signal landed in "2/10 unsloth + # extras", which is the next cell down. # No base-packages cell: --local sets skip_base, so install_python_stack returns # before "base packages" ever prints -- that leg ran to completion, proving nothing. - - {os: macos-14, label: unsloth-extras, marker: 'unsloth extras', beat: 0, experimental: true} - - {os: macos-14, label: data-designer, marker: 'data designer deps', beat: 0, experimental: true} + - {os: macos-14, label: unsloth-extras, marker: 'unsloth extras', beat: 0} + - {os: macos-14, label: data-designer, marker: 'data designer deps', beat: 0} # Linux: same teardown path, different package manager and process semantics. - - {os: ubuntu-latest, label: studio-deps, marker: 'studio deps', beat: 0, experimental: false} - - {os: ubuntu-latest, label: torch, marker: '\[TAURI:STEP\] Installing PyTorch', beat: 3, experimental: false} + - {os: ubuntu-latest, label: studio-deps, marker: 'studio deps', beat: 0} + - {os: ubuntu-latest, label: torch, marker: '\[TAURI:STEP\] Installing PyTorch', beat: 3} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -149,6 +155,23 @@ jobs: tail -30 logs/install.log || true exit 1 fi + # ...and it must have died from OUR signal rather than on its own. The installer + # can also fail naturally in the same window -- a dependency error exits 1 -- and + # that leg would test a broken installer while claiming to test an interrupted + # one. Recording whether `kill` returned 0 does not separate them: the leader is + # still an unreaped member of its own group, so signalling the group succeeds + # even when every process in it is already a zombie. The exit status does: every + # POSIX leg of staging run 30424366953 reported 143. + case "$installer_exit" in + 143|137) ;; + *) + echo "::error::installer exited $installer_exit, which is neither SIGTERM" + echo "::error::(143, install.sh's trap at install.sh:716) nor SIGKILL (137)." + echo "::error::It died on its own, so this leg interrupted nothing." + tail -30 logs/install.log || true + exit 1 + ;; + esac # ...and the signal must land in the phase this leg is NAMED for. Warning-only # left staging run 30419729244 fully green with the venv leg's install.log # byte-identical to the torch leg's, and both "studio deps" legs killed past @@ -180,10 +203,9 @@ jobs: exit "$rc" - name: A re-run must repair, not short-circuit - # NO_CLI included: a kill at venv or torch lands before "Installing Unsloth" + # NO_CLI included: a kill at torch lands before "Installing Unsloth" # (install.sh:2125 / :3667 / :3961), so those legs always take NO_CLI and skipping - # the re-run left three non-experimental legs asserting only that a marker - # appeared. The bug's second half is `install.sh` seeing a "current" version and + # the re-run left three of them asserting only that a marker appeared. The bug's second half is `install.sh` seeing a "current" version and # no-opping over a broken venv. HEALTHY also needs the install reported complete, # so the data-designer leg (killed before the manifest is written last, # install_python_stack.py:3255) arrives here instead of skipping that assertion. @@ -269,7 +291,7 @@ jobs: $kv = $line -split '=', 2 if ($kv.Count -eq 2) { $vals[$kv[0]] = $kv[1] } } - Write-Host "reason=$($vals['interrupt_reason']) killed=$($vals['interrupt_killed']) exit=$($vals['installer_exit'])" + Write-Host "reason=$($vals['interrupt_reason']) killed=$($vals['interrupt_killed']) root_killed=$($vals['interrupt_root_killed']) exit=$($vals['installer_exit'])" if ($vals['interrupt_reason'] -ne 'marker-hit') { Write-Host "::error::installer never reached '${{ matrix.marker }}' (reason=$($vals['interrupt_reason']))." Write-Host '::error::This leg proves nothing: without this check it passes via the' @@ -290,6 +312,19 @@ jobs: Get-Content logs/install.log -Tail 30 -ErrorAction SilentlyContinue exit 1 } + # ...and the signal has to have been DELIVERED. A non-zero code is weaker proof + # here than on POSIX, where only a signal produces 143/137: install.ps1 failing + # on its own also exits non-zero, so a natural failure landing between the + # driver's last HasExited check and Stop-Tree would otherwise read as a kill. + # Stop-Process throws on a process that is already gone, so the driver records + # false exactly when it found nothing left to interrupt. + if ($vals['interrupt_root_killed'] -ne 'true') { + Write-Host '::error::the driver never terminated the installer -- it was already' + Write-Host '::error::gone when Stop-Tree reached it, so it failed on its own and' + Write-Host '::error::this leg interrupted nothing.' + Get-Content logs/install.log -Tail 30 -ErrorAction SilentlyContinue + exit 1 + } # Same landing check as the POSIX leg: a phase already over when the poll saw the # marker means the kill hit a LATER phase, so the leg duplicates another one. if ($vals['interrupt_phase_mismatch'] -eq 'true') { From 75e5f6a4877696f9e9fdf0c39a20d9af34f9401f Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 05:59:00 +0000 Subject: [PATCH 24/25] Signal at the marker, with no beat to overshoot the phase Staging run 30426111484 failed the macOS torch leg on the landing check: the 3s beat carried the signal from "Installing PyTorch" into "Installing Unsloth", because the PyTorch step, which this workflow called minutes long, finished in under three seconds. The beat only ever existed to land mid-work, and it cannot do that safely: every label prints before its work starts, so detection is already inside the phase, and any wait is a bet on how long that phase runs. It lost in 30419729244 and again here. So the beat is gone rather than retuned, and with it the matrix knob and the driver parameter on both platforms. The landing check stays and can still fail, since a phase shorter than one poll is seen only after it ends. The Windows driver also polled every 500ms while its own comment claimed a fifth of a second. That is 2.5 slices of overshoot the POSIX side does not carry, and it is now 200ms like the POSIX loop. --- .github/scripts/interrupt-install.ps1 | 31 +++++++-------- .github/scripts/interrupt-install.sh | 26 +++++------- .github/workflows/interrupted-install-ci.yml | 42 ++++++++++---------- 3 files changed, 46 insertions(+), 53 deletions(-) diff --git a/.github/scripts/interrupt-install.ps1 b/.github/scripts/interrupt-install.ps1 index 26a7664201..60378943e8 100644 --- a/.github/scripts/interrupt-install.ps1 +++ b/.github/scripts/interrupt-install.ps1 @@ -15,8 +15,7 @@ param( [string]$Marker = '', [string]$LogPath = 'logs/install.log', [string]$InstallArgs = '', - [int]$KillAtSeconds = 900, - [int]$KillAfterMarkerSeconds = 0 + [int]$KillAtSeconds = 900 ) $ErrorActionPreference = 'Continue' @@ -111,30 +110,30 @@ function Test-MarkedPhaseOver { $killed = $false $reason = '' -# Fifth-of-a-second slices: every phase label prints BEFORE its work starts, so the poll -# delay is the whole distance between the label and the signal. +# Fifth-of-a-second slices, matching the POSIX driver: every phase label prints BEFORE its +# work starts, so this delay IS the whole distance between the label and the signal, and it +# is the only thing that can push the kill past the end of a short phase. It slept 500ms +# while the comment claimed a fifth, so it carried 2.5 slices of overshoot the POSIX side +# does not. for ($i = 0; $i -lt ($KillAtSeconds * 5); $i++) { if ($proc.HasExited) { $reason = 'exited-before-marker'; break } if ($Marker) { $hit = Select-String -Path $LogPath -Pattern $Marker -SimpleMatch:$false -ErrorAction SilentlyContinue if ($hit) { - # Same as the POSIX driver: no beat by default, because the label prints before the - # work, so killing at detection is already inside the phase while a flat beat sends - # the signal into a LATER phase. The loop stops the moment the marked phase ends. - for ($j = 0; $j -lt ($KillAfterMarkerSeconds * 5); $j++) { - if (Test-MarkedPhaseOver) { break } - Start-Sleep -Milliseconds 200 - if ($proc.HasExited) { break } - } - # The installer can finish inside the delay; recording marker-hit before it - # let a COMPLETED install satisfy the landing assertion and probe HEALTHY. - if ($proc.HasExited) { $reason = 'exited-during-marker-delay'; break } + # Same as the POSIX driver: signal at detection, never after a delay. The label + # prints before the work, so the kill is inside the phase the moment the line + # appears, and any wait is a bet on the phase outlasting it that staging runs + # 30419729244 and 30426111484 both lost. + # + # The installer can still exit on its own between the match and the signal, which + # would record marker-hit over an install that interrupted nothing. + if ($proc.HasExited) { $reason = 'exited-before-signal'; break } $reason = 'marker-hit' $killed = $true break } } - Start-Sleep -Milliseconds 500 + Start-Sleep -Milliseconds 200 } if (-not $killed -and -not $proc.HasExited) { if (-not $reason) { $reason = 'deadline' }; $killed = $true } diff --git a/.github/scripts/interrupt-install.sh b/.github/scripts/interrupt-install.sh index 1fd255b037..7e73d12c27 100755 --- a/.github/scripts/interrupt-install.sh +++ b/.github/scripts/interrupt-install.sh @@ -9,8 +9,7 @@ # # Usage: bash .github/scripts/interrupt-install.sh "" "" [-- install args] # log regex to wait for before killing, e.g. "studio deps"; "" kills at deadline. -# Env: KILL_AT_SECONDS deadline (default 900), KILL_GRACE grace before SIGKILL (default 10), -# KILL_AFTER_MARKER_SECONDS beat between the marker and the signal (default 0) +# Env: KILL_AT_SECONDS deadline (default 900), KILL_GRACE grace before SIGKILL (default 10) set -uo pipefail MARKER="${1:-}" @@ -83,20 +82,17 @@ for i in $(seq 1 $(( KILL_AT_SECONDS * 5 ))); do break fi if [ -n "$MARKER" ] && grep -qE "$MARKER" "$LOG" 2>/dev/null; then - # No beat by default: the label prints before the work, so killing at detection is - # already inside the phase, while a flat 3s beat is what pushed 5 of the 12 legs in - # staging run 30419729244 into a LATER phase (venv -> torch produced a byte-identical - # log to the torch leg). Legs whose phase runs for minutes pass a beat explicitly to - # land mid-work; the loop still stops the moment the marked phase ends. - for _ in $(seq 1 $(( ${KILL_AFTER_MARKER_SECONDS:-0} * 5 ))); do - marked_phase_over && break - sleep 0.2 - kill -0 "$PID" 2>/dev/null || break - done - # ...but a cached step can FINISH inside the beat. Recording marker-hit before it - # handed the landing assertion a COMPLETED install that interrupted nothing. + # Signal at detection, never after a delay. Every label prints BEFORE its work starts, + # so the kill is inside the phase the moment the line appears, and any wait at all is a + # bet on how long that phase runs. The bet loses: a flat 3s wait put 5 of the 12 legs + # of staging run 30419729244 into a LATER phase, and in 30426111484 it carried the + # macOS torch leg from "Installing PyTorch" into "Installing Unsloth" -- a step the + # workflow called minutes long finished in under three seconds. + # + # Between the grep and the signal the installer can still exit on its own, which would + # record marker-hit over an install that interrupted nothing. if ! kill -0 "$PID" 2>/dev/null; then - reason="exited-during-marker-delay" + reason="exited-before-signal" break fi reason="marker-hit" diff --git a/.github/workflows/interrupted-install-ci.yml b/.github/workflows/interrupted-install-ci.yml index df34bef1b8..ca0d6db1ff 100644 --- a/.github/workflows/interrupted-install-ci.yml +++ b/.github/workflows/interrupted-install-ci.yml @@ -69,19 +69,20 @@ jobs: fail-fast: false matrix: include: - # `beat` is the delay between the marker and the signal, in seconds. It is 0 - # everywhere except the three steps that provably run for minutes: every phase - # label prints BEFORE its work starts, so killing at detection is already inside - # the phase, while any beat longer than the phase lands the signal in the NEXT - # one. A flat 3s beat is what made 5 of the 12 legs of staging run 30419729244 - # interrupt a later phase than their label claims. + # Only a marker: the driver signals the moment that line appears, and no leg gets + # to wait first. Every label prints BEFORE its work starts, so the kill is inside + # the phase at detection, and a delay only bets on how long the phase runs. The + # bet lost twice, both times turning a leg into a duplicate of the next one: a + # flat 3s wait moved 5 of the 12 legs of staging run 30419729244, and in + # 30426111484 it carried the macOS torch leg into "Installing Unsloth" because + # the PyTorch step, called minutes long here, finished in under three seconds. # # Every leg is a hard gate. There is no continue-on-error cell: a leg allowed to # fail is a warning wearing a red icon, and this workflow's entire claim is that # a killed install cannot report itself healthy. # # The exact reported case: killed during the sub-step that installs structlog. - - {os: macos-14, label: studio-deps, marker: 'studio deps', beat: 0} + - {os: macos-14, label: studio-deps, marker: 'studio deps'} # Coarse phases, earliest to latest -- each leaves a different partial venv. # No venv cell. "Creating virtual environment" ran 0.107s in staging run # 30419729244 (03:31:07.371 -> 07.478 to "Installing PyTorch"), shorter than any @@ -89,22 +90,22 @@ jobs: # was tried (30423181897 and 30424366953 both). It was the torch leg with a # different label. Lost with it: a venv caught half-written. That state is not # reachable by interruption at this resolution, and it is the only thing lost -- - # the torch leg's signal lands ~3s into a multi-minute download, so what it - # leaves behind is already a complete venv with nothing installed into it. - - {os: macos-14, label: torch, marker: '\[TAURI:STEP\] Installing PyTorch', beat: 3} - - {os: macos-14, label: unsloth, marker: '\[TAURI:STEP\] Installing Unsloth', beat: 3} - - {os: macos-14, label: setup, marker: '\[TAURI:STEP\] Running Unsloth setup', beat: 3} + # the torch leg lands at the top of the PyTorch step, so what it leaves behind is + # already a complete venv with nothing installed into it. + - {os: macos-14, label: torch, marker: '\[TAURI:STEP\] Installing PyTorch'} + - {os: macos-14, label: unsloth, marker: '\[TAURI:STEP\] Installing Unsloth'} + - {os: macos-14, label: setup, marker: '\[TAURI:STEP\] Running Unsloth setup'} # Other dependency-pass sub-steps around the named one. No pip-bootstrap cell for # the same reason as venv: "1/10 pip bootstrap" is over before a poll can see it, # so in both 30419729244 and 30424366953 the signal landed in "2/10 unsloth # extras", which is the next cell down. # No base-packages cell: --local sets skip_base, so install_python_stack returns # before "base packages" ever prints -- that leg ran to completion, proving nothing. - - {os: macos-14, label: unsloth-extras, marker: 'unsloth extras', beat: 0} - - {os: macos-14, label: data-designer, marker: 'data designer deps', beat: 0} + - {os: macos-14, label: unsloth-extras, marker: 'unsloth extras'} + - {os: macos-14, label: data-designer, marker: 'data designer deps'} # Linux: same teardown path, different package manager and process semantics. - - {os: ubuntu-latest, label: studio-deps, marker: 'studio deps', beat: 0} - - {os: ubuntu-latest, label: torch, marker: '\[TAURI:STEP\] Installing PyTorch', beat: 3} + - {os: ubuntu-latest, label: studio-deps, marker: 'studio deps'} + - {os: ubuntu-latest, label: torch, marker: '\[TAURI:STEP\] Installing PyTorch'} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -121,7 +122,6 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} KILL_AT_SECONDS: '1500' - KILL_AFTER_MARKER_SECONDS: '${{ matrix.beat }}' run: | # --local is load-bearing. Without it install.sh:3996 resolves # `unsloth>=2026.7.5` from PyPI, so the venv gets the PUBLISHED CLI, every @@ -266,9 +266,8 @@ jobs: include: # install.ps1:121 parses `--no-torch`; `-SkipTorch` matches no case there and is # silently dropped. The torch leg must NOT skip torch or its marker never appears. - # `beat` as in the POSIX job: 0 for the sub-step, 3 for the minutes-long torch step. - - {label: studio-deps, marker: 'studio deps', beat: 0, installArgs: '--tauri --no-torch --local'} - - {label: torch, marker: 'Installing PyTorch', beat: 3, installArgs: '--tauri --local'} + - {label: studio-deps, marker: 'studio deps', installArgs: '--tauri --no-torch --local'} + - {label: torch, marker: 'Installing PyTorch', installArgs: '--tauri --local'} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -280,8 +279,7 @@ jobs: run: | pwsh -NoProfile -File .github/scripts/interrupt-install.ps1 ` -Marker '${{ matrix.marker }}' -LogPath logs/install.log ` - -InstallArgs '${{ matrix.installArgs }}' -KillAtSeconds 1500 ` - -KillAfterMarkerSeconds ${{ matrix.beat }} + -InstallArgs '${{ matrix.installArgs }}' -KillAtSeconds 1500 - name: The kill must have landed where it was aimed shell: pwsh From 4493b14647ee6c1f09617803935a4b08c6b43cb1 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 06:11:02 +0000 Subject: [PATCH 25/25] Kill the installer before its children, not after The depth-first walk killed the child install.ps1 was waiting on and only then the root, which races the leader's own reaction to that death. It is not a theoretical race: in staging run 30424366953 install.ps1 had already printed "unsloth studio setup failed (exit code -1)" by the time Stop-Process reached it. A leader that wins the race makes Stop-Process throw, and the new root-kill assertion would then fail a leg whose interruption the driver really did deliver. The tree is now snapshotted first, since a dead parent leaves nothing to walk, then the root goes down ahead of its descendants. A dead leader cannot react to a child and cannot respawn one either, which is what the depth-first order was for. --- .github/scripts/interrupt-install.ps1 | 31 ++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/.github/scripts/interrupt-install.ps1 b/.github/scripts/interrupt-install.ps1 index 60378943e8..5bbff97d45 100644 --- a/.github/scripts/interrupt-install.ps1 +++ b/.github/scripts/interrupt-install.ps1 @@ -59,17 +59,38 @@ Write-Host "[interrupt] installer pid=$($proc.Id) marker='$Marker' deadline=${Ki # false exactly when there was nothing left to interrupt. $script:rootKilled = $false +function Get-Descendants([int]$RootId) { + # Depth-first, deepest first. CIM gives the parent link Windows does not expose via + # process groups. + $ids = @() + foreach ($k in @(Get-CimInstance Win32_Process -Filter "ParentProcessId=$RootId" -ErrorAction SilentlyContinue)) { + $kid = [int]$k.ProcessId + $ids += Get-Descendants $kid + $ids += $kid + } + return $ids +} + function Stop-Tree([int]$RootId) { - # Depth-first, so a parent cannot respawn a child we already killed. CIM gives the - # parent link Windows does not expose via process groups. - $kids = @(Get-CimInstance Win32_Process -Filter "ParentProcessId=$RootId" -ErrorAction SilentlyContinue) - foreach ($k in $kids) { Stop-Tree ([int]$k.ProcessId) } + # Snapshot the whole tree BEFORE killing anything. Once a parent is gone its children are + # orphaned and there is no ParentProcessId left to walk, so the walk has to happen first. + $descendants = @(Get-Descendants $RootId) + # Then the ROOT, ahead of its children. install.ps1 watches the child it is waiting on: + # in staging run 30424366953 it had already printed "unsloth studio setup failed (exit + # code -1)" by the time Stop-Process reached it. Killing children first therefore races + # the leader's own exit, and a leader that wins that race makes Stop-Process throw over + # an interruption the driver really did deliver, failing the leg for nothing. Dead first, + # it cannot react to anything, and it cannot respawn what we are about to kill either. try { Stop-Process -Id $RootId -Force -ErrorAction Stop - Write-Host "[interrupt] killed pid=$RootId" + Write-Host "[interrupt] killed installer pid=$RootId" if ($RootId -eq $proc.Id) { $script:rootKilled = $true } } catch { if ($RootId -eq $proc.Id) { Write-Host "[interrupt] installer pid=$RootId was already gone: $_" } } + foreach ($id in $descendants) { + try { Stop-Process -Id $id -Force -ErrorAction Stop; Write-Host "[interrupt] killed pid=$id" } + catch { } + } } # A leg can be aimed at either kind of phase the installer prints, and only one of them is