Signal at the marker, with no beat to overshoot the phase
Staging run 30426111484 failed the macOS torch leg on the landing check: the 3s beat carried the signal from "Installing PyTorch" into "Installing Unsloth", because the PyTorch step, which this workflow called minutes long, finished in under three seconds. The beat only ever existed to land mid-work, and it cannot do that safely: every label prints before its work starts, so detection is already inside the phase, and any wait is a bet on how long that phase runs. It lost in 30419729244 and again here. So the beat is gone rather than retuned, and with it the matrix knob and the driver parameter on both platforms. The landing check stays and can still fail, since a phase shorter than one poll is seen only after it ends. The Windows driver also polled every 500ms while its own comment claimed a fifth of a second. That is 2.5 slices of overshoot the POSIX side does not carry, and it is now 200ms like the POSIX loop.
This commit is contained in:
parent
e5b40c11cf
commit
75e5f6a487
3 changed files with 46 additions and 53 deletions
31
.github/scripts/interrupt-install.ps1
vendored
31
.github/scripts/interrupt-install.ps1
vendored
|
|
@ -15,8 +15,7 @@ param(
|
|||
[string]$Marker = '',
|
||||
[string]$LogPath = 'logs/install.log',
|
||||
[string]$InstallArgs = '',
|
||||
[int]$KillAtSeconds = 900,
|
||||
[int]$KillAfterMarkerSeconds = 0
|
||||
[int]$KillAtSeconds = 900
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Continue'
|
||||
|
|
@ -111,30 +110,30 @@ function Test-MarkedPhaseOver {
|
|||
|
||||
$killed = $false
|
||||
$reason = ''
|
||||
# Fifth-of-a-second slices: every phase label prints BEFORE its work starts, so the poll
|
||||
# delay is the whole distance between the label and the signal.
|
||||
# Fifth-of-a-second slices, matching the POSIX driver: every phase label prints BEFORE its
|
||||
# work starts, so this delay IS the whole distance between the label and the signal, and it
|
||||
# is the only thing that can push the kill past the end of a short phase. It slept 500ms
|
||||
# while the comment claimed a fifth, so it carried 2.5 slices of overshoot the POSIX side
|
||||
# does not.
|
||||
for ($i = 0; $i -lt ($KillAtSeconds * 5); $i++) {
|
||||
if ($proc.HasExited) { $reason = 'exited-before-marker'; break }
|
||||
if ($Marker) {
|
||||
$hit = Select-String -Path $LogPath -Pattern $Marker -SimpleMatch:$false -ErrorAction SilentlyContinue
|
||||
if ($hit) {
|
||||
# Same as the POSIX driver: no beat by default, because the label prints before the
|
||||
# work, so killing at detection is already inside the phase while a flat beat sends
|
||||
# the signal into a LATER phase. The loop stops the moment the marked phase ends.
|
||||
for ($j = 0; $j -lt ($KillAfterMarkerSeconds * 5); $j++) {
|
||||
if (Test-MarkedPhaseOver) { break }
|
||||
Start-Sleep -Milliseconds 200
|
||||
if ($proc.HasExited) { break }
|
||||
}
|
||||
# The installer can finish inside the delay; recording marker-hit before it
|
||||
# let a COMPLETED install satisfy the landing assertion and probe HEALTHY.
|
||||
if ($proc.HasExited) { $reason = 'exited-during-marker-delay'; break }
|
||||
# Same as the POSIX driver: signal at detection, never after a delay. The label
|
||||
# prints before the work, so the kill is inside the phase the moment the line
|
||||
# appears, and any wait is a bet on the phase outlasting it that staging runs
|
||||
# 30419729244 and 30426111484 both lost.
|
||||
#
|
||||
# The installer can still exit on its own between the match and the signal, which
|
||||
# would record marker-hit over an install that interrupted nothing.
|
||||
if ($proc.HasExited) { $reason = 'exited-before-signal'; break }
|
||||
$reason = 'marker-hit'
|
||||
$killed = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
Start-Sleep -Milliseconds 500
|
||||
Start-Sleep -Milliseconds 200
|
||||
}
|
||||
if (-not $killed -and -not $proc.HasExited) { if (-not $reason) { $reason = 'deadline' }; $killed = $true }
|
||||
|
||||
|
|
|
|||
26
.github/scripts/interrupt-install.sh
vendored
26
.github/scripts/interrupt-install.sh
vendored
|
|
@ -9,8 +9,7 @@
|
|||
#
|
||||
# Usage: bash .github/scripts/interrupt-install.sh "<marker>" "<logfile>" [-- install args]
|
||||
# <marker> log regex to wait for before killing, e.g. "studio deps"; "" kills at deadline.
|
||||
# Env: KILL_AT_SECONDS deadline (default 900), KILL_GRACE grace before SIGKILL (default 10),
|
||||
# KILL_AFTER_MARKER_SECONDS beat between the marker and the signal (default 0)
|
||||
# Env: KILL_AT_SECONDS deadline (default 900), KILL_GRACE grace before SIGKILL (default 10)
|
||||
set -uo pipefail
|
||||
|
||||
MARKER="${1:-}"
|
||||
|
|
@ -83,20 +82,17 @@ for i in $(seq 1 $(( KILL_AT_SECONDS * 5 ))); do
|
|||
break
|
||||
fi
|
||||
if [ -n "$MARKER" ] && grep -qE "$MARKER" "$LOG" 2>/dev/null; then
|
||||
# No beat by default: the label prints before the work, so killing at detection is
|
||||
# already inside the phase, while a flat 3s beat is what pushed 5 of the 12 legs in
|
||||
# staging run 30419729244 into a LATER phase (venv -> torch produced a byte-identical
|
||||
# log to the torch leg). Legs whose phase runs for minutes pass a beat explicitly to
|
||||
# land mid-work; the loop still stops the moment the marked phase ends.
|
||||
for _ in $(seq 1 $(( ${KILL_AFTER_MARKER_SECONDS:-0} * 5 ))); do
|
||||
marked_phase_over && break
|
||||
sleep 0.2
|
||||
kill -0 "$PID" 2>/dev/null || break
|
||||
done
|
||||
# ...but a cached step can FINISH inside the beat. Recording marker-hit before it
|
||||
# handed the landing assertion a COMPLETED install that interrupted nothing.
|
||||
# Signal at detection, never after a delay. Every label prints BEFORE its work starts,
|
||||
# so the kill is inside the phase the moment the line appears, and any wait at all is a
|
||||
# bet on how long that phase runs. The bet loses: a flat 3s wait put 5 of the 12 legs
|
||||
# of staging run 30419729244 into a LATER phase, and in 30426111484 it carried the
|
||||
# macOS torch leg from "Installing PyTorch" into "Installing Unsloth" -- a step the
|
||||
# workflow called minutes long finished in under three seconds.
|
||||
#
|
||||
# Between the grep and the signal the installer can still exit on its own, which would
|
||||
# record marker-hit over an install that interrupted nothing.
|
||||
if ! kill -0 "$PID" 2>/dev/null; then
|
||||
reason="exited-during-marker-delay"
|
||||
reason="exited-before-signal"
|
||||
break
|
||||
fi
|
||||
reason="marker-hit"
|
||||
|
|
|
|||
42
.github/workflows/interrupted-install-ci.yml
vendored
42
.github/workflows/interrupted-install-ci.yml
vendored
|
|
@ -69,19 +69,20 @@ jobs:
|
|||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
# `beat` is the delay between the marker and the signal, in seconds. It is 0
|
||||
# everywhere except the three steps that provably run for minutes: every phase
|
||||
# label prints BEFORE its work starts, so killing at detection is already inside
|
||||
# the phase, while any beat longer than the phase lands the signal in the NEXT
|
||||
# one. A flat 3s beat is what made 5 of the 12 legs of staging run 30419729244
|
||||
# interrupt a later phase than their label claims.
|
||||
# Only a marker: the driver signals the moment that line appears, and no leg gets
|
||||
# to wait first. Every label prints BEFORE its work starts, so the kill is inside
|
||||
# the phase at detection, and a delay only bets on how long the phase runs. The
|
||||
# bet lost twice, both times turning a leg into a duplicate of the next one: a
|
||||
# flat 3s wait moved 5 of the 12 legs of staging run 30419729244, and in
|
||||
# 30426111484 it carried the macOS torch leg into "Installing Unsloth" because
|
||||
# the PyTorch step, called minutes long here, finished in under three seconds.
|
||||
#
|
||||
# Every leg is a hard gate. There is no continue-on-error cell: a leg allowed to
|
||||
# fail is a warning wearing a red icon, and this workflow's entire claim is that
|
||||
# a killed install cannot report itself healthy.
|
||||
#
|
||||
# The exact reported case: killed during the sub-step that installs structlog.
|
||||
- {os: macos-14, label: studio-deps, marker: 'studio deps', beat: 0}
|
||||
- {os: macos-14, label: studio-deps, marker: 'studio deps'}
|
||||
# Coarse phases, earliest to latest -- each leaves a different partial venv.
|
||||
# No venv cell. "Creating virtual environment" ran 0.107s in staging run
|
||||
# 30419729244 (03:31:07.371 -> 07.478 to "Installing PyTorch"), shorter than any
|
||||
|
|
@ -89,22 +90,22 @@ jobs:
|
|||
# was tried (30423181897 and 30424366953 both). It was the torch leg with a
|
||||
# different label. Lost with it: a venv caught half-written. That state is not
|
||||
# reachable by interruption at this resolution, and it is the only thing lost --
|
||||
# the torch leg's signal lands ~3s into a multi-minute download, so what it
|
||||
# leaves behind is already a complete venv with nothing installed into it.
|
||||
- {os: macos-14, label: torch, marker: '\[TAURI:STEP\] Installing PyTorch', beat: 3}
|
||||
- {os: macos-14, label: unsloth, marker: '\[TAURI:STEP\] Installing Unsloth', beat: 3}
|
||||
- {os: macos-14, label: setup, marker: '\[TAURI:STEP\] Running Unsloth setup', beat: 3}
|
||||
# the torch leg lands at the top of the PyTorch step, so what it leaves behind is
|
||||
# already a complete venv with nothing installed into it.
|
||||
- {os: macos-14, label: torch, marker: '\[TAURI:STEP\] Installing PyTorch'}
|
||||
- {os: macos-14, label: unsloth, marker: '\[TAURI:STEP\] Installing Unsloth'}
|
||||
- {os: macos-14, label: setup, marker: '\[TAURI:STEP\] Running Unsloth setup'}
|
||||
# Other dependency-pass sub-steps around the named one. No pip-bootstrap cell for
|
||||
# the same reason as venv: "1/10 pip bootstrap" is over before a poll can see it,
|
||||
# so in both 30419729244 and 30424366953 the signal landed in "2/10 unsloth
|
||||
# extras", which is the next cell down.
|
||||
# No base-packages cell: --local sets skip_base, so install_python_stack returns
|
||||
# before "base packages" ever prints -- that leg ran to completion, proving nothing.
|
||||
- {os: macos-14, label: unsloth-extras, marker: 'unsloth extras', beat: 0}
|
||||
- {os: macos-14, label: data-designer, marker: 'data designer deps', beat: 0}
|
||||
- {os: macos-14, label: unsloth-extras, marker: 'unsloth extras'}
|
||||
- {os: macos-14, label: data-designer, marker: 'data designer deps'}
|
||||
# Linux: same teardown path, different package manager and process semantics.
|
||||
- {os: ubuntu-latest, label: studio-deps, marker: 'studio deps', beat: 0}
|
||||
- {os: ubuntu-latest, label: torch, marker: '\[TAURI:STEP\] Installing PyTorch', beat: 3}
|
||||
- {os: ubuntu-latest, label: studio-deps, marker: 'studio deps'}
|
||||
- {os: ubuntu-latest, label: torch, marker: '\[TAURI:STEP\] Installing PyTorch'}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
|
@ -121,7 +122,6 @@ jobs:
|
|||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
KILL_AT_SECONDS: '1500'
|
||||
KILL_AFTER_MARKER_SECONDS: '${{ matrix.beat }}'
|
||||
run: |
|
||||
# --local is load-bearing. Without it install.sh:3996 resolves
|
||||
# `unsloth>=2026.7.5` from PyPI, so the venv gets the PUBLISHED CLI, every
|
||||
|
|
@ -266,9 +266,8 @@ jobs:
|
|||
include:
|
||||
# install.ps1:121 parses `--no-torch`; `-SkipTorch` matches no case there and is
|
||||
# silently dropped. The torch leg must NOT skip torch or its marker never appears.
|
||||
# `beat` as in the POSIX job: 0 for the sub-step, 3 for the minutes-long torch step.
|
||||
- {label: studio-deps, marker: 'studio deps', beat: 0, installArgs: '--tauri --no-torch --local'}
|
||||
- {label: torch, marker: 'Installing PyTorch', beat: 3, installArgs: '--tauri --local'}
|
||||
- {label: studio-deps, marker: 'studio deps', installArgs: '--tauri --no-torch --local'}
|
||||
- {label: torch, marker: 'Installing PyTorch', installArgs: '--tauri --local'}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
|
@ -280,8 +279,7 @@ jobs:
|
|||
run: |
|
||||
pwsh -NoProfile -File .github/scripts/interrupt-install.ps1 `
|
||||
-Marker '${{ matrix.marker }}' -LogPath logs/install.log `
|
||||
-InstallArgs '${{ matrix.installArgs }}' -KillAtSeconds 1500 `
|
||||
-KillAfterMarkerSeconds ${{ matrix.beat }}
|
||||
-InstallArgs '${{ matrix.installArgs }}' -KillAtSeconds 1500
|
||||
|
||||
- name: The kill must have landed where it was aimed
|
||||
shell: pwsh
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue