Signal at the marker, with no beat to overshoot the phase
Staging run 30426111484 failed the macOS torch leg on the landing check: the 3s beat carried the signal from "Installing PyTorch" into "Installing Unsloth", because the PyTorch step, which this workflow called minutes long, finished in under three seconds. The beat only ever existed to land mid-work, and it cannot do that safely: every label prints before its work starts, so detection is already inside the phase, and any wait is a bet on how long that phase runs. It lost in 30419729244 and again here. So the beat is gone rather than retuned, and with it the matrix knob and the driver parameter on both platforms. The landing check stays and can still fail, since a phase shorter than one poll is seen only after it ends. The Windows driver also polled every 500ms while its own comment claimed a fifth of a second. That is 2.5 slices of overshoot the POSIX side does not carry, and it is now 200ms like the POSIX loop.
This commit is contained in:
parent
e5b40c11cf
commit
75e5f6a487
3 changed files with 46 additions and 53 deletions
31
.github/scripts/interrupt-install.ps1
vendored
31
.github/scripts/interrupt-install.ps1
vendored
|
|
@ -15,8 +15,7 @@ param(
|
|||
[string]$Marker = '',
|
||||
[string]$LogPath = 'logs/install.log',
|
||||
[string]$InstallArgs = '',
|
||||
[int]$KillAtSeconds = 900,
|
||||
[int]$KillAfterMarkerSeconds = 0
|
||||
[int]$KillAtSeconds = 900
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Continue'
|
||||
|
|
@ -111,30 +110,30 @@ function Test-MarkedPhaseOver {
|
|||
|
||||
$killed = $false
|
||||
$reason = ''
|
||||
# Fifth-of-a-second slices: every phase label prints BEFORE its work starts, so the poll
|
||||
# delay is the whole distance between the label and the signal.
|
||||
# Fifth-of-a-second slices, matching the POSIX driver: every phase label prints BEFORE its
|
||||
# work starts, so this delay IS the whole distance between the label and the signal, and it
|
||||
# is the only thing that can push the kill past the end of a short phase. It slept 500ms
|
||||
# while the comment claimed a fifth, so it carried 2.5 slices of overshoot the POSIX side
|
||||
# does not.
|
||||
for ($i = 0; $i -lt ($KillAtSeconds * 5); $i++) {
|
||||
if ($proc.HasExited) { $reason = 'exited-before-marker'; break }
|
||||
if ($Marker) {
|
||||
$hit = Select-String -Path $LogPath -Pattern $Marker -SimpleMatch:$false -ErrorAction SilentlyContinue
|
||||
if ($hit) {
|
||||
# Same as the POSIX driver: no beat by default, because the label prints before the
|
||||
# work, so killing at detection is already inside the phase while a flat beat sends
|
||||
# the signal into a LATER phase. The loop stops the moment the marked phase ends.
|
||||
for ($j = 0; $j -lt ($KillAfterMarkerSeconds * 5); $j++) {
|
||||
if (Test-MarkedPhaseOver) { break }
|
||||
Start-Sleep -Milliseconds 200
|
||||
if ($proc.HasExited) { break }
|
||||
}
|
||||
# The installer can finish inside the delay; recording marker-hit before it
|
||||
# let a COMPLETED install satisfy the landing assertion and probe HEALTHY.
|
||||
if ($proc.HasExited) { $reason = 'exited-during-marker-delay'; break }
|
||||
# Same as the POSIX driver: signal at detection, never after a delay. The label
|
||||
# prints before the work, so the kill is inside the phase the moment the line
|
||||
# appears, and any wait is a bet on the phase outlasting it that staging runs
|
||||
# 30419729244 and 30426111484 both lost.
|
||||
#
|
||||
# The installer can still exit on its own between the match and the signal, which
|
||||
# would record marker-hit over an install that interrupted nothing.
|
||||
if ($proc.HasExited) { $reason = 'exited-before-signal'; break }
|
||||
$reason = 'marker-hit'
|
||||
$killed = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
Start-Sleep -Milliseconds 500
|
||||
Start-Sleep -Milliseconds 200
|
||||
}
|
||||
if (-not $killed -and -not $proc.HasExited) { if (-not $reason) { $reason = 'deadline' }; $killed = $true }
|
||||
|
||||
|
|
|
|||
26
.github/scripts/interrupt-install.sh
vendored
26
.github/scripts/interrupt-install.sh
vendored
|
|
@ -9,8 +9,7 @@
|
|||
#
|
||||
# Usage: bash .github/scripts/interrupt-install.sh "<marker>" "<logfile>" [-- install args]
|
||||
# <marker> log regex to wait for before killing, e.g. "studio deps"; "" kills at deadline.
|
||||
# Env: KILL_AT_SECONDS deadline (default 900), KILL_GRACE grace before SIGKILL (default 10),
|
||||
# KILL_AFTER_MARKER_SECONDS beat between the marker and the signal (default 0)
|
||||
# Env: KILL_AT_SECONDS deadline (default 900), KILL_GRACE grace before SIGKILL (default 10)
|
||||
set -uo pipefail
|
||||
|
||||
MARKER="${1:-}"
|
||||
|
|
@ -83,20 +82,17 @@ for i in $(seq 1 $(( KILL_AT_SECONDS * 5 ))); do
|
|||
break
|
||||
fi
|
||||
if [ -n "$MARKER" ] && grep -qE "$MARKER" "$LOG" 2>/dev/null; then
|
||||
# No beat by default: the label prints before the work, so killing at detection is
|
||||
# already inside the phase, while a flat 3s beat is what pushed 5 of the 12 legs in
|
||||
# staging run 30419729244 into a LATER phase (venv -> torch produced a byte-identical
|
||||
# log to the torch leg). Legs whose phase runs for minutes pass a beat explicitly to
|
||||
# land mid-work; the loop still stops the moment the marked phase ends.
|
||||
for _ in $(seq 1 $(( ${KILL_AFTER_MARKER_SECONDS:-0} * 5 ))); do
|
||||
marked_phase_over && break
|
||||
sleep 0.2
|
||||
kill -0 "$PID" 2>/dev/null || break
|
||||
done
|
||||
# ...but a cached step can FINISH inside the beat. Recording marker-hit before it
|
||||
# handed the landing assertion a COMPLETED install that interrupted nothing.
|
||||
# Signal at detection, never after a delay. Every label prints BEFORE its work starts,
|
||||
# so the kill is inside the phase the moment the line appears, and any wait at all is a
|
||||
# bet on how long that phase runs. The bet loses: a flat 3s wait put 5 of the 12 legs
|
||||
# of staging run 30419729244 into a LATER phase, and in 30426111484 it carried the
|
||||
# macOS torch leg from "Installing PyTorch" into "Installing Unsloth" -- a step the
|
||||
# workflow called minutes long finished in under three seconds.
|
||||
#
|
||||
# Between the grep and the signal the installer can still exit on its own, which would
|
||||
# record marker-hit over an install that interrupted nothing.
|
||||
if ! kill -0 "$PID" 2>/dev/null; then
|
||||
reason="exited-during-marker-delay"
|
||||
reason="exited-before-signal"
|
||||
break
|
||||
fi
|
||||
reason="marker-hit"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue