Assert no source build on the hosted Windows legs and keep winget for the desktop lane
This commit is contained in:
Daniel Han
parent
22495b3485
commit
404e38baf9
4 changed files with 130 additions and 39 deletions
69
.github/scripts/assert-nobuild.ps1
vendored
Normal file
69
.github/scripts/assert-nobuild.ps1
vendored
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
|
||||
|
||||
# The `nobuild` contract from clean-machine-assert.sh, for Windows.
|
||||
#
|
||||
# Why a port and not `shell: bash`: the clean-machine scrub drops every `*\Git\*`
|
||||
# entry from PATH and from the Machine/User registry copies, and the bash version
|
||||
# needs sed/grep/tr/sort out of C:\Program Files\Git\usr\bin. This also runs inside
|
||||
# the servercore container, which has no bash at all. Both Windows lanes call this
|
||||
# one file so the sdist allowlist cannot drift between them.
|
||||
#
|
||||
# Usage: assert-nobuild.ps1 -LogPath logs/install.log (exit 1 = a source build)
|
||||
[CmdletBinding()]
|
||||
param([Parameter(Mandatory = $true)][string] $LogPath)
|
||||
|
||||
if (-not (Test-Path -LiteralPath $LogPath)) {
|
||||
Write-Host "::error::nobuild requested but $LogPath is missing"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# "Built an sdist" is NOT "needed a compiler". Every name here was checked against
|
||||
# its own sdist: setuptools.build_meta backend, no ext_modules, no .c/.cpp/.pyx/.rs
|
||||
# file, so its PEP 517 build is a pure-Python copy step. UNSLOTH_ALLOW_SDIST extends
|
||||
# the list. Kept identical to clean-machine-assert.sh's `_allow`.
|
||||
$allow = @('openai-whisper', 'argbind', 'randomname', 'antlr4-python3-runtime', 'triton-kernels')
|
||||
if ($env:UNSLOTH_ALLOW_SDIST) {
|
||||
$allow += ($env:UNSLOTH_ALLOW_SDIST -split '\s+' | Where-Object { $_ })
|
||||
}
|
||||
# Lowercased and underscore-folded on both sides: a distribution name and the name uv
|
||||
# prints can disagree on the separator (triton_kernels vs triton-kernels).
|
||||
$allow = @($allow | ForEach-Object { $_.ToLowerInvariant() -replace '_', '-' })
|
||||
|
||||
# [char]27, not "`e": the `e escape is PowerShell 6+, and this runs under Windows
|
||||
# PowerShell 5.1 too, where "`e" degrades to a literal "e" and the strip would eat
|
||||
# real text instead of ANSI codes.
|
||||
$esc = [char]27
|
||||
$text = (Get-Content -LiteralPath $LogPath -Raw) -replace "$esc\[[0-9;]*[A-Za-z]", ''
|
||||
$built = @()
|
||||
foreach ($line in ($text -split "`r?`n")) {
|
||||
# A local-path build is something the caller pointed at (the CI source overlay),
|
||||
# never something dependency resolution chose. Index dependencies always print
|
||||
# `<name>==<version>`, so no signal is lost.
|
||||
if ($line -imatch 'building [a-z0-9._-]+ @ file://') { continue }
|
||||
# pip prints `Building wheel for <pkg>`, uv prints `Building <pkg>==<ver>`
|
||||
# (astral-sh/uv#11165). Requiring `==` or ` @ ` after the name keeps this off the
|
||||
# installer's own lowercase "building frontend..." progress text.
|
||||
foreach ($m in [regex]::Matches($line, '(?i)building wheel for ([a-z0-9._-]+)|building ([a-z0-9._-]+)(==| @ )')) {
|
||||
$name = if ($m.Groups[1].Success) { $m.Groups[1].Value } else { $m.Groups[2].Value }
|
||||
$built += ($name.ToLowerInvariant() -replace '_', '-')
|
||||
}
|
||||
}
|
||||
$built = @($built | Sort-Object -Unique)
|
||||
$bad = @($built | Where-Object { $allow -notcontains $_ })
|
||||
|
||||
$rc = 0
|
||||
if ($bad.Count -gt 0) {
|
||||
Write-Host "::error::built from source: $($bad -join ' ') -- these must resolve to wheels on a clean machine"
|
||||
$rc = 1
|
||||
} else {
|
||||
Write-Host "[assert] OK no non-allowlisted source build (built: $(if ($built) { $built -join ' ' } else { 'none' }))"
|
||||
}
|
||||
# Independent of package names: a compiler error means a toolchain was needed.
|
||||
$compilerErr = Select-String -Path $LogPath -Pattern "error: command '(cc|gcc|clang|cl)' failed", 'clang: error', 'cargo: not found', 'Microsoft Visual C\+\+ 14.0 or greater is required'
|
||||
if ($compilerErr) {
|
||||
Write-Host '::error::compiler invocation appears in the install log'
|
||||
$compilerErr | Select-Object -First 10 | ForEach-Object { Write-Host " $($_.Line)" }
|
||||
$rc = 1
|
||||
}
|
||||
exit $rc
|
||||
40
.github/scripts/virgin-windows-install.ps1
vendored
40
.github/scripts/virgin-windows-install.ps1
vendored
|
|
@ -132,40 +132,14 @@ if ($Overlay -and $rc -eq 0) {
|
|||
}
|
||||
|
||||
Section 'assert: no non-allowlisted source build'
|
||||
# PowerShell port of .github/scripts/clean-machine-assert.sh's `nobuild`. Same
|
||||
# contract: pip prints "Building wheel for <pkg>", uv prints "Building <pkg>==<ver>"
|
||||
# (astral-sh/uv#11165), and a local-path build (`Building <pkg> @ file://`) is
|
||||
# something the caller pointed at, never something resolution chose.
|
||||
if (-not (Test-Path -LiteralPath $LogPath)) {
|
||||
$failures += "nobuild requested but $LogPath is missing"
|
||||
# Shared with the hosted Windows legs so the sdist allowlist lives in one place; the
|
||||
# script prints its own diagnosis, so only the verdict is folded in here.
|
||||
$nobuild = Join-Path $PSScriptRoot 'assert-nobuild.ps1'
|
||||
if (-not (Test-Path -LiteralPath $nobuild)) {
|
||||
$failures += "assert-nobuild.ps1 is missing next to this script, so the no-build contract went unchecked"
|
||||
} else {
|
||||
$allow = @('openai-whisper', 'argbind', 'randomname', 'antlr4-python3-runtime', 'triton-kernels')
|
||||
# [char]27, not "`e": the `e escape sequence is PowerShell 6+, and this script runs
|
||||
# under Windows PowerShell 5.1, where "`e" silently degrades to a literal "e" and
|
||||
# the strip would eat real text instead of ANSI codes.
|
||||
$esc = [char]27
|
||||
$text = (Get-Content -LiteralPath $LogPath -Raw) -replace "$esc\[[0-9;]*[A-Za-z]", ''
|
||||
$built = @()
|
||||
foreach ($line in ($text -split "`r?`n")) {
|
||||
if ($line -imatch 'building [a-z0-9._-]+ @ file://') { continue }
|
||||
foreach ($m in [regex]::Matches($line, '(?i)building wheel for ([a-z0-9._-]+)|building ([a-z0-9._-]+)(==| @ )')) {
|
||||
$name = if ($m.Groups[1].Success) { $m.Groups[1].Value } else { $m.Groups[2].Value }
|
||||
$built += ($name.ToLowerInvariant() -replace '_', '-')
|
||||
}
|
||||
}
|
||||
$built = $built | Sort-Object -Unique
|
||||
$bad = @($built | Where-Object { $allow -notcontains $_ })
|
||||
if ($bad.Count -gt 0) {
|
||||
$failures += "built from source: $($bad -join ' ') -- these must resolve to wheels on a clean machine"
|
||||
} else {
|
||||
Write-Host "no non-allowlisted source build (built: $(if ($built) { $built -join ' ' } else { 'none' }))"
|
||||
}
|
||||
# Independent of package names: a compiler error means a toolchain was needed.
|
||||
$compilerErr = Select-String -Path $LogPath -Pattern "error: command '(cc|gcc|clang|cl)' failed", 'clang: error', 'cargo: not found', 'Microsoft Visual C\+\+ 14.0 or greater is required'
|
||||
if ($compilerErr) {
|
||||
$failures += "compiler invocation appears in the install log"
|
||||
$compilerErr | Select-Object -First 10 | ForEach-Object { Write-Host " $($_.Line)" }
|
||||
}
|
||||
& $nobuild -LogPath $LogPath
|
||||
if ($LASTEXITCODE -ne 0) { $failures += "a non-allowlisted source build appears in the install log" }
|
||||
}
|
||||
|
||||
# ── Verdict ───────────────────────────────────────────────────────────────────
|
||||
|
|
|
|||
23
.github/workflows/clean-machine-install-ci.yml
vendored
23
.github/workflows/clean-machine-install-ci.yml
vendored
|
|
@ -66,6 +66,7 @@ on:
|
|||
# The virgin Windows container lane lives in this workflow too.
|
||||
- '.github/scripts/virgin-windows-*.ps1'
|
||||
- '.github/scripts/ensure-docker-daemon.ps1'
|
||||
- '.github/scripts/assert-nobuild.ps1'
|
||||
- '.github/workflows/clean-machine-install-ci.yml'
|
||||
push:
|
||||
branches: [main]
|
||||
|
|
@ -84,6 +85,7 @@ on:
|
|||
# The virgin Windows container lane lives in this workflow too.
|
||||
- '.github/scripts/virgin-windows-*.ps1'
|
||||
- '.github/scripts/ensure-docker-daemon.ps1'
|
||||
- '.github/scripts/assert-nobuild.ps1'
|
||||
- '.github/workflows/clean-machine-install-ci.yml'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
|
|
@ -1021,10 +1023,11 @@ jobs:
|
|||
$newPath = ($kept -join ';')
|
||||
"PATH=$newPath" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
# install.ps1's Refresh-SessionPath (318-337, called at 1246/1278/1295/1360/
|
||||
# 1369/2797) rebuilds $env:Path from the Machine and User registry values, so a
|
||||
# process-only scrub lasts until the first bootstrap refresh, after which
|
||||
# 1369/2797) merges the Machine and User registry PATHs back into $env:Path, so
|
||||
# a process-only scrub lasts until the first bootstrap refresh, after which
|
||||
# Git/CMake/VS/LLVM are back and the rest of the install is not clean. The
|
||||
# runner is ephemeral, so rewrite the registry copies too. Expand first:
|
||||
# runner is ephemeral, so rewrite the registry copies too. It is a merge, not a
|
||||
# replace, so the shim above keeps resolving. Expand first:
|
||||
# SetEnvironmentVariable rewrites REG_EXPAND_SZ as REG_SZ
|
||||
# (dotnet/runtime#1442).
|
||||
foreach ($scope in 'Machine','User') {
|
||||
|
|
@ -1295,6 +1298,20 @@ jobs:
|
|||
}
|
||||
Write-Host 'no CMake and no VS Build Tools install; the prebuilt contract held'
|
||||
|
||||
- name: Assert no source build
|
||||
if: always() && steps.install.outcome != 'skipped'
|
||||
shell: pwsh
|
||||
run: |
|
||||
# The step above only catches a NEW CMake or VS Build Tools install. The
|
||||
# image's Visual Studio survives a PATH scrub: setup.ps1's Find-VsBuildTools
|
||||
# (763-800) reaches it through vswhere and a Program Files scan, and the
|
||||
# visible leg logs `vs Visual Studio 18 2026 (vswhere)` on the same machine
|
||||
# whose pre-flight printed `cl ABSENT`. So a dependency that lost its Windows
|
||||
# wheel would compile against that MSVC and the leg would stay green, while
|
||||
# macOS and Linux caught it. uv really does build sdists here (openai-whisper,
|
||||
# antlr4-python3-runtime, randomname, argbind), so this is the live path.
|
||||
& "$env:GITHUB_WORKSPACE/.github/scripts/assert-nobuild.ps1" -LogPath logs/install.log
|
||||
|
||||
- name: Assert torch loads, and record what that does and does not prove
|
||||
if: steps.install.outcome == 'success'
|
||||
shell: pwsh
|
||||
|
|
|
|||
|
|
@ -534,11 +534,29 @@ jobs:
|
|||
run: |
|
||||
$drop = @('hostedtoolcache\windows\Python', 'WindowsApps', '\Git\', 'CMake',
|
||||
'Microsoft Visual Studio', 'BuildTools', 'LLVM', 'MSYS', 'mingw')
|
||||
# winget is an app-execution alias under ...\Local\Microsoft\WindowsApps, so
|
||||
# the WindowsApps fragment -- there to take the Store's python.exe alias away
|
||||
# -- drops the OS package manager with it. winget is not developer tooling;
|
||||
# every consumer Windows machine this bundle ships to has it, and the bundled
|
||||
# install.ps1 reaches for it for the git that studio/setup.ps1:1657-1669 still
|
||||
# gates on unconditionally. Without it this lane only re-runs the no-winget
|
||||
# fallback that clean-machine-install-ci.yml already covers and pins on its
|
||||
# winget=masked row, and it does so as a hard failure. Resolve winget before
|
||||
# the scrub and hand it back through a shim, exactly as that workflow does.
|
||||
$wingetCmd = Get-Command winget -ErrorAction SilentlyContinue
|
||||
if (-not $wingetCmd) {
|
||||
Write-Host '::error::winget was not on PATH before the strip; this image ships it and the bundled installer needs it'
|
||||
exit 1
|
||||
}
|
||||
$shim = Join-Path $env:RUNNER_TEMP 'winget-shim'
|
||||
New-Item -ItemType Directory -Force -Path $shim | Out-Null
|
||||
Set-Content -LiteralPath (Join-Path $shim 'winget.cmd') -Encoding ascii `
|
||||
-Value "@`"$($wingetCmd.Source)`" %*"
|
||||
$scrub = {
|
||||
param($entries)
|
||||
,@($entries | Where-Object { $p = $_; $p -and -not ($drop | Where-Object { $p -like "*$_*" }) })
|
||||
}
|
||||
"PATH=$((& $scrub ($env:PATH -split ';')) -join ';')" |
|
||||
"PATH=$shim;$((& $scrub ($env:PATH -split ';')) -join ';')" |
|
||||
Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
# Take the toolcache Python off disk, not just off PATH: py.exe lives in
|
||||
# C:\Windows (which must stay) and uv does its own interpreter discovery,
|
||||
|
|
@ -551,9 +569,11 @@ jobs:
|
|||
}
|
||||
}
|
||||
# The bundled install.ps1 this job runs calls Refresh-SessionPath (318-337),
|
||||
# which rebuilds $env:Path from the Machine and User registry values, so a
|
||||
# which merges the Machine and User registry PATHs back into $env:Path, so a
|
||||
# process-only scrub lasts until the first refresh and Git/CMake/VS/LLVM come
|
||||
# back. The runner is ephemeral, so rewrite the registry copies too. Expand
|
||||
# back from the registry. The runner is ephemeral, so rewrite the registry
|
||||
# copies too. (A merge keeps what the process already had, which is why the
|
||||
# winget shim above survives.) Expand
|
||||
# first: SetEnvironmentVariable rewrites REG_EXPAND_SZ as REG_SZ
|
||||
# (dotnet/runtime#1442).
|
||||
foreach ($scope in 'Machine','User') {
|
||||
|
|
@ -604,6 +624,17 @@ jobs:
|
|||
# machine that is in fact clean.
|
||||
$global:LASTEXITCODE = 0
|
||||
}
|
||||
# The shim is the only reason winget resolves after the WindowsApps drop. It
|
||||
# survives the installer's own refreshes because Refresh-SessionPath
|
||||
# (install.ps1:318-337) and setup.ps1's Refresh-Environment MERGE the current
|
||||
# $env:Path back in rather than replace it -- but assert it, or this lane
|
||||
# silently degrades into the no-winget leg the installer workflow already pins.
|
||||
$winget = Get-Command winget -ErrorAction SilentlyContinue
|
||||
Write-Host ("winget {0}" -f $(if ($winget) { $winget.Source } else { 'ABSENT' }))
|
||||
if (-not $winget) {
|
||||
Write-Host '::error::winget did not survive the strip; the bundled installer would take the no-winget fallback instead of the consumer path'
|
||||
exit 1
|
||||
}
|
||||
if ($leaked) {
|
||||
Write-Host "::error::developer tooling survived the strip: $($leaked -join '; ')"
|
||||
exit 1
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue