diff --git a/.github/scripts/assert-nobuild.ps1 b/.github/scripts/assert-nobuild.ps1 new file mode 100644 index 0000000000..64a1ba9813 --- /dev/null +++ b/.github/scripts/assert-nobuild.ps1 @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +# The `nobuild` contract from clean-machine-assert.sh, for Windows. +# +# Why a port and not `shell: bash`: the clean-machine scrub drops every `*\Git\*` +# entry from PATH and from the Machine/User registry copies, and the bash version +# needs sed/grep/tr/sort out of C:\Program Files\Git\usr\bin. This also runs inside +# the servercore container, which has no bash at all. Both Windows lanes call this +# one file so the sdist allowlist cannot drift between them. +# +# Usage: assert-nobuild.ps1 -LogPath logs/install.log (exit 1 = a source build) +[CmdletBinding()] +param([Parameter(Mandatory = $true)][string] $LogPath) + +if (-not (Test-Path -LiteralPath $LogPath)) { + Write-Host "::error::nobuild requested but $LogPath is missing" + exit 1 +} + +# "Built an sdist" is NOT "needed a compiler". Every name here was checked against +# its own sdist: setuptools.build_meta backend, no ext_modules, no .c/.cpp/.pyx/.rs +# file, so its PEP 517 build is a pure-Python copy step. UNSLOTH_ALLOW_SDIST extends +# the list. Kept identical to clean-machine-assert.sh's `_allow`. +$allow = @('openai-whisper', 'argbind', 'randomname', 'antlr4-python3-runtime', 'triton-kernels') +if ($env:UNSLOTH_ALLOW_SDIST) { + $allow += ($env:UNSLOTH_ALLOW_SDIST -split '\s+' | Where-Object { $_ }) +} +# Lowercased and underscore-folded on both sides: a distribution name and the name uv +# prints can disagree on the separator (triton_kernels vs triton-kernels). +$allow = @($allow | ForEach-Object { $_.ToLowerInvariant() -replace '_', '-' }) + +# [char]27, not "`e": the `e escape is PowerShell 6+, and this runs under Windows +# PowerShell 5.1 too, where "`e" degrades to a literal "e" and the strip would eat +# real text instead of ANSI codes. +$esc = [char]27 +$text = (Get-Content -LiteralPath $LogPath -Raw) -replace "$esc\[[0-9;]*[A-Za-z]", '' +$built = @() +foreach ($line in ($text -split "`r?`n")) { + # A local-path build is something the caller pointed at (the CI source overlay), + # never something dependency resolution chose. Index dependencies always print + # `==`, so no signal is lost. + if ($line -imatch 'building [a-z0-9._-]+ @ file://') { continue } + # pip prints `Building wheel for `, uv prints `Building ==` + # (astral-sh/uv#11165). Requiring `==` or ` @ ` after the name keeps this off the + # installer's own lowercase "building frontend..." progress text. + foreach ($m in [regex]::Matches($line, '(?i)building wheel for ([a-z0-9._-]+)|building ([a-z0-9._-]+)(==| @ )')) { + $name = if ($m.Groups[1].Success) { $m.Groups[1].Value } else { $m.Groups[2].Value } + $built += ($name.ToLowerInvariant() -replace '_', '-') + } +} +$built = @($built | Sort-Object -Unique) +$bad = @($built | Where-Object { $allow -notcontains $_ }) + +$rc = 0 +if ($bad.Count -gt 0) { + Write-Host "::error::built from source: $($bad -join ' ') -- these must resolve to wheels on a clean machine" + $rc = 1 +} else { + Write-Host "[assert] OK no non-allowlisted source build (built: $(if ($built) { $built -join ' ' } else { 'none' }))" +} +# Independent of package names: a compiler error means a toolchain was needed. +$compilerErr = Select-String -Path $LogPath -Pattern "error: command '(cc|gcc|clang|cl)' failed", 'clang: error', 'cargo: not found', 'Microsoft Visual C\+\+ 14.0 or greater is required' +if ($compilerErr) { + Write-Host '::error::compiler invocation appears in the install log' + $compilerErr | Select-Object -First 10 | ForEach-Object { Write-Host " $($_.Line)" } + $rc = 1 +} +exit $rc diff --git a/.github/scripts/virgin-windows-install.ps1 b/.github/scripts/virgin-windows-install.ps1 index 2f89759c6c..2d7b7a6289 100644 --- a/.github/scripts/virgin-windows-install.ps1 +++ b/.github/scripts/virgin-windows-install.ps1 @@ -132,40 +132,14 @@ if ($Overlay -and $rc -eq 0) { } Section 'assert: no non-allowlisted source build' -# PowerShell port of .github/scripts/clean-machine-assert.sh's `nobuild`. Same -# contract: pip prints "Building wheel for ", uv prints "Building ==" -# (astral-sh/uv#11165), and a local-path build (`Building @ file://`) is -# something the caller pointed at, never something resolution chose. -if (-not (Test-Path -LiteralPath $LogPath)) { - $failures += "nobuild requested but $LogPath is missing" +# Shared with the hosted Windows legs so the sdist allowlist lives in one place; the +# script prints its own diagnosis, so only the verdict is folded in here. +$nobuild = Join-Path $PSScriptRoot 'assert-nobuild.ps1' +if (-not (Test-Path -LiteralPath $nobuild)) { + $failures += "assert-nobuild.ps1 is missing next to this script, so the no-build contract went unchecked" } else { - $allow = @('openai-whisper', 'argbind', 'randomname', 'antlr4-python3-runtime', 'triton-kernels') - # [char]27, not "`e": the `e escape sequence is PowerShell 6+, and this script runs - # under Windows PowerShell 5.1, where "`e" silently degrades to a literal "e" and - # the strip would eat real text instead of ANSI codes. - $esc = [char]27 - $text = (Get-Content -LiteralPath $LogPath -Raw) -replace "$esc\[[0-9;]*[A-Za-z]", '' - $built = @() - foreach ($line in ($text -split "`r?`n")) { - if ($line -imatch 'building [a-z0-9._-]+ @ file://') { continue } - foreach ($m in [regex]::Matches($line, '(?i)building wheel for ([a-z0-9._-]+)|building ([a-z0-9._-]+)(==| @ )')) { - $name = if ($m.Groups[1].Success) { $m.Groups[1].Value } else { $m.Groups[2].Value } - $built += ($name.ToLowerInvariant() -replace '_', '-') - } - } - $built = $built | Sort-Object -Unique - $bad = @($built | Where-Object { $allow -notcontains $_ }) - if ($bad.Count -gt 0) { - $failures += "built from source: $($bad -join ' ') -- these must resolve to wheels on a clean machine" - } else { - Write-Host "no non-allowlisted source build (built: $(if ($built) { $built -join ' ' } else { 'none' }))" - } - # Independent of package names: a compiler error means a toolchain was needed. - $compilerErr = Select-String -Path $LogPath -Pattern "error: command '(cc|gcc|clang|cl)' failed", 'clang: error', 'cargo: not found', 'Microsoft Visual C\+\+ 14.0 or greater is required' - if ($compilerErr) { - $failures += "compiler invocation appears in the install log" - $compilerErr | Select-Object -First 10 | ForEach-Object { Write-Host " $($_.Line)" } - } + & $nobuild -LogPath $LogPath + if ($LASTEXITCODE -ne 0) { $failures += "a non-allowlisted source build appears in the install log" } } # ── Verdict ─────────────────────────────────────────────────────────────────── diff --git a/.github/workflows/clean-machine-install-ci.yml b/.github/workflows/clean-machine-install-ci.yml index db7f966bc7..6cc469c74c 100644 --- a/.github/workflows/clean-machine-install-ci.yml +++ b/.github/workflows/clean-machine-install-ci.yml @@ -66,6 +66,7 @@ on: # The virgin Windows container lane lives in this workflow too. - '.github/scripts/virgin-windows-*.ps1' - '.github/scripts/ensure-docker-daemon.ps1' + - '.github/scripts/assert-nobuild.ps1' - '.github/workflows/clean-machine-install-ci.yml' push: branches: [main] @@ -84,6 +85,7 @@ on: # The virgin Windows container lane lives in this workflow too. - '.github/scripts/virgin-windows-*.ps1' - '.github/scripts/ensure-docker-daemon.ps1' + - '.github/scripts/assert-nobuild.ps1' - '.github/workflows/clean-machine-install-ci.yml' workflow_dispatch: inputs: @@ -1021,10 +1023,11 @@ jobs: $newPath = ($kept -join ';') "PATH=$newPath" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 # install.ps1's Refresh-SessionPath (318-337, called at 1246/1278/1295/1360/ - # 1369/2797) rebuilds $env:Path from the Machine and User registry values, so a - # process-only scrub lasts until the first bootstrap refresh, after which + # 1369/2797) merges the Machine and User registry PATHs back into $env:Path, so + # a process-only scrub lasts until the first bootstrap refresh, after which # Git/CMake/VS/LLVM are back and the rest of the install is not clean. The - # runner is ephemeral, so rewrite the registry copies too. Expand first: + # runner is ephemeral, so rewrite the registry copies too. It is a merge, not a + # replace, so the shim above keeps resolving. Expand first: # SetEnvironmentVariable rewrites REG_EXPAND_SZ as REG_SZ # (dotnet/runtime#1442). foreach ($scope in 'Machine','User') { @@ -1295,6 +1298,20 @@ jobs: } Write-Host 'no CMake and no VS Build Tools install; the prebuilt contract held' + - name: Assert no source build + if: always() && steps.install.outcome != 'skipped' + shell: pwsh + run: | + # The step above only catches a NEW CMake or VS Build Tools install. The + # image's Visual Studio survives a PATH scrub: setup.ps1's Find-VsBuildTools + # (763-800) reaches it through vswhere and a Program Files scan, and the + # visible leg logs `vs Visual Studio 18 2026 (vswhere)` on the same machine + # whose pre-flight printed `cl ABSENT`. So a dependency that lost its Windows + # wheel would compile against that MSVC and the leg would stay green, while + # macOS and Linux caught it. uv really does build sdists here (openai-whisper, + # antlr4-python3-runtime, randomname, argbind), so this is the live path. + & "$env:GITHUB_WORKSPACE/.github/scripts/assert-nobuild.ps1" -LogPath logs/install.log + - name: Assert torch loads, and record what that does and does not prove if: steps.install.outcome == 'success' shell: pwsh diff --git a/.github/workflows/desktop-app-clean-machine-ci.yml b/.github/workflows/desktop-app-clean-machine-ci.yml index ee332dfa72..acb695f9bf 100644 --- a/.github/workflows/desktop-app-clean-machine-ci.yml +++ b/.github/workflows/desktop-app-clean-machine-ci.yml @@ -534,11 +534,29 @@ jobs: run: | $drop = @('hostedtoolcache\windows\Python', 'WindowsApps', '\Git\', 'CMake', 'Microsoft Visual Studio', 'BuildTools', 'LLVM', 'MSYS', 'mingw') + # winget is an app-execution alias under ...\Local\Microsoft\WindowsApps, so + # the WindowsApps fragment -- there to take the Store's python.exe alias away + # -- drops the OS package manager with it. winget is not developer tooling; + # every consumer Windows machine this bundle ships to has it, and the bundled + # install.ps1 reaches for it for the git that studio/setup.ps1:1657-1669 still + # gates on unconditionally. Without it this lane only re-runs the no-winget + # fallback that clean-machine-install-ci.yml already covers and pins on its + # winget=masked row, and it does so as a hard failure. Resolve winget before + # the scrub and hand it back through a shim, exactly as that workflow does. + $wingetCmd = Get-Command winget -ErrorAction SilentlyContinue + if (-not $wingetCmd) { + Write-Host '::error::winget was not on PATH before the strip; this image ships it and the bundled installer needs it' + exit 1 + } + $shim = Join-Path $env:RUNNER_TEMP 'winget-shim' + New-Item -ItemType Directory -Force -Path $shim | Out-Null + Set-Content -LiteralPath (Join-Path $shim 'winget.cmd') -Encoding ascii ` + -Value "@`"$($wingetCmd.Source)`" %*" $scrub = { param($entries) ,@($entries | Where-Object { $p = $_; $p -and -not ($drop | Where-Object { $p -like "*$_*" }) }) } - "PATH=$((& $scrub ($env:PATH -split ';')) -join ';')" | + "PATH=$shim;$((& $scrub ($env:PATH -split ';')) -join ';')" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 # Take the toolcache Python off disk, not just off PATH: py.exe lives in # C:\Windows (which must stay) and uv does its own interpreter discovery, @@ -551,9 +569,11 @@ jobs: } } # The bundled install.ps1 this job runs calls Refresh-SessionPath (318-337), - # which rebuilds $env:Path from the Machine and User registry values, so a + # which merges the Machine and User registry PATHs back into $env:Path, so a # process-only scrub lasts until the first refresh and Git/CMake/VS/LLVM come - # back. The runner is ephemeral, so rewrite the registry copies too. Expand + # back from the registry. The runner is ephemeral, so rewrite the registry + # copies too. (A merge keeps what the process already had, which is why the + # winget shim above survives.) Expand # first: SetEnvironmentVariable rewrites REG_EXPAND_SZ as REG_SZ # (dotnet/runtime#1442). foreach ($scope in 'Machine','User') { @@ -604,6 +624,17 @@ jobs: # machine that is in fact clean. $global:LASTEXITCODE = 0 } + # The shim is the only reason winget resolves after the WindowsApps drop. It + # survives the installer's own refreshes because Refresh-SessionPath + # (install.ps1:318-337) and setup.ps1's Refresh-Environment MERGE the current + # $env:Path back in rather than replace it -- but assert it, or this lane + # silently degrades into the no-winget leg the installer workflow already pins. + $winget = Get-Command winget -ErrorAction SilentlyContinue + Write-Host ("winget {0}" -f $(if ($winget) { $winget.Source } else { 'ABSENT' })) + if (-not $winget) { + Write-Host '::error::winget did not survive the strip; the bundled installer would take the no-winget fallback instead of the consumer path' + exit 1 + } if ($leaked) { Write-Host "::error::developer tooling survived the strip: $($leaked -join '; ')" exit 1