fix(studio): reuse venv Python in setup instead of re-probing system (#6033)
* fix(studio): reuse venv Python in setup instead of re-probing system * Reuse venv Python for studio setup Pass the venv interpreter from install.ps1 to studio/setup.ps1 via UNSLOTH_SETUP_PYTHON and prefer it over probing the system. Added Resolve-ReusedSetupPython to accept the handed-off path (or derive the venv python when setup runs standalone), validate it (Python 3.11–3.13 and non-conda), and inject its Scripts dir onto PATH. When a reused interpreter is accepted, py.exe enumeration and further system probing are skipped. install.ps1 also sets the env var before running setup and removes it on cleanup to avoid leaving state behind. This prevents setup from being tripped by unsupported Python 3.14 or Windows Store stubs on PATH. * Harden setup Python detection for PR #6033: py -All, shared conda check, bare ~ guard --------- Co-authored-by: Daniel Han <danielhanchen@gmail.com> Co-authored-by: Lee Jackson <130007945+Imagineer99@users.noreply.github.com>
This commit is contained in:
Etherll
GitHub
parent
de8d9b0691
commit
582fb0a0ce
2 changed files with 90 additions and 48 deletions
17
install.ps1
17
install.ps1
|
|
@ -983,10 +983,13 @@ shell.Run cmd, 0, False
|
|||
function Find-CompatiblePython {
|
||||
# Try the Python Launcher first (most reliable on Windows)
|
||||
# py.exe resolves to the standard CPython install, not conda.
|
||||
$pyLauncher = Get-Command py -CommandType Application -ErrorAction SilentlyContinue
|
||||
if ($pyLauncher -and $pyLauncher.Source -notmatch $script:CondaSkipPattern) {
|
||||
# Prefer the requested $PythonVersion, then newest-first fallback.
|
||||
$minors = @($PythonVersion) + (@("3.13", "3.12", "3.11") | Where-Object { $_ -ne $PythonVersion })
|
||||
# Prefer the requested $PythonVersion, then newest-first fallback.
|
||||
$minors = @($PythonVersion) + (@("3.13", "3.12", "3.11") | Where-Object { $_ -ne $PythonVersion })
|
||||
# Enumerate every py.exe on PATH with -All (Windows PowerShell 5.1
|
||||
# returns only the first launcher without it) and search each for a
|
||||
# supported, non-conda interpreter.
|
||||
foreach ($pyLauncher in @(Get-Command py -All -CommandType Application -ErrorAction SilentlyContinue)) {
|
||||
if ($pyLauncher.Source -match $script:CondaSkipPattern) { continue }
|
||||
foreach ($minor in $minors) {
|
||||
try {
|
||||
$out = & $pyLauncher.Source "-$minor" --version 2>&1 | Out-String
|
||||
|
|
@ -2089,6 +2092,11 @@ shell.Run cmd, 0, False
|
|||
$studioArgs = @('studio', 'setup')
|
||||
if ($script:UnslothVerbose) { $studioArgs += '--verbose' }
|
||||
$env:UNSLOTH_INSTALL_ROLLBACK_MANAGED = "1"
|
||||
# Hand the venv interpreter to setup.ps1 so it reuses the Python we already
|
||||
# resolved and built the venv with, instead of re-probing the system (which
|
||||
# can trip over an unsupported `python` 3.14 or a Store stub on PATH even
|
||||
# though the venv is fine). setup.ps1 Test-Path-guards this before use.
|
||||
$env:UNSLOTH_SETUP_PYTHON = Join-Path $VenvDir "Scripts\python.exe"
|
||||
try {
|
||||
& $UnslothExe @studioArgs
|
||||
$setupExit = $LASTEXITCODE
|
||||
|
|
@ -2099,6 +2107,7 @@ shell.Run cmd, 0, False
|
|||
Remove-Item Env:UNSLOTH_STUDIO_HOME -ErrorAction SilentlyContinue
|
||||
}
|
||||
Remove-Item Env:UNSLOTH_INSTALL_ROLLBACK_MANAGED -ErrorAction SilentlyContinue
|
||||
Remove-Item Env:UNSLOTH_SETUP_PYTHON -ErrorAction SilentlyContinue
|
||||
}
|
||||
if ($setupExit -ne 0) {
|
||||
Write-Host "[ERROR] unsloth studio setup failed (exit code $setupExit)" -ForegroundColor Red
|
||||
|
|
|
|||
121
studio/setup.ps1
121
studio/setup.ps1
|
|
@ -1552,10 +1552,48 @@ if ($IsPipInstall) {
|
|||
}
|
||||
}
|
||||
|
||||
# 1g. Python (>= 3.11 and < 3.14). Prefer the Studio venv that install.ps1
|
||||
# just created, then py.exe so a 3.14 ahead of 3.13 on PATH does not trip the gate.
|
||||
# Conda CPython ships modified DLL search paths that break torch's c10.dll
|
||||
# loading on Windows; a venv made from conda Python inherits its base_prefix,
|
||||
# so check the executable path AND sys.base_prefix.
|
||||
$CondaSkipPattern = '(?i)(conda|miniconda|anaconda|miniforge|mambaforge)'
|
||||
function Test-IsConda {
|
||||
param([string]$Exe)
|
||||
if ($Exe -match $CondaSkipPattern) { return $true }
|
||||
try {
|
||||
$basePrefix = (& $Exe -c "import sys; print(sys.base_prefix)" 2>$null | Out-String).Trim()
|
||||
if ($basePrefix -match $CondaSkipPattern) { return $true }
|
||||
} catch { }
|
||||
return $false
|
||||
}
|
||||
|
||||
# 1g. Python (>= 3.11 and < 3.14). Prefer the interpreter install.ps1 already
|
||||
# resolved and built the venv with (UNSLOTH_SETUP_PYTHON), or the existing
|
||||
# venv python, before re-probing a system where a 3.14 or a WindowsApps stub
|
||||
# ahead on PATH would trip the gate. setup.ps1 only updates packages in that
|
||||
# venv, so the handoff is safe to reuse once validated.
|
||||
function Resolve-ReusedSetupPython {
|
||||
if (-not [string]::IsNullOrWhiteSpace($env:UNSLOTH_SETUP_PYTHON) -and
|
||||
(Test-Path -LiteralPath $env:UNSLOTH_SETUP_PYTHON)) {
|
||||
return $env:UNSLOTH_SETUP_PYTHON
|
||||
}
|
||||
# Standalone `unsloth studio setup/update` (install.ps1 did not run): derive
|
||||
# the venv python from the studio root, mirroring the resolver below.
|
||||
$root = if (-not [string]::IsNullOrWhiteSpace($env:UNSLOTH_STUDIO_HOME)) { $env:UNSLOTH_STUDIO_HOME.Trim() }
|
||||
elseif (-not [string]::IsNullOrWhiteSpace($env:STUDIO_HOME)) { $env:STUDIO_HOME.Trim() }
|
||||
else { Join-Path $env:USERPROFILE ".unsloth\studio" }
|
||||
if ($root -eq "~") {
|
||||
# Join-Path with an empty child throws on Windows PowerShell 5.1.
|
||||
$root = $env:USERPROFILE
|
||||
} elseif ($root -like "~/*" -or $root -like "~\*") {
|
||||
$root = Join-Path $env:USERPROFILE $root.Substring(1).TrimStart('/', '\')
|
||||
}
|
||||
$venvPy = Join-Path $root "unsloth_studio\Scripts\python.exe"
|
||||
if (Test-Path -LiteralPath $venvPy) { return $venvPy }
|
||||
return $null
|
||||
}
|
||||
$ReusedSetupPython = Resolve-ReusedSetupPython
|
||||
|
||||
$HasPython = $null -ne (Get-Command python -ErrorAction SilentlyContinue)
|
||||
$PyLauncher = Get-Command py -CommandType Application -ErrorAction SilentlyContinue
|
||||
$PythonOk = $false
|
||||
$DetectedPyVer = $null
|
||||
|
||||
|
|
@ -1584,28 +1622,24 @@ function Add-PythonDirToProcessPath {
|
|||
} catch { }
|
||||
}
|
||||
|
||||
$_prereqStudioHome = $null
|
||||
if (-not [string]::IsNullOrWhiteSpace($env:UNSLOTH_STUDIO_HOME)) {
|
||||
$_prereqStudioHome = $env:UNSLOTH_STUDIO_HOME.Trim()
|
||||
} elseif (-not [string]::IsNullOrWhiteSpace($env:STUDIO_HOME)) {
|
||||
$_prereqStudioHome = $env:STUDIO_HOME.Trim()
|
||||
} else {
|
||||
$_prereqStudioHome = Join-Path $env:USERPROFILE ".unsloth\studio"
|
||||
}
|
||||
if ($_prereqStudioHome -eq "~" -or $_prereqStudioHome -like "~/*" -or $_prereqStudioHome -like "~\*") {
|
||||
$_prereqStudioHome = (Join-Path $env:USERPROFILE $_prereqStudioHome.Substring(1).TrimStart('/','\'))
|
||||
}
|
||||
$_prereqVenvPython = Join-Path $_prereqStudioHome "unsloth_studio\Scripts\python.exe"
|
||||
if (Test-Path -LiteralPath $_prereqVenvPython) {
|
||||
$_venvPyVer = Get-CompatiblePythonVersion $_prereqVenvPython
|
||||
if ($_venvPyVer) {
|
||||
$DetectedPyVer = $_venvPyVer
|
||||
Add-PythonDirToProcessPath $_prereqVenvPython
|
||||
# Reuse the install.ps1 / venv interpreter before any system probe.
|
||||
if ($ReusedSetupPython) {
|
||||
$_reusedVer = Get-CompatiblePythonVersion $ReusedSetupPython
|
||||
if ($_reusedVer -and -not (Test-IsConda $ReusedSetupPython)) {
|
||||
$DetectedPyVer = $_reusedVer
|
||||
Add-PythonDirToProcessPath $ReusedSetupPython
|
||||
$PythonOk = $true
|
||||
}
|
||||
}
|
||||
|
||||
if (-not $PythonOk -and $PyLauncher) {
|
||||
# Fall back to every py.exe on PATH (all-users and per-user launchers can both
|
||||
# register). -All is required: Windows PowerShell 5.1 returns only the first
|
||||
# launcher without it, and the PowerShell 7 multi-match array breaks the call
|
||||
# operator if used directly.
|
||||
$PyLaunchers = if ($PythonOk) { @() } else { @(Get-Command py -All -CommandType Application -ErrorAction SilentlyContinue) }
|
||||
|
||||
foreach ($PyLauncher in $PyLaunchers) {
|
||||
if ($PyLauncher.Source -match $CondaSkipPattern) { continue }
|
||||
foreach ($minor in @("3.13", "3.12", "3.11")) {
|
||||
try {
|
||||
$out = & $PyLauncher.Source "-$minor" --version 2>&1 | Out-String
|
||||
|
|
@ -1625,6 +1659,7 @@ if (-not $PythonOk -and $PyLauncher) {
|
|||
}
|
||||
} catch { }
|
||||
}
|
||||
if ($PythonOk) { break }
|
||||
}
|
||||
|
||||
if (-not $PythonOk -and $HasPython) {
|
||||
|
|
@ -1857,36 +1892,33 @@ if (Test-Path $OxcValidatorDir) {
|
|||
Write-Host ""
|
||||
substep "setting up Python environment..."
|
||||
|
||||
# Find Python -- skip Anaconda/Miniconda distributions.
|
||||
# Conda-bundled CPython ships modified DLL search paths that break
|
||||
# torch's c10.dll loading on Windows. Standalone CPython (python.org,
|
||||
# winget, uv) does not have this issue.
|
||||
# Uses Get-Command -All to look past conda entries that shadow a valid
|
||||
# standalone Python further down PATH, and probes py.exe (the Python
|
||||
# Launcher) which reliably finds python.org installs.
|
||||
#
|
||||
# NOTE: A venv created from conda Python inherits conda's base_prefix
|
||||
# even though the venv path itself does not contain "conda". We check
|
||||
# both the executable path AND sys.base_prefix to catch this case.
|
||||
$CondaSkipPattern = '(?i)(conda|miniconda|anaconda|miniforge|mambaforge)'
|
||||
# Find Python -- skip Anaconda/Miniconda distributions ($CondaSkipPattern and
|
||||
# Test-IsConda are defined above the 1g gate). Standalone CPython (python.org,
|
||||
# winget, uv) does not have conda's torch c10.dll loading issue.
|
||||
$PythonCmd = $null
|
||||
|
||||
# Helper: check if a Python executable is conda-based by inspecting
|
||||
# both the path and sys.base_prefix (catches venvs created from conda).
|
||||
function Test-IsConda {
|
||||
param([string]$Exe)
|
||||
if ($Exe -match $CondaSkipPattern) { return $true }
|
||||
# 0. Reuse the interpreter install.ps1 already resolved and built the venv with
|
||||
# (UNSLOTH_SETUP_PYTHON, or the existing venv python) before probing the
|
||||
# system -- it is already validated as supported and non-conda.
|
||||
if ($ReusedSetupPython) {
|
||||
try {
|
||||
$basePrefix = (& $Exe -c "import sys; print(sys.base_prefix)" 2>$null | Out-String).Trim()
|
||||
if ($basePrefix -match $CondaSkipPattern) { return $true }
|
||||
$out = & $ReusedSetupPython --version 2>&1 | Out-String
|
||||
if ($out -match 'Python 3\.(\d+)') {
|
||||
$pyMinor = [int]$Matches[1]
|
||||
if ($pyMinor -ge 11 -and $pyMinor -le 13 -and -not (Test-IsConda $ReusedSetupPython)) {
|
||||
$PythonCmd = $ReusedSetupPython
|
||||
}
|
||||
}
|
||||
} catch { }
|
||||
return $false
|
||||
}
|
||||
|
||||
# 1. Try the Python Launcher (py.exe) first -- most reliable on Windows.
|
||||
# py.exe is installed by python.org and resolves to standalone CPython.
|
||||
$pyLauncher = Get-Command py -CommandType Application -ErrorAction SilentlyContinue
|
||||
if ($pyLauncher -and $pyLauncher.Source -notmatch $CondaSkipPattern) {
|
||||
# Enumerate every launcher with -All (Windows PowerShell 5.1 returns only
|
||||
# the first match without it) and search each for a supported, non-conda
|
||||
# interpreter.
|
||||
$PyLaunchersResolve = if ($PythonCmd) { @() } else { @(Get-Command py -All -CommandType Application -ErrorAction SilentlyContinue) }
|
||||
foreach ($pyLauncher in $PyLaunchersResolve) {
|
||||
if ($pyLauncher.Source -match $CondaSkipPattern) { continue }
|
||||
foreach ($minor in @("3.13", "3.12", "3.11")) {
|
||||
try {
|
||||
$out = & $pyLauncher.Source "-$minor" --version 2>&1 | Out-String
|
||||
|
|
@ -1904,6 +1936,7 @@ if ($pyLauncher -and $pyLauncher.Source -notmatch $CondaSkipPattern) {
|
|||
}
|
||||
} catch { }
|
||||
}
|
||||
if ($PythonCmd) { break }
|
||||
}
|
||||
|
||||
# 2. Fall back to scanning python3.x / python3 / python on PATH.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue