fix: clearer Studio setup error when GPU driver is too old for the installed CUDA toolkit (#5993)
* fix: clearer Studio setup error when GPU driver is too old for the installed CUDA toolkit * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Honor optional color arg in setup.sh substep so driver/toolkit warnings render in C_WARN * Add regression test that setup.sh _cuda_version_gt compares numerically for PR #5993 * Update Resolve-CudaToolkit test for the new driver-too-old messaging setup.ps1 now routes the too-new-toolkit case through Write-CudaDriverToolkitMismatch instead of the old 'is installed but INCOMPATIBLE' banner. Extract that helper alongside Resolve-CudaToolkit so the child pwsh can run it, and assert the new driver-too-old guidance (and the one-line source-build error) instead of the removed INCOMPATIBLE text. * Address review nits: document Windows hard-exit asymmetry and add toolkit/driver edge tests setup.ps1: note that only a forced source build reaches the hard-exit branch (the prebuilt path returned above), unlike setup.sh which degrades to CPU. test_selection_logic.py: cover the CUDA UMD Version variant, the empty nvcc version guard, and the too_old (< 12.4) short-circuit. * fix(studio): allow CUDA minor-version compat and try installed toolkits before CPU fallback The driver check now compares CUDA major versions only, per NVIDIA minor-version compatibility, and when the selected nvcc is still too new the setup iterates other installed toolkits and uses the newest driver-compatible one before falling back to a CPU llama.cpp build. Same rule mirrored in setup.ps1. * style: apply ruff kwarg-spacing format after rebase * Accept a same-major CUDA toolkit found only on PATH in the Windows fallback The major-only compatibility fix updated the side-by-side scan (Find-Nvcc -MaxVersion) and the CUDA_PATH check, but the fallback that runs when Find-Nvcc -MaxVersion returns null still recorded any plain Find-Nvcc result as an incompatible toolkit without re-checking the major. A same-major toolkit discoverable only via PATH, process CUDA_PATH, or a custom location (e.g. toolkit 13.3 with a driver supporting CUDA 13.2) was therefore rejected even though it is compatible. Re-apply the same major-only rule in the fallback: use the toolkit when its major is within the driver's, otherwise record it as too-new. Adds a regression test covering the PATH-only same-major case. * Add CUDA driver/toolkit selection edge-case tests for Studio setup * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Tighten comments in Studio CUDA driver/toolkit setup Collapse multi-line comments, drop obvious ones, keep the load-bearing intent (the major-compat invariant, the Windows hard-exit vs setup.sh-CPU asymmetry, the PATH-only fallback rationale). Comment-only; no code change. * Clarify the Windows source-build hard-exit comment The path is reached by any committed source build (forced, or after a prebuilt-install failure), not only a forced one. Comment-only. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Drop two obvious comments in setup.ps1 CUDA detection --------- Co-authored-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Daniel Han <danielhanchen@gmail.com>
This commit is contained in:
Matt Van Horn
GitHub
parent
4d2f29ff2a
commit
5f622f6c2f
4 changed files with 684 additions and 85 deletions
|
|
@ -220,14 +220,10 @@ function Get-InstalledLlamaPrebuiltRelease {
|
|||
function Find-Nvcc {
|
||||
param([string]$MaxVersion = "")
|
||||
|
||||
# If MaxVersion is set, we need to find a toolkit <= that version.
|
||||
# CUDA toolkits install side-by-side under C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\vX.Y\
|
||||
|
||||
$toolkitBase = 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA'
|
||||
|
||||
if ($MaxVersion -and (Test-Path $toolkitBase)) {
|
||||
$drMajor = [int]$MaxVersion.Split('.')[0]
|
||||
$drMinor = [int]$MaxVersion.Split('.')[1]
|
||||
|
||||
# Get all installed CUDA dirs, sorted descending (highest first)
|
||||
$cudaDirs = Get-ChildItem -Directory $toolkitBase | Where-Object {
|
||||
|
|
@ -236,8 +232,8 @@ function Find-Nvcc {
|
|||
|
||||
foreach ($dir in $cudaDirs) {
|
||||
if ($dir.Name -match '^v(\d+)\.(\d+)') {
|
||||
$tkMajor = [int]$Matches[1]; $tkMinor = [int]$Matches[2]
|
||||
$compatible = ($tkMajor -lt $drMajor) -or ($tkMajor -eq $drMajor -and $tkMinor -le $drMinor)
|
||||
$tkMajor = [int]$Matches[1]
|
||||
$compatible = ($tkMajor -le $drMajor)
|
||||
if ($compatible) {
|
||||
$nvcc = Join-Path $dir.FullName 'bin\nvcc.exe'
|
||||
if (Test-Path $nvcc) {
|
||||
|
|
@ -278,6 +274,19 @@ function Find-Nvcc {
|
|||
return $null
|
||||
}
|
||||
|
||||
function Write-CudaDriverToolkitMismatch {
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$ToolkitVersion,
|
||||
[Parameter(Mandatory = $true)][string]$DriverMaxCuda,
|
||||
[string]$Color = "Yellow"
|
||||
)
|
||||
$toolkitMajor = $ToolkitVersion.Split('.')[0]
|
||||
$driverMajor = $DriverMaxCuda.Split('.')[0]
|
||||
substep "CUDA Toolkit $ToolkitVersion is a major-version mismatch: toolkit major $toolkitMajor exceeds driver CUDA major $driverMajor ($DriverMaxCuda)." $Color
|
||||
substep "Update the NVIDIA GPU driver to run CUDA Toolkit $ToolkitVersion, or install a CUDA $driverMajor.x toolkit." $Color
|
||||
substep "Or let Studio use the prebuilt CUDA bundle; it does not need the local toolkit." $Color
|
||||
}
|
||||
|
||||
# Detect CUDA Compute Capability via nvidia-smi.
|
||||
# Returns e.g. "80" for A100 (8.0), "89" for RTX 4090 (8.9), etc.
|
||||
# Returns $null if detection fails.
|
||||
|
|
@ -1062,17 +1071,13 @@ if ($vsResult) {
|
|||
# or installed. Without it, detection is best-effort and only sets the flag.
|
||||
function Resolve-CudaToolkit {
|
||||
param([switch]$RequireOrExit)
|
||||
# IMPORTANT: The CUDA Toolkit version must be <= the max CUDA version the
|
||||
# NVIDIA driver supports. nvidia-smi reports this as "CUDA Version: X.Y".
|
||||
# If we install a toolkit newer than the driver supports, llama-server will
|
||||
# fail at runtime with "ggml_cuda_init: failed to initialize CUDA: (null)".
|
||||
# Toolkit major must be <= the driver's max CUDA major (nvidia-smi "CUDA Version: X.Y");
|
||||
# a newer-major toolkit fails at runtime ("ggml_cuda_init: failed to initialize CUDA").
|
||||
|
||||
# -- Detect max CUDA version the driver supports --
|
||||
$DriverMaxCuda = $null
|
||||
try {
|
||||
$smiOut = & $NvidiaSmiExe 2>&1 | Out-String
|
||||
# Newer NVIDIA drivers (e.g. 610.x) report the driver max CUDA as
|
||||
# "CUDA UMD Version: X.Y" rather than "CUDA Version: X.Y"; accept both.
|
||||
# Newer drivers report "CUDA UMD Version: X.Y" instead of "CUDA Version: X.Y"; accept both.
|
||||
if ($smiOut -match "CUDA(?: UMD)? Version:\s+([\d]+)\.([\d]+)") {
|
||||
$DriverMaxCuda = "$($Matches[1]).$($Matches[2])"
|
||||
substep "driver supports up to CUDA $DriverMaxCuda"
|
||||
|
|
@ -1096,7 +1101,6 @@ $NvccPath = $null
|
|||
|
||||
if ($DriverMaxCuda) {
|
||||
$drMajorCuda = [int]$DriverMaxCuda.Split('.')[0]
|
||||
$drMinorCuda = [int]$DriverMaxCuda.Split('.')[1]
|
||||
|
||||
# --- Step 1: Check existing CUDA_PATH first ---
|
||||
$existingCudaPath = [Environment]::GetEnvironmentVariable('CUDA_PATH', 'Machine')
|
||||
|
|
@ -1108,7 +1112,7 @@ if ($DriverMaxCuda) {
|
|||
$verOut = & $candidateNvcc --version 2>&1 | Out-String
|
||||
if ($verOut -match 'release\s+(\d+)\.(\d+)') {
|
||||
$tkMaj = [int]$Matches[1]; $tkMin = [int]$Matches[2]
|
||||
$isCompat = ($tkMaj -lt $drMajorCuda) -or ($tkMaj -eq $drMajorCuda -and $tkMin -le $drMinorCuda)
|
||||
$isCompat = ($tkMaj -le $drMajorCuda)
|
||||
if ($isCompat) {
|
||||
# Also verify the toolkit supports our GPU architecture
|
||||
$archOk = $true
|
||||
|
|
@ -1124,7 +1128,7 @@ if ($DriverMaxCuda) {
|
|||
substep "using existing CUDA Toolkit at CUDA_PATH (nvcc: $NvccPath)"
|
||||
}
|
||||
} else {
|
||||
substep "CUDA_PATH ($existingCudaPath) has CUDA $tkMaj.$tkMin which exceeds driver max $DriverMaxCuda" "Yellow"
|
||||
substep "CUDA_PATH ($existingCudaPath) has CUDA $tkMaj.$tkMin with major $tkMaj, which exceeds driver CUDA major $drMajorCuda ($DriverMaxCuda)" "Yellow"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1141,12 +1145,19 @@ if ($DriverMaxCuda) {
|
|||
}
|
||||
}
|
||||
} else {
|
||||
# Check if there's an incompatible (too new) toolkit installed
|
||||
# No side-by-side match: a major-compatible toolkit may still be on
|
||||
# PATH/CUDA_PATH/a custom dir; use it, else record it as too-new.
|
||||
$AnyNvcc = Find-Nvcc
|
||||
if ($AnyNvcc) {
|
||||
$NvccOut = & $AnyNvcc --version 2>&1 | Out-String
|
||||
if ($NvccOut -match "release\s+([\d]+\.[\d]+)") {
|
||||
$IncompatibleToolkit = $Matches[1]
|
||||
if ($NvccOut -match "release\s+(\d+)\.(\d+)") {
|
||||
$tkMaj = [int]$Matches[1]; $tkMin = [int]$Matches[2]
|
||||
if ($tkMaj -le $drMajorCuda) {
|
||||
$NvccPath = $AnyNvcc
|
||||
substep "found compatible CUDA Toolkit (nvcc: $NvccPath)"
|
||||
} else {
|
||||
$IncompatibleToolkit = "$tkMaj.$tkMin"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1155,26 +1166,18 @@ if ($DriverMaxCuda) {
|
|||
$NvccPath = Find-Nvcc
|
||||
}
|
||||
|
||||
# -- If incompatible toolkit is blocking, tell user to uninstall it --
|
||||
# A newer-major toolkit blocked by the driver: explain the mismatch.
|
||||
if (-not $NvccPath -and $IncompatibleToolkit) {
|
||||
Write-CudaDriverToolkitMismatch -ToolkitVersion $IncompatibleToolkit -DriverMaxCuda $DriverMaxCuda
|
||||
if (-not $RequireOrExit) {
|
||||
substep "CUDA Toolkit $IncompatibleToolkit exceeds driver max $DriverMaxCuda -- skipping; prebuilt llama.cpp needs no local toolkit" "Yellow"
|
||||
$script:CudaToolkitReady = $false
|
||||
return
|
||||
}
|
||||
# Reached only by a source build (forced, or after a prebuilt-install failure);
|
||||
# with no compatible toolkit it must fail (setup.sh degrades to CPU instead).
|
||||
Write-Host "" -ForegroundColor Red
|
||||
Write-Host "========================================================================" -ForegroundColor Red
|
||||
Write-Host "[ERROR] CUDA Toolkit $IncompatibleToolkit is installed but INCOMPATIBLE" -ForegroundColor Red
|
||||
Write-Host " with your NVIDIA driver (which supports up to CUDA $DriverMaxCuda)." -ForegroundColor Red
|
||||
Write-Host "" -ForegroundColor Red
|
||||
Write-Host " This will cause 'failed to initialize CUDA' errors at runtime." -ForegroundColor Red
|
||||
Write-Host "" -ForegroundColor Red
|
||||
Write-Host " To fix:" -ForegroundColor Yellow
|
||||
Write-Host " 1. Open Control Panel -> Programs -> Uninstall a program" -ForegroundColor Yellow
|
||||
Write-Host " 2. Uninstall 'NVIDIA CUDA Toolkit $IncompatibleToolkit'" -ForegroundColor Yellow
|
||||
Write-Host " 3. Re-run setup.bat (it will install CUDA $DriverMaxCuda automatically)" -ForegroundColor Yellow
|
||||
Write-Host "" -ForegroundColor Yellow
|
||||
Write-Host " Alternatively, update your NVIDIA driver to one that supports CUDA $IncompatibleToolkit." -ForegroundColor Gray
|
||||
Write-Host "[ERROR] CUDA source build cannot use the installed toolkit with this driver." -ForegroundColor Red
|
||||
Write-Host "========================================================================" -ForegroundColor Red
|
||||
exit 1
|
||||
}
|
||||
|
|
@ -1187,7 +1190,6 @@ if (-not $NvccPath -and $RequireOrExit) {
|
|||
if ($DriverMaxCuda) {
|
||||
# Query winget for available CUDA Toolkit versions
|
||||
$drMajor = [int]$DriverMaxCuda.Split('.')[0]
|
||||
$drMinor = [int]$DriverMaxCuda.Split('.')[1]
|
||||
$AvailableVersions = @()
|
||||
try {
|
||||
$rawOutput = winget show Nvidia.CUDA --versions --accept-source-agreements 2>&1 | Out-String
|
||||
|
|
@ -1200,13 +1202,12 @@ if (-not $NvccPath -and $RequireOrExit) {
|
|||
}
|
||||
} catch {}
|
||||
|
||||
# Filter to compatible versions (<= driver max) and pick the highest
|
||||
# Filter to compatible major versions and pick the highest
|
||||
$BestVersion = $null
|
||||
foreach ($ver in $AvailableVersions) {
|
||||
$parts = $ver.Split('.')
|
||||
$vMajor = [int]$parts[0]
|
||||
$vMinor = [int]$parts[1]
|
||||
if ($vMajor -lt $drMajor -or ($vMajor -eq $drMajor -and $vMinor -le $drMinor)) {
|
||||
if ($vMajor -le $drMajor) {
|
||||
$BestVersion = $ver
|
||||
break # list is descending, first match is highest compatible
|
||||
}
|
||||
|
|
@ -1224,7 +1225,7 @@ if (-not $NvccPath -and $RequireOrExit) {
|
|||
substep "CUDA Toolkit $BestVersion installed (nvcc: $NvccPath)"
|
||||
}
|
||||
} else {
|
||||
substep "no compatible CUDA Toolkit version found in winget (need <= $DriverMaxCuda)" "Yellow"
|
||||
substep "no compatible CUDA Toolkit version found in winget (need CUDA major <= $drMajor)" "Yellow"
|
||||
}
|
||||
} else {
|
||||
substep "Installing CUDA Toolkit (latest) via winget..."
|
||||
|
|
@ -1246,7 +1247,7 @@ if (-not $NvccPath) {
|
|||
}
|
||||
Write-Host "[ERROR] CUDA Toolkit (nvcc) is required but could not be found or installed." -ForegroundColor Red
|
||||
if ($DriverMaxCuda) {
|
||||
Write-Host " Install CUDA Toolkit $DriverMaxCuda from https://developer.nvidia.com/cuda-toolkit-archive" -ForegroundColor Yellow
|
||||
Write-Host " Install a CUDA Toolkit with major version $($DriverMaxCuda.Split('.')[0]) from https://developer.nvidia.com/cuda-toolkit-archive" -ForegroundColor Yellow
|
||||
} else {
|
||||
Write-Host " Install CUDA Toolkit from https://developer.nvidia.com/cuda-downloads" -ForegroundColor Yellow
|
||||
}
|
||||
|
|
|
|||
182
studio/setup.sh
182
studio/setup.sh
|
|
@ -59,8 +59,9 @@ fi
|
|||
# ── Output helpers ──
|
||||
# Consistent column layout: 2-space indent, 15-char label (fits llama-quantize), then value.
|
||||
# Usage: step <label> <message> [color] (color defaults to C_OK)
|
||||
# Usage: substep <message> [color] (color defaults to C_DIM)
|
||||
step() { printf " ${C_DIM}%-15.15s${C_RST}${3:-$C_OK}%s${C_RST}\n" "$1" "$2"; }
|
||||
substep() { printf " ${C_DIM}%-15s%s${C_RST}\n" "" "$1"; }
|
||||
substep() { printf " %-15s${2:-$C_DIM}%s${C_RST}\n" "" "$1"; }
|
||||
|
||||
_is_verbose() {
|
||||
[ "${UNSLOTH_VERBOSE:-0}" = "1" ]
|
||||
|
|
@ -154,6 +155,107 @@ _nvcc_meets_llama_minimum() {
|
|||
echo "$_raw"
|
||||
}
|
||||
|
||||
_cuda_driver_max_version() {
|
||||
command -v nvidia-smi >/dev/null 2>&1 || return 0
|
||||
nvidia-smi 2>/dev/null \
|
||||
| sed -nE 's/.*CUDA( UMD)? Version:[[:space:]]*([0-9]+)\.([0-9]+).*/\2.\3/p' \
|
||||
| head -1 || true
|
||||
}
|
||||
|
||||
_cuda_version_gt() {
|
||||
local _left=${1:-}
|
||||
local _right=${2:-}
|
||||
if ! [[ "$_left" =~ ^([0-9]+)\.([0-9]+)$ ]]; then
|
||||
return 1
|
||||
fi
|
||||
local _left_major=$((10#${BASH_REMATCH[1]}))
|
||||
local _left_minor=$((10#${BASH_REMATCH[2]}))
|
||||
if ! [[ "$_right" =~ ^([0-9]+)\.([0-9]+)$ ]]; then
|
||||
return 1
|
||||
fi
|
||||
local _right_major=$((10#${BASH_REMATCH[1]}))
|
||||
local _right_minor=$((10#${BASH_REMATCH[2]}))
|
||||
|
||||
if [ "$_left_major" -gt "$_right_major" ]; then
|
||||
return 0
|
||||
fi
|
||||
if [ "$_left_major" -eq "$_right_major" ] && [ "$_left_minor" -gt "$_right_minor" ]; then
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
_cuda_toolkit_major_gt_driver() {
|
||||
local _toolkit_version=${1:-}
|
||||
local _driver_version=${2:-}
|
||||
if ! [[ "$_toolkit_version" =~ ^([0-9]+)\.([0-9]+)$ ]]; then
|
||||
return 1
|
||||
fi
|
||||
local _toolkit_major=$((10#${BASH_REMATCH[1]}))
|
||||
if ! [[ "$_driver_version" =~ ^([0-9]+)\.([0-9]+)$ ]]; then
|
||||
return 1
|
||||
fi
|
||||
local _driver_major=$((10#${BASH_REMATCH[1]}))
|
||||
[ "$_toolkit_major" -gt "$_driver_major" ]
|
||||
}
|
||||
|
||||
_cuda_nvcc_candidate_paths() {
|
||||
if command -v nvcc >/dev/null 2>&1; then
|
||||
command -v nvcc
|
||||
fi
|
||||
if [ -x /usr/local/cuda/bin/nvcc ]; then
|
||||
printf '%s\n' "/usr/local/cuda/bin/nvcc"
|
||||
fi
|
||||
ls -d /usr/local/cuda-*/bin/nvcc 2>/dev/null | sort -V -r 2>/dev/null || true
|
||||
}
|
||||
|
||||
_cuda_find_compatible_nvcc_for_driver() {
|
||||
local _driver_version=$1
|
||||
local _exclude_path=${2:-}
|
||||
local _candidate _seen _check _status _version
|
||||
local _best_path="" _best_version=""
|
||||
_seen="
|
||||
"
|
||||
while IFS= read -r _candidate; do
|
||||
[ -n "$_candidate" ] || continue
|
||||
[ "$_candidate" != "$_exclude_path" ] || continue
|
||||
[ -x "$_candidate" ] || continue
|
||||
case "$_seen" in
|
||||
*"
|
||||
$_candidate
|
||||
"*) continue ;;
|
||||
esac
|
||||
_seen="${_seen}${_candidate}
|
||||
"
|
||||
_check="$(_nvcc_meets_llama_minimum "$_candidate")"
|
||||
_status="$(printf '%s\n' "$_check" | sed -n '1p')"
|
||||
_version="$(printf '%s\n' "$_check" | sed -n '2p')"
|
||||
[ "$_status" = "ok" ] || continue
|
||||
[ -n "$_version" ] || continue
|
||||
if _cuda_toolkit_major_gt_driver "$_version" "$_driver_version"; then
|
||||
continue
|
||||
fi
|
||||
if [ -z "$_best_version" ] || _cuda_version_gt "$_version" "$_best_version"; then
|
||||
_best_path="$_candidate"
|
||||
_best_version="$_version"
|
||||
fi
|
||||
done <<EOF
|
||||
$(_cuda_nvcc_candidate_paths)
|
||||
EOF
|
||||
[ -n "$_best_path" ] || return 1
|
||||
printf '%s\n%s\n' "$_best_path" "$_best_version"
|
||||
}
|
||||
|
||||
_print_cuda_driver_toolkit_mismatch() {
|
||||
local _toolkit_version=$1
|
||||
local _driver_version=$2
|
||||
local _toolkit_major=${_toolkit_version%%.*}
|
||||
local _driver_major=${_driver_version%%.*}
|
||||
substep "CUDA Toolkit $_toolkit_version is a major-version mismatch: toolkit major $_toolkit_major exceeds driver CUDA major $_driver_major ($_driver_version)." "$C_WARN"
|
||||
substep "Update the NVIDIA GPU driver to run CUDA Toolkit $_toolkit_version, or install a CUDA $_driver_major.x toolkit." "$C_WARN"
|
||||
substep "Or let Studio use the prebuilt CUDA bundle; it does not need the local toolkit." "$C_WARN"
|
||||
}
|
||||
|
||||
print_llama_error_log() {
|
||||
local log_file=$1
|
||||
[ -s "$log_file" ] || return 0
|
||||
|
|
@ -1162,38 +1264,58 @@ else
|
|||
GPU_BACKEND=""
|
||||
_BUILD_DESC="building (CPU, CUDA toolkit < 12.4)"
|
||||
else
|
||||
CMAKE_ARGS="$CMAKE_ARGS -DGGML_CUDA=ON"
|
||||
|
||||
CUDA_ARCHS=""
|
||||
if command -v nvidia-smi &>/dev/null; then
|
||||
_raw_caps=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null || true)
|
||||
while IFS= read -r _cap; do
|
||||
_cap=$(echo "$_cap" | tr -d '[:space:]')
|
||||
if [[ "$_cap" =~ ^([0-9]+)\.([0-9]+)$ ]]; then
|
||||
_arch="${BASH_REMATCH[1]}${BASH_REMATCH[2]}"
|
||||
# Append if not already present
|
||||
case ";$CUDA_ARCHS;" in
|
||||
*";$_arch;"*) ;;
|
||||
*) CUDA_ARCHS="${CUDA_ARCHS:+$CUDA_ARCHS;}$_arch" ;;
|
||||
esac
|
||||
fi
|
||||
done <<< "$_raw_caps"
|
||||
_DRIVER_MAX_CUDA="$(_cuda_driver_max_version)"
|
||||
_CUDA_TOOLKIT_ALLOWED=true
|
||||
if [ -n "$_NVCC_VER" ] && [ -n "$_DRIVER_MAX_CUDA" ] && \
|
||||
_cuda_toolkit_major_gt_driver "$_NVCC_VER" "$_DRIVER_MAX_CUDA"; then
|
||||
_BLOCKED_NVCC_VER="$_NVCC_VER"
|
||||
if _ALT_NVCC_CHECK="$(_cuda_find_compatible_nvcc_for_driver "$_DRIVER_MAX_CUDA" "$NVCC_PATH")"; then
|
||||
NVCC_PATH="$(printf '%s\n' "$_ALT_NVCC_CHECK" | sed -n '1p')"
|
||||
_NVCC_VER="$(printf '%s\n' "$_ALT_NVCC_CHECK" | sed -n '2p')"
|
||||
GPU_BACKEND="cuda"
|
||||
export PATH="$(dirname "$NVCC_PATH"):$PATH"
|
||||
substep "CUDA Toolkit $_BLOCKED_NVCC_VER is a major-version mismatch with driver CUDA $_DRIVER_MAX_CUDA; using compatible CUDA Toolkit $_NVCC_VER at $NVCC_PATH." "$C_WARN"
|
||||
else
|
||||
_print_cuda_driver_toolkit_mismatch "$_NVCC_VER" "$_DRIVER_MAX_CUDA"
|
||||
substep "falling back to CPU llama.cpp build for this run." "$C_WARN"
|
||||
NVCC_PATH=""
|
||||
GPU_BACKEND=""
|
||||
_BUILD_DESC="building (CPU, CUDA toolkit major > driver)"
|
||||
_CUDA_TOOLKIT_ALLOWED=false
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -n "$CUDA_ARCHS" ]; then
|
||||
CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCHS}"
|
||||
_BUILD_DESC="building (CUDA, sm_${CUDA_ARCHS//;/+sm_})"
|
||||
else
|
||||
_BUILD_DESC="building (CUDA)"
|
||||
if [ "$_CUDA_TOOLKIT_ALLOWED" = true ]; then
|
||||
CMAKE_ARGS="$CMAKE_ARGS -DGGML_CUDA=ON"
|
||||
|
||||
CUDA_ARCHS=""
|
||||
if command -v nvidia-smi &>/dev/null; then
|
||||
_raw_caps=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null || true)
|
||||
while IFS= read -r _cap; do
|
||||
_cap=$(echo "$_cap" | tr -d '[:space:]')
|
||||
if [[ "$_cap" =~ ^([0-9]+)\.([0-9]+)$ ]]; then
|
||||
_arch="${BASH_REMATCH[1]}${BASH_REMATCH[2]}"
|
||||
case ";$CUDA_ARCHS;" in
|
||||
*";$_arch;"*) ;;
|
||||
*) CUDA_ARCHS="${CUDA_ARCHS:+$CUDA_ARCHS;}$_arch" ;;
|
||||
esac
|
||||
fi
|
||||
done <<< "$_raw_caps"
|
||||
fi
|
||||
|
||||
if [ -n "$CUDA_ARCHS" ]; then
|
||||
CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCHS}"
|
||||
_BUILD_DESC="building (CUDA, sm_${CUDA_ARCHS//;/+sm_})"
|
||||
else
|
||||
_BUILD_DESC="building (CUDA)"
|
||||
fi
|
||||
|
||||
CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_CUDA_FLAGS=--threads=0"
|
||||
|
||||
# Allow a host gcc/clang newer than nvcc's whitelist (else a fresh
|
||||
# toolkit aborts with "unsupported GNU version"); via env to avoid word-splitting.
|
||||
export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS:+$NVCC_PREPEND_FLAGS }-allow-unsupported-compiler"
|
||||
fi
|
||||
|
||||
CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_CUDA_FLAGS=--threads=0"
|
||||
|
||||
# Accept a host gcc/clang newer than nvcc's whitelist; a fresh
|
||||
# toolkit (e.g. CUDA 13.3) otherwise aborts with "#error --
|
||||
# unsupported GNU version". Via env, not CMAKE_ARGS, to avoid
|
||||
# word-splitting.
|
||||
export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS:+$NVCC_PREPEND_FLAGS }-allow-unsupported-compiler"
|
||||
fi
|
||||
elif [ "$GPU_BACKEND" = "rocm" ]; then
|
||||
# Resolve hipcc symlinks to find the real ROCm root
|
||||
|
|
|
|||
|
|
@ -11,8 +11,11 @@ No GPU, no network, no torch required -- all I/O is monkeypatched.
|
|||
"""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
|
|
@ -3062,3 +3065,443 @@ class TestCpuFallback:
|
|||
# is gated on a degraded source build for arm64.
|
||||
assert "ggml-org/llama.cpp" in source
|
||||
assert "_LLAMA_CPP_DEGRADED" in source
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# setup.sh / setup.ps1: CUDA toolkit newer than driver diagnostics
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform == "win32", reason = "bash-only Studio installer tests")
|
||||
class TestCudaDriverToolkitMismatchMessage:
|
||||
_SETUP_SH = PACKAGE_ROOT / "studio" / "setup.sh"
|
||||
_SETUP_PS1 = PACKAGE_ROOT / "studio" / "setup.ps1"
|
||||
|
||||
def _setup_sh_cuda_helper_fragment(self):
|
||||
source = self._SETUP_SH.read_text(encoding = "utf-8")
|
||||
start = source.index("_nvcc_meets_llama_minimum()")
|
||||
end = source.index("print_llama_error_log()")
|
||||
return source[start:end]
|
||||
|
||||
def _run_bash(
|
||||
self,
|
||||
script,
|
||||
*,
|
||||
env = None,
|
||||
):
|
||||
proc = subprocess.run(
|
||||
["/bin/bash", "-c", script],
|
||||
check = True,
|
||||
text = True,
|
||||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.PIPE,
|
||||
env = {**os.environ, **(env or {})},
|
||||
)
|
||||
return proc.stdout + proc.stderr
|
||||
|
||||
def _fake_nvidia_smi(self, tmp_path, output):
|
||||
mock_bin = tmp_path / "bin"
|
||||
mock_bin.mkdir()
|
||||
nvidia_smi = mock_bin / "nvidia-smi"
|
||||
nvidia_smi.write_text(
|
||||
textwrap.dedent(
|
||||
f"""\
|
||||
#!/usr/bin/env bash
|
||||
cat <<'OUT'
|
||||
{output}
|
||||
OUT
|
||||
"""
|
||||
),
|
||||
encoding = "utf-8",
|
||||
)
|
||||
nvidia_smi.chmod(0o755)
|
||||
return mock_bin
|
||||
|
||||
def test_setup_sh_same_major_minor_mismatch_is_accepted(self, tmp_path):
|
||||
mock_bin = self._fake_nvidia_smi(
|
||||
tmp_path,
|
||||
"| NVIDIA-SMI 580.95 Driver Version: 580.95 CUDA Version: 13.1 |",
|
||||
)
|
||||
script = textwrap.dedent(
|
||||
f"""\
|
||||
set -euo pipefail
|
||||
C_WARN=
|
||||
substep() {{ printf '%s\\n' "$1"; }}
|
||||
{self._setup_sh_cuda_helper_fragment()}
|
||||
_driver="$(_cuda_driver_max_version)"
|
||||
if _cuda_toolkit_major_gt_driver "13.3" "$_driver"; then
|
||||
_print_cuda_driver_toolkit_mismatch "13.3" "$_driver"
|
||||
else
|
||||
printf 'compatible:%s\\n' "$_driver"
|
||||
fi
|
||||
"""
|
||||
)
|
||||
output = self._run_bash(
|
||||
script,
|
||||
env = {"PATH": f"{mock_bin}:{os.environ.get('PATH', '')}"},
|
||||
)
|
||||
assert "compatible:13.1" in output
|
||||
assert "major-version mismatch" not in output
|
||||
|
||||
def test_setup_sh_driver_major_too_old_message_names_major_mismatch(self, tmp_path):
|
||||
mock_bin = self._fake_nvidia_smi(
|
||||
tmp_path,
|
||||
"| NVIDIA-SMI 570.95 Driver Version: 570.95 CUDA Version: 12.9 |",
|
||||
)
|
||||
script = textwrap.dedent(
|
||||
f"""\
|
||||
set -euo pipefail
|
||||
C_WARN=
|
||||
substep() {{ printf '%s\\n' "$1"; }}
|
||||
{self._setup_sh_cuda_helper_fragment()}
|
||||
_driver="$(_cuda_driver_max_version)"
|
||||
if _cuda_toolkit_major_gt_driver "13.3" "$_driver"; then
|
||||
_print_cuda_driver_toolkit_mismatch "13.3" "$_driver"
|
||||
fi
|
||||
"""
|
||||
)
|
||||
output = self._run_bash(
|
||||
script,
|
||||
env = {"PATH": f"{mock_bin}:{os.environ.get('PATH', '')}"},
|
||||
)
|
||||
assert (
|
||||
"CUDA Toolkit 13.3 is a major-version mismatch: toolkit major 13 "
|
||||
"exceeds driver CUDA major 12 (12.9)."
|
||||
) in output
|
||||
assert (
|
||||
"Update the NVIDIA GPU driver to run CUDA Toolkit 13.3, or install "
|
||||
"a CUDA 12.x toolkit."
|
||||
) in output
|
||||
assert "prebuilt CUDA bundle" in output
|
||||
|
||||
def test_setup_sh_happy_path_does_not_print_mismatch(self, tmp_path):
|
||||
mock_bin = self._fake_nvidia_smi(
|
||||
tmp_path,
|
||||
"| NVIDIA-SMI 580.95 Driver Version: 580.95 CUDA Version: 13.3 |",
|
||||
)
|
||||
script = textwrap.dedent(
|
||||
f"""\
|
||||
set -euo pipefail
|
||||
C_WARN=
|
||||
substep() {{ printf '%s\\n' "$1"; }}
|
||||
{self._setup_sh_cuda_helper_fragment()}
|
||||
_driver="$(_cuda_driver_max_version)"
|
||||
if _cuda_toolkit_major_gt_driver "13.3" "$_driver"; then
|
||||
_print_cuda_driver_toolkit_mismatch "13.3" "$_driver"
|
||||
else
|
||||
printf 'compatible:%s\\n' "$_driver"
|
||||
fi
|
||||
"""
|
||||
)
|
||||
output = self._run_bash(
|
||||
script,
|
||||
env = {"PATH": f"{mock_bin}:{os.environ.get('PATH', '')}"},
|
||||
)
|
||||
assert "compatible:13.3" in output
|
||||
assert "Unsloth supports CUDA Toolkit" not in output
|
||||
|
||||
def test_setup_sh_skips_check_without_nvidia_smi(self, tmp_path):
|
||||
empty_bin = tmp_path / "empty-bin"
|
||||
empty_bin.mkdir()
|
||||
script = textwrap.dedent(
|
||||
f"""\
|
||||
set -euo pipefail
|
||||
C_WARN=
|
||||
substep() {{ printf '%s\\n' "$1"; }}
|
||||
{self._setup_sh_cuda_helper_fragment()}
|
||||
_driver="$(_cuda_driver_max_version)"
|
||||
if [ -n "$_driver" ] && _cuda_toolkit_major_gt_driver "13.3" "$_driver"; then
|
||||
_print_cuda_driver_toolkit_mismatch "13.3" "$_driver"
|
||||
else
|
||||
printf 'skipped\\n'
|
||||
fi
|
||||
"""
|
||||
)
|
||||
output = self._run_bash(script, env = {"PATH": str(empty_bin)})
|
||||
assert "skipped" in output
|
||||
assert "Unsloth supports CUDA Toolkit" not in output
|
||||
|
||||
def test_setup_sh_unparsable_nvidia_smi_output_falls_back(self, tmp_path):
|
||||
mock_bin = self._fake_nvidia_smi(
|
||||
tmp_path,
|
||||
"| NVIDIA-SMI 580.95 Driver Version: 580.95 CUDA Version: N/A |",
|
||||
)
|
||||
script = textwrap.dedent(
|
||||
f"""\
|
||||
set -euo pipefail
|
||||
C_WARN=
|
||||
substep() {{ printf '%s\\n' "$1"; }}
|
||||
{self._setup_sh_cuda_helper_fragment()}
|
||||
_driver="$(_cuda_driver_max_version)"
|
||||
if [ -n "$_driver" ] && _cuda_toolkit_major_gt_driver "13.3" "$_driver"; then
|
||||
_print_cuda_driver_toolkit_mismatch "13.3" "$_driver"
|
||||
else
|
||||
printf 'fallback:%s\\n' "${{_driver:-generic}}"
|
||||
fi
|
||||
"""
|
||||
)
|
||||
output = self._run_bash(
|
||||
script,
|
||||
env = {"PATH": f"{mock_bin}:{os.environ.get('PATH', '')}"},
|
||||
)
|
||||
assert "fallback:generic" in output
|
||||
assert "Unsloth supports CUDA Toolkit" not in output
|
||||
|
||||
def test_setup_sh_cuda_version_gt_compares_numerically(self):
|
||||
# 13.9 vs 13.10 is where a lexical compare goes wrong (9 > 1).
|
||||
script = textwrap.dedent(
|
||||
f"""\
|
||||
set -euo pipefail
|
||||
{self._setup_sh_cuda_helper_fragment()}
|
||||
for pair in "13.9 13.10" "13.10 13.9" "14.0 13.9" "13.3 13.3"; do
|
||||
if _cuda_version_gt $pair; then r=gt; else r=le; fi
|
||||
printf '%s -> %s\\n' "$pair" "$r"
|
||||
done
|
||||
"""
|
||||
)
|
||||
output = self._run_bash(script)
|
||||
assert "13.9 13.10 -> le" in output
|
||||
assert "13.10 13.9 -> gt" in output
|
||||
assert "14.0 13.9 -> gt" in output
|
||||
assert "13.3 13.3 -> le" in output
|
||||
|
||||
def test_setup_ps1_mirrors_driver_mismatch_guidance(self):
|
||||
source = self._SETUP_PS1.read_text(encoding = "utf-8")
|
||||
assert "Write-CudaDriverToolkitMismatch" in source
|
||||
assert (
|
||||
"CUDA Toolkit $ToolkitVersion is a major-version mismatch: toolkit "
|
||||
"major $toolkitMajor exceeds driver CUDA major $driverMajor"
|
||||
) in source
|
||||
assert (
|
||||
"Update the NVIDIA GPU driver to run CUDA Toolkit $ToolkitVersion, "
|
||||
"or install a CUDA $driverMajor.x toolkit." in source
|
||||
)
|
||||
assert (
|
||||
"Or let Studio use the prebuilt CUDA bundle; it does not need the local toolkit."
|
||||
) in source
|
||||
assert (
|
||||
"Write-CudaDriverToolkitMismatch -ToolkitVersion $IncompatibleToolkit "
|
||||
"-DriverMaxCuda $DriverMaxCuda"
|
||||
) in source
|
||||
|
||||
def _fake_nvcc(self, tmp_path, release):
|
||||
mock_bin = tmp_path / f"nvcc-bin-{release.replace('.', '-')}"
|
||||
mock_bin.mkdir()
|
||||
nvcc = mock_bin / "nvcc"
|
||||
nvcc.write_text(
|
||||
textwrap.dedent(
|
||||
f"""\
|
||||
#!/usr/bin/env bash
|
||||
printf '%s\\n' 'Cuda compilation tools, release {release}, V{release}.0'
|
||||
"""
|
||||
),
|
||||
encoding = "utf-8",
|
||||
)
|
||||
nvcc.chmod(0o755)
|
||||
return nvcc
|
||||
|
||||
def test_setup_sh_major_mismatch_uses_newest_compatible_detected_toolkit(self, tmp_path):
|
||||
blocked_nvcc = self._fake_nvcc(tmp_path, "13.3")
|
||||
older_nvcc = self._fake_nvcc(tmp_path, "12.6")
|
||||
compatible_nvcc = self._fake_nvcc(tmp_path, "12.8")
|
||||
script = textwrap.dedent(
|
||||
f"""\
|
||||
set -euo pipefail
|
||||
C_WARN=
|
||||
substep() {{ printf '%s\\n' "$1"; }}
|
||||
{self._setup_sh_cuda_helper_fragment()}
|
||||
_cuda_nvcc_candidate_paths() {{
|
||||
printf '%s\\n' "{blocked_nvcc}" "{older_nvcc}" "{compatible_nvcc}"
|
||||
}}
|
||||
NVCC_PATH="{blocked_nvcc}"
|
||||
GPU_BACKEND="cuda"
|
||||
_BUILD_DESC="building"
|
||||
_NVCC_CHECK="$(_nvcc_meets_llama_minimum "$NVCC_PATH")"
|
||||
_NVCC_STATUS="$(printf '%s\\n' "$_NVCC_CHECK" | sed -n '1p')"
|
||||
_NVCC_VER="$(printf '%s\\n' "$_NVCC_CHECK" | sed -n '2p')"
|
||||
_DRIVER_MAX_CUDA="12.9"
|
||||
_CUDA_TOOLKIT_ALLOWED=true
|
||||
if [ "$_NVCC_STATUS" != "too_old" ] && \
|
||||
[ -n "$_NVCC_VER" ] && \
|
||||
_cuda_toolkit_major_gt_driver "$_NVCC_VER" "$_DRIVER_MAX_CUDA"; then
|
||||
_BLOCKED_NVCC_VER="$_NVCC_VER"
|
||||
if _ALT_NVCC_CHECK="$(_cuda_find_compatible_nvcc_for_driver "$_DRIVER_MAX_CUDA" "$NVCC_PATH")"; then
|
||||
NVCC_PATH="$(printf '%s\\n' "$_ALT_NVCC_CHECK" | sed -n '1p')"
|
||||
_NVCC_VER="$(printf '%s\\n' "$_ALT_NVCC_CHECK" | sed -n '2p')"
|
||||
GPU_BACKEND="cuda"
|
||||
substep "CUDA Toolkit $_BLOCKED_NVCC_VER is a major-version mismatch with driver CUDA $_DRIVER_MAX_CUDA; using compatible CUDA Toolkit $_NVCC_VER at $NVCC_PATH."
|
||||
else
|
||||
NVCC_PATH=""
|
||||
GPU_BACKEND=""
|
||||
_BUILD_DESC="building (CPU, CUDA toolkit major > driver)"
|
||||
_CUDA_TOOLKIT_ALLOWED=false
|
||||
fi
|
||||
fi
|
||||
printf 'NVCC_PATH=%s\\n' "$NVCC_PATH"
|
||||
printf 'NVCC_VER=%s\\n' "$_NVCC_VER"
|
||||
printf 'GPU_BACKEND=%s\\n' "$GPU_BACKEND"
|
||||
printf 'BUILD_DESC=%s\\n' "$_BUILD_DESC"
|
||||
printf 'ALLOWED=%s\\n' "$_CUDA_TOOLKIT_ALLOWED"
|
||||
"""
|
||||
)
|
||||
output = self._run_bash(script)
|
||||
assert f"NVCC_PATH={compatible_nvcc}" in output
|
||||
assert "NVCC_VER=12.8" in output
|
||||
assert "GPU_BACKEND=cuda" in output
|
||||
assert "ALLOWED=true" in output
|
||||
assert "CPU" not in output
|
||||
|
||||
def test_setup_sh_parses_cuda_umd_version_variant(self, tmp_path):
|
||||
# Newer drivers report "CUDA UMD Version"; the helper must read it too.
|
||||
mock_bin = self._fake_nvidia_smi(
|
||||
tmp_path,
|
||||
"| NVIDIA-SMI 610.00 Driver Version: 610.00 CUDA UMD Version: 13.0 |",
|
||||
)
|
||||
script = textwrap.dedent(
|
||||
f"""\
|
||||
set -euo pipefail
|
||||
{self._setup_sh_cuda_helper_fragment()}
|
||||
printf '%s' "$(_cuda_driver_max_version)"
|
||||
"""
|
||||
)
|
||||
output = self._run_bash(
|
||||
script,
|
||||
env = {"PATH": f"{mock_bin}:{os.environ.get('PATH', '')}"},
|
||||
)
|
||||
assert output.strip() == "13.0"
|
||||
|
||||
def test_setup_sh_empty_toolkit_version_skips_mismatch(self, tmp_path):
|
||||
# The real guard requires a non-empty nvcc version before warning.
|
||||
mock_bin = self._fake_nvidia_smi(
|
||||
tmp_path,
|
||||
"| NVIDIA-SMI 580.95 Driver Version: 580.95 CUDA Version: 13.0 |",
|
||||
)
|
||||
script = textwrap.dedent(
|
||||
f"""\
|
||||
set -euo pipefail
|
||||
C_WARN=
|
||||
substep() {{ printf '%s\\n' "$1"; }}
|
||||
{self._setup_sh_cuda_helper_fragment()}
|
||||
_nvcc=""
|
||||
_driver="$(_cuda_driver_max_version)"
|
||||
if [ -n "$_nvcc" ] && [ -n "$_driver" ] && _cuda_toolkit_major_gt_driver "$_nvcc" "$_driver"; then
|
||||
_print_cuda_driver_toolkit_mismatch "$_nvcc" "$_driver"
|
||||
else
|
||||
printf 'skipped\\n'
|
||||
fi
|
||||
"""
|
||||
)
|
||||
output = self._run_bash(
|
||||
script,
|
||||
env = {"PATH": f"{mock_bin}:{os.environ.get('PATH', '')}"},
|
||||
)
|
||||
assert "skipped" in output
|
||||
assert "Unsloth supports CUDA Toolkit" not in output
|
||||
|
||||
def test_setup_sh_nvcc_below_minimum_is_too_old(self, tmp_path):
|
||||
# CUDA toolkit < 12.4 short-circuits to the too_old branch (no mismatch).
|
||||
nvcc = self._fake_nvcc(tmp_path, "12.0")
|
||||
script = textwrap.dedent(
|
||||
f"""\
|
||||
set -euo pipefail
|
||||
{self._setup_sh_cuda_helper_fragment()}
|
||||
_nvcc_meets_llama_minimum "{nvcc}"
|
||||
"""
|
||||
)
|
||||
output = self._run_bash(script)
|
||||
assert output.splitlines()[0] == "too_old"
|
||||
assert "12.0" in output
|
||||
|
||||
def test_setup_sh_compatible_finder_rejects_too_old_only_candidate(self, tmp_path):
|
||||
# Only candidate is below the llama minimum: finder must fail (CPU fallback), not pick 12.0.
|
||||
blocked_nvcc = self._fake_nvcc(tmp_path, "13.3")
|
||||
too_old_nvcc = self._fake_nvcc(tmp_path, "12.0")
|
||||
script = textwrap.dedent(
|
||||
f"""\
|
||||
set -euo pipefail
|
||||
{self._setup_sh_cuda_helper_fragment()}
|
||||
_cuda_nvcc_candidate_paths() {{
|
||||
printf '%s\\n' "{blocked_nvcc}" "{too_old_nvcc}"
|
||||
}}
|
||||
if _ALT="$(_cuda_find_compatible_nvcc_for_driver "12.9" "{blocked_nvcc}")"; then
|
||||
printf 'FOUND:%s\\n' "$_ALT"
|
||||
else
|
||||
printf 'NONE\\n'
|
||||
fi
|
||||
"""
|
||||
)
|
||||
output = self._run_bash(script)
|
||||
assert "NONE" in output
|
||||
assert "FOUND" not in output
|
||||
|
||||
def _cuda_build_decision_output(self, *, nvcc_path, driver):
|
||||
# Mirror setup.sh's source-build decision: keep the toolkit, switch, or degrade to CPU.
|
||||
script = textwrap.dedent(
|
||||
f"""\
|
||||
set -euo pipefail
|
||||
C_WARN=
|
||||
substep() {{ printf '%s\\n' "$1"; }}
|
||||
{self._setup_sh_cuda_helper_fragment()}
|
||||
NVCC_PATH="{nvcc_path}"
|
||||
GPU_BACKEND="cuda"
|
||||
_DRIVER_MAX_CUDA="{driver}"
|
||||
_NVCC_CHECK="$(_nvcc_meets_llama_minimum "$NVCC_PATH")"
|
||||
_NVCC_STATUS="$(printf '%s\\n' "$_NVCC_CHECK" | sed -n '1p')"
|
||||
_NVCC_VER="$(printf '%s\\n' "$_NVCC_CHECK" | sed -n '2p')"
|
||||
_CUDA_TOOLKIT_ALLOWED=true
|
||||
if [ "$_NVCC_STATUS" = "too_old" ]; then
|
||||
NVCC_PATH=""; GPU_BACKEND=""; _CUDA_TOOLKIT_ALLOWED=false
|
||||
elif [ -n "$_NVCC_VER" ] && [ -n "$_DRIVER_MAX_CUDA" ] && _cuda_toolkit_major_gt_driver "$_NVCC_VER" "$_DRIVER_MAX_CUDA"; then
|
||||
if _ALT="$(_cuda_find_compatible_nvcc_for_driver "$_DRIVER_MAX_CUDA" "$NVCC_PATH")"; then
|
||||
NVCC_PATH="$(printf '%s\\n' "$_ALT" | sed -n '1p')"
|
||||
_NVCC_VER="$(printf '%s\\n' "$_ALT" | sed -n '2p')"
|
||||
else
|
||||
NVCC_PATH=""; GPU_BACKEND=""; _CUDA_TOOLKIT_ALLOWED=false
|
||||
fi
|
||||
fi
|
||||
printf 'NVCC_PATH=%s\\n' "$NVCC_PATH"
|
||||
printf 'NVCC_VER=%s\\n' "$_NVCC_VER"
|
||||
printf 'GPU_BACKEND=%s\\n' "$GPU_BACKEND"
|
||||
printf 'ALLOWED=%s\\n' "$_CUDA_TOOLKIT_ALLOWED"
|
||||
"""
|
||||
)
|
||||
return self._run_bash(script)
|
||||
|
||||
def test_setup_sh_same_major_newer_minor_keeps_original_toolkit(self, tmp_path):
|
||||
# Same-major newer-minor (13.3 vs driver 13.0): build CUDA with it, never fall back.
|
||||
toolkit = self._fake_nvcc(tmp_path, "13.3")
|
||||
output = self._cuda_build_decision_output(nvcc_path = toolkit, driver = "13.0")
|
||||
assert f"NVCC_PATH={toolkit}" in output
|
||||
assert "NVCC_VER=13.3" in output
|
||||
assert "GPU_BACKEND=cuda" in output
|
||||
assert "ALLOWED=true" in output
|
||||
|
||||
def test_setup_sh_missing_driver_version_still_enables_cuda(self, tmp_path):
|
||||
# No driver CUDA version from nvidia-smi: keep CUDA enabled (pre-fix behavior), not CPU.
|
||||
toolkit = self._fake_nvcc(tmp_path, "13.3")
|
||||
output = self._cuda_build_decision_output(nvcc_path = toolkit, driver = "")
|
||||
assert f"NVCC_PATH={toolkit}" in output
|
||||
assert "GPU_BACKEND=cuda" in output
|
||||
assert "ALLOWED=true" in output
|
||||
|
||||
def test_setup_sh_compatible_finder_rejects_newer_major_only_candidate(self, tmp_path):
|
||||
# Only alternative is still newer-major than the driver: finder must fail, not pick it.
|
||||
blocked_nvcc = self._fake_nvcc(tmp_path, "13.3")
|
||||
other_newer_nvcc = self._fake_nvcc(tmp_path, "13.1")
|
||||
script = textwrap.dedent(
|
||||
f"""\
|
||||
set -euo pipefail
|
||||
{self._setup_sh_cuda_helper_fragment()}
|
||||
_cuda_nvcc_candidate_paths() {{
|
||||
printf '%s\\n' "{blocked_nvcc}" "{other_newer_nvcc}"
|
||||
}}
|
||||
if _ALT="$(_cuda_find_compatible_nvcc_for_driver "12.9" "{blocked_nvcc}")"; then
|
||||
printf 'FOUND:%s\\n' "$_ALT"
|
||||
else
|
||||
printf 'NONE\\n'
|
||||
fi
|
||||
"""
|
||||
)
|
||||
output = self._run_bash(script)
|
||||
assert "NONE" in output
|
||||
assert "FOUND" not in output
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env pwsh
|
||||
# Unit test for Resolve-CudaToolkit in studio/setup.ps1. No GPU required: the
|
||||
# detection helpers (nvidia-smi, nvcc, Find-Nvcc, ...) are stubbed so the real
|
||||
# function logic runs against a spoofed Blackwell sm_120 / driver 13.2 host.
|
||||
# function logic runs against spoofed Blackwell sm_120 driver/toolkit scenarios.
|
||||
#
|
||||
# The function is extracted via AST and run in a child pwsh per scenario, because
|
||||
# the -RequireOrExit path calls `exit` (which would otherwise kill this harness).
|
||||
|
|
@ -22,13 +22,24 @@ $fn = $ast.FindAll({ param($n)
|
|||
if ($fn.Count -ne 1) { throw "expected exactly one Resolve-CudaToolkit, found $($fn.Count)" }
|
||||
$fnText = $fn[0].Extent.Text
|
||||
|
||||
# --- Spoof executables: nvidia-smi reports driver max CUDA 13.2; nvcc 13.3 ---
|
||||
# Resolve-CudaToolkit calls Write-CudaDriverToolkitMismatch, so extract it too.
|
||||
$mismatchFn = $ast.FindAll({ param($n)
|
||||
$n -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $n.Name -eq "Write-CudaDriverToolkitMismatch"
|
||||
}, $true)
|
||||
if ($mismatchFn.Count -ne 1) { throw "expected exactly one Write-CudaDriverToolkitMismatch, found $($mismatchFn.Count)" }
|
||||
$mismatchText = $mismatchFn[0].Extent.Text
|
||||
|
||||
# --- Spoof executables for driver/toolkit compatibility scenarios ---
|
||||
$work = Join-Path ([System.IO.Path]::GetTempPath()) ("rct_" + [guid]::NewGuid().ToString("N"))
|
||||
New-Item -ItemType Directory -Force -Path $work | Out-Null
|
||||
$smiFake = Join-Path $work "nvidia-smi.ps1"
|
||||
$nvccFake = Join-Path $work "nvcc.ps1"
|
||||
Set-Content -LiteralPath $smiFake -Value "'CUDA Version: 13.2'"
|
||||
Set-Content -LiteralPath $nvccFake -Value "'Cuda compilation tools, release 13.3, V13.3.0'"
|
||||
$smiMajorMismatchFake = Join-Path $work "nvidia-smi-12.9.ps1"
|
||||
$smiSameMajorFake = Join-Path $work "nvidia-smi-13.2.ps1"
|
||||
$nvccIncompatibleFake = Join-Path $work "nvcc-13.3.ps1"
|
||||
$nvccCompatibleFake = Join-Path $work "nvcc-12.8.ps1"
|
||||
Set-Content -LiteralPath $smiMajorMismatchFake -Value "'CUDA Version: 12.9'"
|
||||
Set-Content -LiteralPath $smiSameMajorFake -Value "'CUDA Version: 13.2'"
|
||||
Set-Content -LiteralPath $nvccIncompatibleFake -Value "'Cuda compilation tools, release 13.3, V13.3.0'"
|
||||
Set-Content -LiteralPath $nvccCompatibleFake -Value "'Cuda compilation tools, release 12.8, V12.8.0'"
|
||||
|
||||
$failures = 0
|
||||
function Check($name, $cond) {
|
||||
|
|
@ -38,13 +49,15 @@ function Check($name, $cond) {
|
|||
|
||||
# Build + run one scenario in a child pwsh; returns @{ Exit; Out }.
|
||||
function Run-Case {
|
||||
param([string]$FindMode, [bool]$Require)
|
||||
param([string]$FindMode, [bool]$Require, [string]$DriverMode = "major-mismatch")
|
||||
$requireLit = if ($Require) { '$true' } else { '$false' }
|
||||
$smiForCase = if ($DriverMode -eq "same-major") { $smiSameMajorFake } else { $smiMajorMismatchFake }
|
||||
$child = @"
|
||||
`$ErrorActionPreference = 'Continue'
|
||||
[Environment]::SetEnvironmentVariable('CUDA_PATH', `$null, 'Process')
|
||||
`$FindNvccMode = '$FindMode'
|
||||
`$NvccFake = '$nvccFake'
|
||||
`$NvccIncompatibleFake = '$nvccIncompatibleFake'
|
||||
`$NvccCompatibleFake = '$nvccCompatibleFake'
|
||||
function substep { param(`$m, `$c) Write-Host " `$m" }
|
||||
function step { param(`$l, `$v, `$c) Write-Host "[`$l] `$v" }
|
||||
function Add-ToUserPath { param(`$Directory, `$Position) `$true }
|
||||
|
|
@ -57,17 +70,20 @@ function winget { `$script:WingetCalled = `$true; 'no matching versions' }
|
|||
function Find-Nvcc {
|
||||
param([string]`$MaxVersion = '')
|
||||
switch (`$FindNvccMode) {
|
||||
'compatible' { return `$NvccFake }
|
||||
'incompatible' { if (`$MaxVersion) { return `$null } else { return `$NvccFake } }
|
||||
'compatible' { return `$NvccCompatibleFake }
|
||||
'same-major' { return `$NvccIncompatibleFake }
|
||||
'incompatible' { if (`$MaxVersion) { return `$null } else { return `$NvccIncompatibleFake } }
|
||||
default { return `$null }
|
||||
}
|
||||
}
|
||||
`$NvidiaSmiExe = '$smiFake'
|
||||
`$NvidiaSmiExe = '$smiForCase'
|
||||
`$VsInstallPath = `$null
|
||||
`$HasNvidiaSmi = `$true
|
||||
`$script:CudaToolkitReady = `$false
|
||||
`$script:NvccPath = `$null; `$script:CudaToolkitRoot = `$null; `$script:CudaArch = `$null
|
||||
|
||||
$mismatchText
|
||||
|
||||
$fnText
|
||||
|
||||
if ($requireLit) { Resolve-CudaToolkit -RequireOrExit } else { Resolve-CudaToolkit }
|
||||
|
|
@ -80,37 +96,54 @@ Write-Host ("RESULT ready={0} nvcc={1} winget={2}" -f `$script:CudaToolkitReady,
|
|||
}
|
||||
|
||||
try {
|
||||
Write-Host "Scenario 1: prebuilt path, too-new toolkit (no -RequireOrExit) -> defers, no exit"
|
||||
Write-Host "Scenario 1: prebuilt path, newer-major toolkit (no -RequireOrExit) -> defers, no exit"
|
||||
$r = Run-Case -FindMode "incompatible" -Require $false
|
||||
Check "exits 0 (not blocked)" ($r.Exit -eq 0)
|
||||
Check "CudaToolkitReady = false" ($r.Out -match "ready=False")
|
||||
Check "winget NOT called" ($r.Out -match "winget=False")
|
||||
Check "no INCOMPATIBLE error text" (-not ($r.Out -match "INCOMPATIBLE"))
|
||||
Check "explains major mismatch" ($r.Out -match "major-version mismatch")
|
||||
Check "does not blame the toolkit" (-not ($r.Out -match "INCOMPATIBLE"))
|
||||
|
||||
Write-Host "Scenario 2: forced source build, too-new toolkit (-RequireOrExit) -> hard exit"
|
||||
Write-Host "Scenario 2: forced source build, newer-major toolkit (-RequireOrExit) -> hard exit"
|
||||
$r = Run-Case -FindMode "incompatible" -Require $true
|
||||
Check "exits non-zero" ($r.Exit -ne 0)
|
||||
Check "preserved INCOMPATIBLE error" ($r.Out -match "is installed but INCOMPATIBLE")
|
||||
Check "explains major mismatch" ($r.Out -match "major-version mismatch")
|
||||
Check "one-line source-build error" ($r.Out -match "CUDA source build cannot use the installed toolkit")
|
||||
|
||||
Write-Host "Scenario 3: compatible toolkit (-RequireOrExit) -> resolves, env set"
|
||||
Write-Host "Scenario 3: same-major newer-minor toolkit (-RequireOrExit) -> resolves, env set"
|
||||
$r = Run-Case -FindMode "same-major" -Require $true -DriverMode "same-major"
|
||||
Check "exits 0" ($r.Exit -eq 0)
|
||||
Check "CudaToolkitReady = true" ($r.Out -match "ready=True")
|
||||
Check "NvccPath published" ($r.Out -match "nvcc=.*nvcc-13\.3")
|
||||
Check "no mismatch warning" (-not ($r.Out -match "major-version mismatch"))
|
||||
|
||||
Write-Host "Scenario 4: compatible older-major toolkit (-RequireOrExit) -> resolves, env set"
|
||||
$r = Run-Case -FindMode "compatible" -Require $true
|
||||
Check "exits 0" ($r.Exit -eq 0)
|
||||
Check "CudaToolkitReady = true" ($r.Out -match "ready=True")
|
||||
Check "NvccPath published" ($r.Out -match "nvcc=.*nvcc")
|
||||
|
||||
Write-Host "Scenario 4: no toolkit, prebuilt path (no -RequireOrExit) -> defers, no winget"
|
||||
Write-Host "Scenario 5: no toolkit, prebuilt path (no -RequireOrExit) -> defers, no winget"
|
||||
$r = Run-Case -FindMode "none" -Require $false
|
||||
Check "exits 0" ($r.Exit -eq 0)
|
||||
Check "CudaToolkitReady = false" ($r.Out -match "ready=False")
|
||||
Check "winget NOT called" ($r.Out -match "winget=False")
|
||||
|
||||
Write-Host "Scenario 5: no toolkit, forced (-RequireOrExit) -> winget attempted then exit"
|
||||
Write-Host "Scenario 6: no toolkit, forced (-RequireOrExit) -> winget attempted then exit"
|
||||
# The function exits before the RESULT line here, so assert on the winget-block
|
||||
# marker in output rather than the flag.
|
||||
$r = Run-Case -FindMode "none" -Require $true
|
||||
Check "winget attempted" ($r.Out -match "installing via winget")
|
||||
Check "exits non-zero" ($r.Exit -ne 0)
|
||||
Check "preserved nvcc-required error" ($r.Out -match "CUDA Toolkit \(nvcc\) is required")
|
||||
|
||||
Write-Host "Scenario 7: same-major toolkit only on PATH, missed by -MaxVersion (-RequireOrExit) -> accepted, not rejected"
|
||||
# -MaxVersion misses it (not in side-by-side base) but plain Find-Nvcc finds it on PATH: must be used.
|
||||
$r = Run-Case -FindMode "incompatible" -Require $true -DriverMode "same-major"
|
||||
Check "exits 0" ($r.Exit -eq 0)
|
||||
Check "CudaToolkitReady = true" ($r.Out -match "ready=True")
|
||||
Check "NvccPath published" ($r.Out -match "nvcc=.*nvcc-13\.3")
|
||||
Check "no mismatch warning" (-not ($r.Out -match "major-version mismatch"))
|
||||
}
|
||||
finally {
|
||||
Remove-Item -Recurse -Force -LiteralPath $work -ErrorAction SilentlyContinue
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue