diff --git a/.github/workflows/studio-backend-ci.yml b/.github/workflows/studio-backend-ci.yml index 3968f2e80a..ec437e0c32 100644 --- a/.github/workflows/studio-backend-ci.yml +++ b/.github/workflows/studio-backend-ci.yml @@ -30,6 +30,13 @@ on: - 'unsloth/**' - 'unsloth_cli/**' - 'tests/**' + # The root installers: tests/sh/*.sh and tests/studio/install/* assert + # against these two files, so a change here must run the suite that + # covers it. Without them an install-only edit (the shape most AMD/ROCm + # routing fixes take) skipped Backend CI entirely. + - 'install.sh' + - 'install.ps1' + - 'scripts/**' - 'pyproject.toml' - '.github/workflows/studio-backend-ci.yml' push: @@ -217,27 +224,32 @@ jobs: tests/studio/test_xpu_spoof_pipeline.py - name: Shell installer tests - # Subset that does not depend on a writable / pristine install.sh - # tree; test_install_host_defaults.sh checks install.ps1 layout - # which has drifted (separate followup). + # Auto-discovered rather than allowlisted. The old hardcoded list had + # silently fallen seven files behind tests/run_all.sh, including + # test_strixhalo_wsl_reroute.sh -- the only shell coverage of the ROCm + # WSL reroute -- so that suite never ran on a PR. Skips are explicit, + # each with a reason, and tests/studio/test_ci_shell_suite_coverage.py + # fails if this step stops discovering the directory or the skip list + # grows without one. + # + # Skipped: + # test_install_host_defaults.sh: asserts an install.ps1 layout that + # has drifted (separate followup). + # test_install_rollback_lifecycle.sh: already runs on both platforms + # in cross-platform-parity-ci.yml. run: | set -e - for s in \ - tests/sh/test_get_torch_index_url.sh \ - tests/sh/test_mac_intel_compat.sh \ - tests/sh/test_node_decision.sh \ - tests/sh/test_studio_home_node_dir.sh \ - tests/sh/test_system_node_readonly.sh \ - tests/sh/test_nvcc_meets_llama_minimum.sh \ - tests/sh/test_resolve_cuda_archs.sh \ - tests/sh/test_staged_validation_enabled.sh \ - tests/sh/test_tauri_install_exit_order.sh \ - tests/sh/test_torch_constraint.sh \ - tests/sh/test_torch_flavor.sh \ - tests/sh/test_with_llama_cpp_dir_flag.sh \ - tests/sh/test_with_llama_cpp_dir_link_behavior.sh; do + skip="test_install_host_defaults.sh test_install_rollback_lifecycle.sh" + found=0 + for s in tests/sh/test_*.sh; do + case " $skip " in + *" $(basename "$s") "*) echo "skipping $s (see workflow comment)"; continue ;; + esac + found=$((found + 1)) echo "::group::$s" bash "$s" echo "::endgroup::" done + [ "$found" -gt 0 ] || { echo "::error::no shell tests discovered under tests/sh"; exit 1; } + echo "ran $found shell installer test files" diff --git a/install.ps1 b/install.ps1 index 9c91d4ba16..a2aff0b69a 100644 --- a/install.ps1 +++ b/install.ps1 @@ -1917,12 +1917,14 @@ exit 0 # (gfx120X/110X/1151/1150/103X); unknown names fall back to CPU. elseif ($ROCmGpuLabel) { $nameArchTable = @( - @{ P = "9070 XT|9080"; A = "gfx1201" } # RDNA 4 (RX 9070 XT / 9080) - @{ P = "9070|9060"; A = "gfx1200" } # RDNA 4 (RX 9070 / 9060) + @{ P = "9070|9080"; A = "gfx1201" } # RDNA 4 (Navi 48: RX 9070 XT / 9070 GRE / 9070 / 9080) + @{ P = "9060"; A = "gfx1200" } # RDNA 4 (Navi 44: RX 9060 XT / 9060) @{ P = "8065S|8060S|8050S|8040S|Strix Halo|Ryzen AI Max|AI Max"; A = "gfx1151" } # RDNA 3.5 (Strix Halo + Gorgon Halo: Radeon 8065S/8060S/8050S/8040S iGPU, Ryzen AI Max / Max+) - @{ P = "890M|880M|860M|840M|Strix Point|Krackan|HX 37[05]|AI 9 HX|AI 9 36[05]|AI 7 35[05]|AI 5 34[05]|AI 7 PRO 35|AI 5 33"; A = "gfx1150" } # RDNA 3.5 (Strix/Krackan Point: Radeon 890M/880M iGPU, Ryzen AI 9 HX 370/375) - @{ P = "RX 7900|RX 7800|RX 7700(?!S)|PRO W7900|PRO W7800|PRO W7700"; A = "gfx1100" } # RDNA 3 desktop/workstation (Navi 31) - @{ P = "RX 7600|RX 7700S|RX 7650|PRO W7600|PRO W7500|PRO V710"; A = "gfx1102" } # RDNA 3 (Navi 33) + @{ P = "890M|880M|Strix Point|HX 37[05]|AI 9 HX|AI 9 36[05]"; A = "gfx1150" } # RDNA 3.5 (Strix Point: Radeon 890M/880M, Ryzen AI 9 HX 370/375) + @{ P = "860M|840M|Krackan|AI 7 35[05]|AI 5 34[05]|AI 7 PRO 35|AI 5 33"; A = "gfx1152" } # RDNA 3.5 (Krackan Point: Radeon 860M/840M, Ryzen AI 7 350 / AI 5 340) + @{ P = "RX 7900|PRO W7900|PRO W7800"; A = "gfx1100" } # RDNA 3 desktop/workstation (Navi 31) + @{ P = "RX 7800|RX 7700(?!S)|PRO W7700|PRO V710"; A = "gfx1101" } # RDNA 3 (Navi 32) + @{ P = "RX 7600|RX 7700S|RX 7650|PRO W7600|PRO W7500"; A = "gfx1102" } # RDNA 3 (Navi 33) @{ P = "780M|760M|740M|Phoenix|Hawk Point|Z1 Extreme|Z2 Extreme"; A = "gfx1103" } # RDNA 3 iGPU (Phoenix / Hawk Point) @{ P = "RX 6900|RX 6800|RX 6750|RX 6700|PRO W6800|PRO W6900"; A = "gfx1030" } # RDNA 2 (Navi 21) -- gfx103X family @{ P = "RX 6650|RX 6600|PRO W6600|PRO W6650"; A = "gfx1032" } # RDNA 2 (Navi 23) -- gfx103X family @@ -2203,6 +2205,7 @@ exit 0 $archFamilyMap = @{ "gfx1201" = "gfx120X-all"; "gfx1200" = "gfx120X-all" # RDNA 4 "gfx1151" = "gfx1151"; "gfx1150" = "gfx1150" # RDNA 3.5 (Strix Halo/Point) + "gfx1152" = "gfx1152" # RDNA 3.5 (Krackan Point) "gfx1103" = "gfx110X-all"; "gfx1102" = "gfx110X-all" # RDNA 3 "gfx1101" = "gfx110X-all"; "gfx1100" = "gfx110X-all" "gfx1036" = "gfx103X-all"; "gfx1035" = "gfx103X-all" # RDNA 2 (RX 6000) @@ -2224,6 +2227,7 @@ exit 0 $torchFloorMap = @{ "gfx1201" = "torch>=2.11.0,<2.12.0"; "gfx1200" = "torch>=2.11.0,<2.12.0" "gfx1151" = "torch>=2.11.0,<2.12.0"; "gfx1150" = "torch>=2.11.0,<2.12.0" + "gfx1152" = "torch>=2.11.0,<2.12.0" } # Companion ranges track the torch ceiling so pip resolves a consistent # trio on AMD's per-arch index (each published independently). Mirrors @@ -2231,10 +2235,12 @@ exit 0 $torchvisionFloorMap = @{ "gfx1201" = "torchvision>=0.26.0,<0.27.0"; "gfx1200" = "torchvision>=0.26.0,<0.27.0" "gfx1151" = "torchvision>=0.26.0,<0.27.0"; "gfx1150" = "torchvision>=0.26.0,<0.27.0" + "gfx1152" = "torchvision>=0.26.0,<0.27.0" } $torchaudioFloorMap = @{ "gfx1201" = "torchaudio>=2.11.0,<2.12.0"; "gfx1200" = "torchaudio>=2.11.0,<2.12.0" "gfx1151" = "torchaudio>=2.11.0,<2.12.0"; "gfx1150" = "torchaudio>=2.11.0,<2.12.0" + "gfx1152" = "torchaudio>=2.11.0,<2.12.0" } $archFamily = if ($ROCmGfxArch -and $archFamilyMap.ContainsKey($ROCmGfxArch)) { $archFamilyMap[$ROCmGfxArch] } else { $null } if ($archFamily) { @@ -2264,7 +2270,7 @@ exit 0 $_pinRocm211 = ([int]$Matches[1] -eq 7 -and [int]$Matches[2] -eq 2) } # Only the 2.11-allowlist gfx arches need the floor; others publish <2.11 and stay bare. - $_pinGfx211 = @('gfx120x-all', 'gfx1151', 'gfx1150') -contains $_pinLeaf + $_pinGfx211 = @('gfx120x-all', 'gfx1151', 'gfx1150', 'gfx1152') -contains $_pinLeaf if ($_pinGfx211 -or $_pinRocm211) { $ROCmIndexUrl = $TorchIndexUrl $ROCmTorchFloor = "torch>=2.11.0,<2.12.0" diff --git a/install.sh b/install.sh index dface28918..3bc2ff4c88 100755 --- a/install.sh +++ b/install.sh @@ -2260,6 +2260,7 @@ _amd_arch_index_family_for_gfx() { gfx1201|gfx1200) echo gfx120X-all ;; gfx1151) echo gfx1151 ;; gfx1150) echo gfx1150 ;; + gfx1152) echo gfx1152 ;; gfx1103|gfx1102|gfx1101|gfx1100) echo gfx110X-all ;; gfx1036|gfx1035|gfx1034|gfx1033|gfx1032|gfx1031|gfx1030) echo gfx103X-all ;; gfx90a) echo gfx90a ;; @@ -2271,12 +2272,14 @@ _amd_arch_index_family_for_gfx() { # Map a GPU marketing name to gfx arch (kept in sync with install.ps1 nameArchTable). _infer_amd_gfx_arch_from_gpu_name() { case "$1" in - *"9070 XT"*|*9080*) echo gfx1201 ;; - *9070*|*9060*) echo gfx1200 ;; + *9070*|*9080*) echo gfx1201 ;; + *9060*) echo gfx1200 ;; *"8065S"*|*"8060S"*|*"8050S"*|*"8040S"*|*"Strix Halo"*|*"Ryzen AI Max"*|*"AI Max"*) echo gfx1151 ;; - *"890M"*|*"880M"*|*"860M"*|*"840M"*|*"Strix Point"*|*"Krackan"*|*"HX 37"*|*"AI 9 HX"*|*"AI 9 36"*|*"AI 7 35"*|*"AI 5 34"*|*"AI 7 PRO 35"*|*"AI 5 33"*) echo gfx1150 ;; - *"RX 7600"*|*"RX 7700S"*|*"RX 7650"*|*"PRO W7600"*|*"PRO W7500"*|*"PRO V710"*) echo gfx1102 ;; - *"RX 7900"*|*"RX 7800"*|*"RX 7700"*|*"PRO W7900"*|*"PRO W7800"*|*"PRO W7700"*) echo gfx1100 ;; + *"890M"*|*"880M"*|*"Strix Point"*|*"HX 37"*|*"AI 9 HX"*|*"AI 9 36"*) echo gfx1150 ;; + *"860M"*|*"840M"*|*"Krackan"*|*"AI 7 35"*|*"AI 5 34"*|*"AI 7 PRO 35"*|*"AI 5 33"*) echo gfx1152 ;; + *"RX 7600"*|*"RX 7700S"*|*"RX 7650"*|*"PRO W7600"*|*"PRO W7500"*) echo gfx1102 ;; + *"RX 7800"*|*"RX 7700"*|*"PRO W7700"*|*"PRO V710"*) echo gfx1101 ;; + *"RX 7900"*|*"PRO W7900"*|*"PRO W7800"*) echo gfx1100 ;; *"780M"*|*"760M"*|*"740M"*|*"Phoenix"*|*"Hawk Point"*|*"Z1 Extreme"*|*"Z2 Extreme"*) echo gfx1103 ;; *"RX 6900"*|*"RX 6800"*|*"RX 6750"*|*"RX 6700"*|*"PRO W6800"*|*"PRO W6900"*) echo gfx1030 ;; *"RX 6650"*|*"RX 6600"*|*"PRO W6600"*|*"PRO W6650"*) echo gfx1032 ;; @@ -2316,10 +2319,14 @@ _infer_linux_amd_gfx_arch() { echo gfx1151 return 0 fi - if [ -n "$_gpu_evidence" ] && grep -qiE '890M|880M|860M|840M|Strix Point|Krackan|HX 37[05]|AI 9 HX|AI 9 36[05]|AI 7 35[05]|AI 5 34[05]|AI 7 PRO 35|AI 5 33' /proc/cpuinfo 2>/dev/null; then + if [ -n "$_gpu_evidence" ] && grep -qiE '890M|880M|Strix Point|HX 37[05]|AI 9 HX|AI 9 36[05]' /proc/cpuinfo 2>/dev/null; then echo gfx1150 return 0 fi + if [ -n "$_gpu_evidence" ] && grep -qiE '860M|840M|Krackan|AI 7 35[05]|AI 5 34[05]|AI 7 PRO 35|AI 5 33' /proc/cpuinfo 2>/dev/null; then + echo gfx1152 + return 0 + fi if command -v lspci >/dev/null 2>&1; then # A non-AMD controller can enumerate first (Intel/ASPEED before an AMD # dGPU), so scan every display-class line and take the first AMD one @@ -3055,7 +3062,7 @@ if [ "$_torch_index_pinned" = false ] && [ "$SKIP_TORCH" = false ] && \ # whole handoff (a user-set override re-exports unchanged). export UNSLOTH_ROCM_GFX_ARCH="$_linux_inferred_gfx" case "$_linux_inferred_gfx" in - gfx1201|gfx1200|gfx1151|gfx1150) + gfx1201|gfx1200|gfx1151|gfx1150|gfx1152) TORCH_CONSTRAINT="torch>=2.11.0,<2.12.0" TORCHVISION_CONSTRAINT="torchvision>=0.26.0,<0.27.0" TORCHAUDIO_CONSTRAINT="torchaudio>=2.11.0,<2.12.0" @@ -3124,7 +3131,7 @@ fi # and a bare name can resolve a 2.12 ABI-mismatched wheel. Match on the FINAL leaf so a # custom mirror with a gfx/rocm7.2 path segment but a cu*/cpu family isn't forced. case "$_torch_index_leaf" in - rocm7.2|gfx120x-all|gfx1151|gfx1150) + rocm7.2|gfx120x-all|gfx1151|gfx1150|gfx1152) TORCH_CONSTRAINT="torch>=2.11.0,<2.12.0" TORCHVISION_CONSTRAINT="torchvision>=0.26.0,<0.27.0" TORCHAUDIO_CONSTRAINT="torchaudio>=2.11.0,<2.12.0" @@ -3243,7 +3250,7 @@ case "$_torch_index_leaf" in fi _strix_gfx="" case "$_runtime_gfx" in - gfx1151|gfx1150) _strix_gfx="$_runtime_gfx" ;; + gfx1151|gfx1150|gfx1152) _strix_gfx="$_runtime_gfx" ;; esac # Skip rocm7.13+ generic indexes: they already ship the fixes, so the # arch build (rocm7.13) would be a downgrade rather than a rescue. @@ -3339,12 +3346,14 @@ elif case "$TORCH_INDEX_URL" in */rocm*|*/gfx*) true ;; *) false ;; esac; then # gfx1102 matched BEFORE gfx1100 so the spaceless "RX 7700S" lands on # gfx1102 (bash case has no negative lookahead like the PS tables). case "$_gpu_disp_mkt" in - *"9070 XT"*|*9080*) _gpu_disp_gfx="gfx1201" ;; # RDNA 4 - *9070*|*9060*) _gpu_disp_gfx="gfx1200" ;; # RDNA 4 + *9070*|*9080*) _gpu_disp_gfx="gfx1201" ;; # RDNA 4 (Navi 48) + *9060*) _gpu_disp_gfx="gfx1200" ;; # RDNA 4 (Navi 44) *"8065S"*|*"8060S"*|*"8050S"*|*"8040S"*|*"Strix Halo"*|*"Ryzen AI Max"*|*"AI Max"*) _gpu_disp_gfx="gfx1151" ;; # RDNA 3.5 (Strix Halo + Gorgon Halo: Radeon 8065S/8060S/8050S/8040S iGPU, Ryzen AI Max / Max+) - *"890M"*|*"880M"*|*"860M"*|*"840M"*|*"Strix Point"*|*"Krackan"*|*"HX 37"*|*"AI 9 HX"*|*"AI 9 36"*|*"AI 7 35"*|*"AI 5 34"*|*"AI 7 PRO 35"*|*"AI 5 33"*) _gpu_disp_gfx="gfx1150" ;; # RDNA 3.5 (Strix/Krackan Point: Radeon 890M/880M iGPU, Ryzen AI 9 HX 370/375) - *"RX 7600"*|*"RX 7700S"*|*"RX 7650"*|*"PRO W7600"*|*"PRO W7500"*|*"PRO V710"*) _gpu_disp_gfx="gfx1102" ;; # RDNA 3 (Navi 33) - *"RX 7900"*|*"RX 7800"*|*"RX 7700"*|*"PRO W7900"*|*"PRO W7800"*|*"PRO W7700"*) _gpu_disp_gfx="gfx1100" ;; # RDNA 3 desktop / workstation (Navi 31) + *"890M"*|*"880M"*|*"Strix Point"*|*"HX 37"*|*"AI 9 HX"*|*"AI 9 36"*) _gpu_disp_gfx="gfx1150" ;; # RDNA 3.5 (Strix Point: Radeon 890M/880M, Ryzen AI 9 HX 370/375) + *"860M"*|*"840M"*|*"Krackan"*|*"AI 7 35"*|*"AI 5 34"*|*"AI 7 PRO 35"*|*"AI 5 33"*) _gpu_disp_gfx="gfx1152" ;; # RDNA 3.5 (Krackan Point: Radeon 860M/840M, Ryzen AI 7 350 / AI 5 340) + *"RX 7600"*|*"RX 7700S"*|*"RX 7650"*|*"PRO W7600"*|*"PRO W7500"*) _gpu_disp_gfx="gfx1102" ;; # RDNA 3 (Navi 33) + *"RX 7800"*|*"RX 7700"*|*"PRO W7700"*|*"PRO V710"*) _gpu_disp_gfx="gfx1101" ;; # RDNA 3 (Navi 32) + *"RX 7900"*|*"PRO W7900"*|*"PRO W7800"*) _gpu_disp_gfx="gfx1100" ;; # RDNA 3 desktop / workstation (Navi 31) *"780M"*|*"760M"*|*"740M"*|*"Phoenix"*|*"Hawk Point"*|*"Z1 Extreme"*|*"Z2 Extreme"*) _gpu_disp_gfx="gfx1103" ;; # RDNA 3 iGPU (Phoenix / Hawk Point) *"RX 6900"*|*"RX 6800"*|*"RX 6750"*|*"RX 6700"*|*"PRO W6800"*|*"PRO W6900"*) _gpu_disp_gfx="gfx1030" ;; # RDNA 2 (Navi 21) *"RX 6650"*|*"RX 6600"*|*"PRO W6600"*|*"PRO W6650"*) _gpu_disp_gfx="gfx1032" ;; # RDNA 2 (Navi 23) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index d46e5a0fe1..e54b5269c1 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -3183,7 +3183,7 @@ class LlamaCppBackend: @staticmethod def _amd_apu_wants_unified_memory(gpu_indices = None) -> bool: - """True only for AMD unified-memory APUs (gfx1150/gfx1151), where + """True only for AMD unified-memory APUs (gfx1150/gfx1151/gfx1152), where GGML_CUDA_ENABLE_UNIFIED_MEMORY lets llama.cpp use shared system RAM (it hurts discrete GPUs). gpu_indices (PHYSICAL ids) scopes the check to the selected GPUs, so a dGPU on a mixed host is not treated as unified-memory; @@ -3213,7 +3213,9 @@ class LlamaCppBackend: ) arch_by_id[pid] = _arch.split(":")[0].strip().lower() for _i in list(gpu_indices) if gpu_indices is not None else list(arch_by_id): - if arch_by_id.get(_i) in {"gfx1150", "gfx1151"}: + # gfx1152 is Krackan Point (Radeon 860M/840M), the third RDNA 3.5 + # APU: same shared GPU/system-RAM pool as Strix Point/Halo. + if arch_by_id.get(_i) in {"gfx1150", "gfx1151", "gfx1152"}: return True except Exception: return False diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py index 03327d3320..baf6329dae 100644 --- a/studio/backend/core/training/worker.py +++ b/studio/backend/core/training/worker.py @@ -764,8 +764,8 @@ def _rocm_classify_unified_memory(props: Any) -> tuple[str, bool]: - ``gcn_arch``: canonical arch string (e.g. ``"gfx1151"``) when a known attribute is present, else ``""``. - ``is_unified``: ``True`` for AMD APUs with a shared GPU/system-RAM pool - (gfx1150 Strix Point, gfx1151 Strix Halo) — these need a lower - ``set_per_process_memory_fraction`` cap to leave OS headroom. + (gfx1150 Strix Point, gfx1151 Strix Halo, gfx1152 Krackan Point) — these + need a lower ``set_per_process_memory_fraction`` cap to leave OS headroom. Classification priority: 1. ``props.is_integrated`` truthy (hipDeviceProp_t.integrated -- the @@ -778,6 +778,7 @@ def _rocm_classify_unified_memory(props: Any) -> tuple[str, bool]: - gfx1151 Strix Halo / Gorgon Halo: ``Radeon 8065S`` (Ryzen AI Max+ 495), ``Radeon 8060S`` (Ryzen AI MAX+ 395), ``Radeon 8050S`` (cut-down SKU) + - gfx1152 Krackan Point: ``Radeon 860M``, ``Radeon 840M`` """ gcn_arch = "" for _attr in ("gcnArchName", "gcn_arch_name", "arch_name", "gfx_arch_name"): @@ -797,9 +798,13 @@ def _rocm_classify_unified_memory(props: Any) -> tuple[str, bool]: return gcn_arch, True if gcn_arch: - return gcn_arch, gcn_arch in {"gfx1150", "gfx1151"} + # gfx1152 is Krackan Point, the third RDNA 3.5 APU: same shared + # GPU/system-RAM pool as Strix Point (gfx1150) and Strix Halo (gfx1151). + return gcn_arch, gcn_arch in {"gfx1150", "gfx1151", "gfx1152"} - # Arch attrs absent — fall back to device-name matching. + # Arch attrs absent — fall back to device-name matching. Only reached under + # _hw.IS_ROCM, so the NVIDIA GeForce 840M cannot collide with the Krackan + # markers here. dev_lower = (getattr(props, "name", "") or "").lower() is_unified = ( "890m" in dev_lower @@ -807,6 +812,8 @@ def _rocm_classify_unified_memory(props: Any) -> tuple[str, bool]: or "8065s" in dev_lower or "8060s" in dev_lower or "8050s" in dev_lower + or "860m" in dev_lower + or "840m" in dev_lower ) return gcn_arch, is_unified @@ -2828,7 +2835,7 @@ def run_training_process(*, event_queue: Any, stop_queue: Any, config: dict) -> # On ROCm, exhausting VRAM can hang the HIP driver instead of raising. # set_per_process_memory_fraction caps the allocator so PyTorch raises # OutOfMemoryError first (NVIDIA already has a graceful OOM path). - # Unified-memory APUs (gfx1150/gfx1151) share GPU+system RAM, so use 0.80 + # Unified-memory APUs (gfx1150/gfx1151/gfx1152) share GPU+system RAM, so use 0.80 # vs 0.90 for discrete. Classify via gcnArchName, else device-name markers. # Non-fatal: skipped if torch is not importable. if _hw.IS_ROCM: diff --git a/studio/backend/tests/test_amd_apu_unified_memory.py b/studio/backend/tests/test_amd_apu_unified_memory.py index 9fd8260bf2..be85fd56d1 100644 --- a/studio/backend/tests/test_amd_apu_unified_memory.py +++ b/studio/backend/tests/test_amd_apu_unified_memory.py @@ -2,7 +2,7 @@ # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 """GGML_CUDA_ENABLE_UNIFIED_MEMORY must be set only for AMD unified-memory APUs -(gfx1150/gfx1151), never for discrete AMD, NVIDIA, CPU or macOS.""" +(gfx1150/gfx1151/gfx1152), never for discrete AMD, NVIDIA, CPU or macOS.""" from __future__ import annotations @@ -35,6 +35,8 @@ def _fake_torch( [ ("6.2.0", ["gfx1151:xnack-"], True), # Strix Halo APU (suffix stripped) ("6.2.0", ["gfx1150"], True), # Strix Point APU + ("6.2.0", ["gfx1152"], True), # Krackan Point APU (Radeon 860M/840M) + ("6.2.0", ["gfx1152:sramecc-:xnack-"], True), # same, feature flags stripped ("6.2.0", ["gfx1100"], False), # discrete RDNA3 ("6.2.0", ["gfx1201"], False), # discrete RDNA4 ("6.2.0", ["gfx942"], False), # MI300X (data center) diff --git a/studio/backend/tests/test_grouped_mm_rdna4_fallback.py b/studio/backend/tests/test_grouped_mm_rdna4_fallback.py new file mode 100644 index 0000000000..675b9c3210 --- /dev/null +++ b/studio/backend/tests/test_grouped_mm_rdna4_fallback.py @@ -0,0 +1,418 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Numerics + gating for the RDNA4 _grouped_mm CPU fallback (PRs #7276 / #7292). + +RDNA4 (gfx1200/gfx1201) ships a null HIP `_grouped_mm` kernel on ROCm <= 7.12 +(fixed in 7.13, ROCm/TheRock #5284). Training MoE models there crashes with +0xC0000005 on Windows and a plain segfault on Linux, so worker.py registers a +Python mm/bmm fallback on the CUDA dispatch key. + +The fallback is silent, GPU-gated, and reimplements a matmul: if it is wrong, an +RX 9070 user does not crash, they train on quietly wrong gradients. Until now the +only coverage was `assert '_gm_lib.impl("_grouped_mm"' in source` -- the math was +never executed once, in any suite. + +worker.py cannot be imported here (module-level structlog/backend imports), so +`_install_grouped_mm_cpu_fallback` is lifted out with ast and driven with a fake +`torch_mod` that forwards to real CPU torch. That also pins the op surface: the +fallback may only use the ops the fake exposes, and the registration is captured +instead of hitting a real CUDA dispatch key that CI runners do not have. + +The two gates around it are exec'd straight out of the source so this file tests +the shipped expressions rather than a copy of them. +""" + +import ast +import re +import textwrap +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch + + +_WORKER_PATH = Path(__file__).resolve().parents[1] / "core" / "training" / "worker.py" +_WORKER_SOURCE = _WORKER_PATH.read_text(encoding = "utf-8") + + +def _load_installer(): + """exec just _install_grouped_mm_cpu_fallback out of worker.py.""" + tree = ast.parse(_WORKER_SOURCE) + fn = [ + n + for n in tree.body + if isinstance(n, ast.FunctionDef) and n.name == "_install_grouped_mm_cpu_fallback" + ] + assert fn, "_install_grouped_mm_cpu_fallback not found in core/training/worker.py" + ns: dict = {} + exec(compile(ast.Module(body = fn, type_ignores = []), str(_WORKER_PATH), "exec"), ns) + return ns["_install_grouped_mm_cpu_fallback"] + + +_install_grouped_mm_cpu_fallback = _load_installer() + + +class _RecordingLibrary: + """Stands in for torch.library.Library: captures the registration instead of + binding it to a CUDA dispatch key no CI runner has.""" + + def __init__(self, namespace, kind): + self.namespace = namespace + self.kind = kind + self.registrations = [] + + def impl(self, name, fn, dispatch_key): + self.registrations.append((name, fn, dispatch_key)) + + +class _RecordingLogger: + def __init__(self): + self.info_calls = [] + self.warning_calls = [] + + def info(self, *args, **kwargs): + self.info_calls.append(args) + + def warning(self, *args, **kwargs): + self.warning_calls.append(args) + + +def _fake_torch(): + """Real CPU torch behind the exact op surface the fallback is allowed to use. + + Anything else the fallback reaches for raises AttributeError here, which is + the point: a new dependency has to be a deliberate edit, not a silent one.""" + return SimpleNamespace( + library = SimpleNamespace(Library = _RecordingLibrary), + mm = torch.mm, + bmm = torch.bmm, + matmul = torch.matmul, + cat = torch.cat, + zeros = torch.zeros, + ) + + +@pytest.fixture +def fallback(): + """The registered _grouped_mm implementation, plus the Library it landed on.""" + torch_mod = _fake_torch() + logger = _RecordingLogger() + lib = _install_grouped_mm_cpu_fallback(torch_mod, logger, "test") + assert lib.registrations, "the fallback registered nothing" + name, fn, key = lib.registrations[0] + return SimpleNamespace(fn = fn, lib = lib, logger = logger, name = name, key = key) + + +class TestRegistration: + """Where the override lands. Getting the namespace or dispatch key wrong is a + silent no-op: training still crashes on the null HIP kernel.""" + + def test_overrides_aten_grouped_mm_on_the_cuda_key(self, fallback): + assert fallback.lib.namespace == "aten" + assert fallback.lib.kind == "IMPL" + assert fallback.name == "_grouped_mm" + # ROCm dispatches through the CUDA key; "HIP"/"PrivateUse1" would not bind. + assert fallback.key == "CUDA" + + def test_registers_exactly_once(self, fallback): + assert len(fallback.lib.registrations) == 1 + + def test_returns_the_library_so_the_caller_can_keep_it_alive(self, fallback): + """A dropped Library is garbage collected and the override silently + unregisters mid-run; worker.py parks it in a module global.""" + assert isinstance(fallback.lib, _RecordingLibrary) + assert "_WINDOWS_ROCM_GROUPED_MM_LIB = _install_grouped_mm_cpu_fallback(" in _WORKER_SOURCE + + def test_logs_the_patch_with_its_label(self, fallback): + assert fallback.logger.info_calls, "the patch must be visible in the run log" + assert "test" in fallback.logger.info_calls[0] + + +class TestUngroupedNumerics: + """offs=None: plain matmul, one path per rank combination. The 3-D case is + the regression #7292 fixed -- an unconditional mm() broke MoE experts.""" + + def test_2d_by_2d_matches_mm(self, fallback): + a = torch.randn(6, 4) + b = torch.randn(4, 5) + torch.testing.assert_close(fallback.fn(a, b), torch.mm(a, b)) + + def test_3d_by_3d_matches_bmm(self, fallback): + a = torch.randn(3, 6, 4) + b = torch.randn(3, 4, 5) + torch.testing.assert_close(fallback.fn(a, b), torch.bmm(a, b)) + + def test_3d_by_2d_matches_matmul(self, fallback): + a = torch.randn(3, 6, 4) + b = torch.randn(4, 5) + torch.testing.assert_close(fallback.fn(a, b), torch.matmul(a, b)) + + def test_2d_by_3d_matches_matmul(self, fallback): + a = torch.randn(6, 4) + b = torch.randn(3, 4, 5) + torch.testing.assert_close(fallback.fn(a, b), torch.matmul(a, b)) + + def test_non_contiguous_inputs_are_handled(self, fallback): + """Transposed views reach _grouped_mm constantly; every path calls + .contiguous() and this catches it if one stops.""" + a = torch.randn(4, 6).t() + b = torch.randn(5, 4).t() + torch.testing.assert_close(fallback.fn(a, b), torch.mm(a, b)) + + +class TestGroupedNumerics: + """offs=[end-row of each group], the MoE token-routing layout.""" + + def test_matches_per_group_mm_with_3d_weights(self, fallback): + a = torch.randn(7, 4) + b = torch.randn(3, 4, 5) + offs = torch.tensor([2, 5, 7]) + expected = torch.cat([a[0:2] @ b[0], a[2:5] @ b[1], a[5:7] @ b[2]], dim = 0) + torch.testing.assert_close(fallback.fn(a, b, offs), expected) + + def test_shared_2d_weight_is_reused_for_every_group(self, fallback): + a = torch.randn(7, 4) + b = torch.randn(4, 5) + offs = torch.tensor([2, 5, 7]) + torch.testing.assert_close(fallback.fn(a, b, offs), a @ b) + + def test_empty_group_produces_no_rows(self, fallback): + """An expert that routed zero tokens (offs[i] == offs[i-1]) must + contribute nothing, not a stray row.""" + a = torch.randn(5, 4) + b = torch.randn(3, 4, 5) + offs = torch.tensor([2, 2, 5]) + expected = torch.cat([a[0:2] @ b[0], a[2:5] @ b[2]], dim = 0) + got = fallback.fn(a, b, offs) + assert got.shape == (5, 5) + torch.testing.assert_close(got, expected) + + def test_rows_past_the_last_offset_are_not_dropped(self, fallback): + """Trailing tokens beyond offs[-1] go through the last expert; dropping + them would silently shrink the output instead of raising.""" + a = torch.randn(7, 4) + b = torch.randn(3, 4, 5) + offs = torch.tensor([2, 5]) + expected = torch.cat([a[0:2] @ b[0], a[2:5] @ b[1], a[5:7] @ b[-1]], dim = 0) + got = fallback.fn(a, b, offs) + assert got.shape[0] == a.shape[0] + torch.testing.assert_close(got, expected) + + def test_zero_rows_returns_an_empty_result_not_an_error(self, fallback): + a = torch.randn(0, 4) + b = torch.randn(3, 4, 5) + offs = torch.tensor([], dtype = torch.int64) + got = fallback.fn(a, b, offs) + assert got.shape == (0, 5) + assert got.dtype == a.dtype + + def test_offsets_may_arrive_as_a_device_tensor_of_any_int_dtype(self, fallback): + a = torch.randn(4, 4) + b = torch.randn(2, 4, 5) + expected = torch.cat([a[0:2] @ b[0], a[2:4] @ b[1]], dim = 0) + for dtype in (torch.int32, torch.int64): + torch.testing.assert_close( + fallback.fn(a, b, torch.tensor([2, 4], dtype = dtype)), expected + ) + + +class TestBiasAndDtype: + def test_bias_is_added(self, fallback): + a = torch.randn(6, 4) + b = torch.randn(4, 5) + bias = torch.randn(5) + torch.testing.assert_close(fallback.fn(a, b, None, bias), torch.mm(a, b) + bias) + + def test_bias_is_added_on_the_grouped_path_too(self, fallback): + a = torch.randn(4, 4) + b = torch.randn(2, 4, 5) + bias = torch.randn(5) + offs = torch.tensor([2, 4]) + expected = torch.cat([a[0:2] @ b[0], a[2:4] @ b[1]], dim = 0) + bias + torch.testing.assert_close(fallback.fn(a, b, offs, bias), expected) + + def test_out_dtype_is_honoured(self, fallback): + a = torch.randn(6, 4) + b = torch.randn(4, 5) + got = fallback.fn(a, b, None, None, torch.float64) + assert got.dtype == torch.float64 + torch.testing.assert_close(got, torch.mm(a, b).to(torch.float64)) + + def test_promotion_from_bias_is_cast_back_to_the_input_dtype(self, fallback): + """Without the restore, a promoted result changes the autograd dtype + downstream of every MoE layer.""" + a = torch.randn(6, 4, dtype = torch.float32) + b = torch.randn(4, 5, dtype = torch.float32) + bias = torch.randn(5, dtype = torch.float64) + got = fallback.fn(a, b, None, bias) + assert got.dtype == torch.float32 + + def test_out_dtype_wins_over_the_input_dtype_restore(self, fallback): + a = torch.randn(6, 4, dtype = torch.float32) + b = torch.randn(4, 5, dtype = torch.float32) + bias = torch.randn(5, dtype = torch.float64) + got = fallback.fn(a, b, None, bias, torch.float64) + assert got.dtype == torch.float64 + + def test_bf16_inputs_stay_bf16(self, fallback): + """The dtype training actually runs in.""" + a = torch.randn(6, 4).to(torch.bfloat16) + b = torch.randn(4, 5).to(torch.bfloat16) + got = fallback.fn(a, b) + assert got.dtype == torch.bfloat16 + torch.testing.assert_close(got.float(), (a.float() @ b.float()), rtol = 2e-2, atol = 2e-2) + + +def _exec_source_snippet(anchor: str, last_line: str, **variables): + """Run a slice of worker.py verbatim, so the gate under test is the shipped + one and not a copy that can drift.""" + start = _WORKER_SOURCE.find(anchor) + assert start != -1, f"gate snippet not found in worker.py: {anchor!r}" + start = _WORKER_SOURCE.rfind("\n", 0, start) + 1 # keep the indent for dedent() + end = _WORKER_SOURCE.find(last_line, start) + assert end != -1, f"end of gate snippet not found: {last_line!r}" + snippet = textwrap.dedent(_WORKER_SOURCE[start : end + len(last_line)]) + ns = {"re": re, **variables} + exec(compile(snippet, str(_WORKER_PATH), "exec"), ns) + return ns + + +class TestLinuxHipVersionGate: + """PR #7292's Linux gate. Too low a floor keeps the slow Python fallback on + fixed ROCm 7.13+; too high reintroduces the segfault on 7.12.""" + + _ANCHOR = '_m = re.match(r"(\\d+)\\.(\\d+)", _hip_str)' + _LAST = '_hip_lt_713 = "rocmsdk" not in _ver' + + def _decide(self, hip_str, version): + ns = _exec_source_snippet(self._ANCHOR, self._LAST, _hip_str = hip_str, _ver = version.lower()) + return ns["_hip_lt_713"] + + @pytest.mark.parametrize( + "hip_str,version,affected", + [ + ("7.12.0", "2.10.0+rocm7.12.0", True), # the broken kernel + ("7.6.0", "2.9.0+rocm7.6.0", True), + ("6.4.0", "2.8.0+rocm6.4.0", True), + ("7.13.0", "2.11.0+rocm7.13.0", False), # AMD's fix + ("7.14.0", "2.11.0+rocm7.14.0", False), + ("8.0.0", "2.12.0+rocm8.0.0", False), + ], + ) + def test_torch_version_hip_decides_when_present(self, hip_str, version, affected): + assert self._decide(hip_str, version) is affected + + @pytest.mark.parametrize( + "version,affected", + [ + ("2.10.0+rocm7.12.0", True), + ("2.11.0+rocm7.13.0", False), + ("2.11.0+rocm7.14.0", False), + ], + ) + def test_falls_back_to_the_rocm_tag_in_torch_version(self, version, affected): + """AMD SDK / Radeon wheels leave torch.version.hip unset.""" + assert self._decide("", version) is affected + + def test_unknown_version_is_assumed_affected(self): + """Fallback is slow but correct; a missed guard is a crash.""" + assert self._decide("", "2.9.0+unknown") is True + + def test_rocmsdk_wheels_without_a_version_are_assumed_fixed(self): + """rocmsdk wheels post-date the gfx120X fix.""" + assert self._decide("", "2.10.0+rocmsdk20260107") is False + + +class TestLinuxRdna4NameMatch: + """The name regex is the fallback when a wheel omits gcnArchName.""" + + def _pattern(self): + """Read whatever pattern worker.py currently uses, not a copy of the one + it used when this test was written. Anchoring on the literal pattern text + would make a *widened* regex -- the dangerous edit, since it silently + forces the slow Python fallback onto RDNA3 users -- fail as "moved" + instead of being checked against the cases below.""" + m = re.search(r"re\.search\(r\"([^\"]+)\",\s*_lin_name\)", _WORKER_SOURCE) + assert m, "could not locate the RDNA4 device-name regex in worker.py" + return m.group(1) + + def test_name_is_lowercased_before_matching(self): + """The pattern is all-lowercase, so it only works against a lowercased + name. Device names arrive mixed case ("AMD Radeon RX 9070 XT").""" + assert self._pattern() == self._pattern().lower(), "pattern is not all-lowercase" + assert re.search( + r"_lin_name\s*=\s*\(getattr\(_props,\s*\"name\",\s*\"\"\)\s*or\s*\"\"\)\.lower\(\)", + _WORKER_SOURCE, + ), "worker.py must lowercase the device name before matching the RDNA4 pattern" + + def test_name_match_is_only_a_fallback_when_arch_is_unknown(self): + """gcnArchName is authoritative when present. Letting the name regex fire + alongside a known arch would misclassify any card whose marketing name + happens to look RDNA4.""" + assert re.search( + r"not _lin_arch and re\.search\(r\"[^\"]+\",\s*_lin_name\)", _WORKER_SOURCE + ), "the RDNA4 name regex must be guarded by `not _lin_arch`" + + @pytest.mark.parametrize( + "name,is_rdna4", + [ + ("AMD Radeon RX 9070 XT", True), + ("AMD Radeon RX 9060 XT", True), + ("Radeon RX9070", True), + ("AMD Radeon AI PRO R9700", True), + ("AMD Radeon RX 7900 XTX", False), # RDNA3, kernel is fine + ("AMD Radeon 8060S Graphics", False), # Strix Halo + ("AMD Radeon RX 6800 XT", False), + ("NVIDIA GeForce RTX 4090", False), + ], + ) + def test_matches_only_rdna4_cards(self, name, is_rdna4): + assert bool(re.search(self._pattern(), name.lower())) is is_rdna4 + + +class TestLinuxGateStructure: + """The block is a few hundred lines into run_training_process and can only be + checked structurally; these pin the parts a refactor would quietly drop.""" + + def _linux_block(self): + start = _WORKER_SOURCE.find("1f-linux") + assert start != -1, "the Linux ROCm gfx120X guard (#7292) is gone from worker.py" + end = _WORKER_SOURCE.find("1g.", start) + assert end != -1 + return _WORKER_SOURCE[start:end] + + def test_gated_on_linux_and_rocm(self): + block = self._linux_block() + assert 'sys.platform.startswith("linux")' in block + assert "_hw.IS_ROCM" in block, "guard must not run on NVIDIA/CPU hosts" + + def test_requires_both_rdna4_and_an_affected_hip(self): + block = self._linux_block() + assert "if _rdna4 and _hip_lt_713:" in block + + def test_scans_every_visible_device(self): + """device_map="balanced" can place layers on a later card, so checking + device 0 alone misses the RDNA4 GPU.""" + block = self._linux_block() + assert "for _i in range(_torch_lin.cuda.device_count()):" in block + + def test_matches_both_rdna4_arch_ids(self): + block = self._linux_block() + assert '("gfx1200", "gfx1201")' in block + + def test_failure_to_patch_is_non_fatal(self): + """A broken patch attempt must not take down the whole training run.""" + block = self._linux_block() + assert "except Exception" in block + assert "logger.warning" in block + + def test_windows_and_linux_share_one_implementation(self): + """Two copies of this fallback would drift; #7292 deliberately hoisted it.""" + assert _WORKER_SOURCE.count("def _install_grouped_mm_cpu_fallback(") == 1 + assert _WORKER_SOURCE.count("_install_grouped_mm_cpu_fallback(") >= 3 # def + win32 + linux + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/studio/backend/tests/test_rocm_oom_guard.py b/studio/backend/tests/test_rocm_oom_guard.py index ad46f6ee41..5cdbe4f2a5 100644 --- a/studio/backend/tests/test_rocm_oom_guard.py +++ b/studio/backend/tests/test_rocm_oom_guard.py @@ -80,6 +80,7 @@ class TestCanonicalGcnArchName: [ ("gfx1150", True), # Strix Point ("gfx1151", True), # Strix Halo + ("gfx1152", True), # Krackan Point (Radeon 860M/840M) ("gfx1100", False), # Navi 31 (RX 7900 XTX) — discrete ("gfx906", False), # MI50 — discrete server GPU ("gfx1201", False), # RX 9070 XT — discrete @@ -166,9 +167,15 @@ class TestDeviceNameFallback: # gfx1151 Gorgon Halo (Ryzen AI Max 400 refresh) "Radeon 8065S Graphics", # Ryzen AI Max+ 495 "AMD Radeon 8065S", + # gfx1152 Krackan Point (Ryzen AI 7 350 / AI 5 340) + "Radeon 860M", + "AMD Radeon 860M Graphics", + "Radeon 840M", + "AMD Radeon 840M Graphics", # case variants "RADEON 8060S GRAPHICS", "radeon 8050s", + "RADEON 860M", ], ) def test_unified_memory_detected(self, device_name: str) -> None: diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py index a29ba0d7e5..8c33cb6ce9 100644 --- a/studio/install_python_stack.py +++ b/studio/install_python_stack.py @@ -96,7 +96,9 @@ def _strix_needs_amd_arch_index(ver: tuple[int, int]) -> bool: # AMD per-arch leaves needing the torch 2.11 floor (the _grouped_mm <2.11 bug). # Mirrors *FloorMap in install.ps1 / setup.ps1; other arches ship <2.11 and stay bare. -_ROCM_GFX_TORCH211_LEAVES: frozenset[str] = frozenset({"gfx120x-all", "gfx1151", "gfx1150"}) +_ROCM_GFX_TORCH211_LEAVES: frozenset[str] = frozenset( + {"gfx120x-all", "gfx1151", "gfx1150", "gfx1152"} +) # pytorch.org rocmX.Y indexes KNOWN to ship torch 2.11 (rocm7.2 only today); don't # floor an unknown newer rocm speculatively. Match install.sh / setup.ps1 / install.ps1. @@ -124,6 +126,7 @@ _WINDOWS_ROCM_TORCH_PKG_SPECS: dict[str, tuple[str, str, str]] = { "gfx1200": _ROCM_TORCH_PKG_SPECS["rocm7.2"], "gfx1151": _ROCM_TORCH_PKG_SPECS["rocm7.2"], "gfx1150": _ROCM_TORCH_PKG_SPECS["rocm7.2"], + "gfx1152": _ROCM_TORCH_PKG_SPECS["rocm7.2"], } _PYTORCH_WHL_BASE = ( os.environ.get("UNSLOTH_PYTORCH_MIRROR") or "https://download.pytorch.org/whl" @@ -369,6 +372,7 @@ _GFX_TO_AMD_INDEX_ARCH: dict[str, str] = { "gfx1200": "gfx120X-all", # RDNA 4 "gfx1151": "gfx1151", "gfx1150": "gfx1150", # RDNA 3.5 (Strix Halo/Point) + "gfx1152": "gfx1152", # RDNA 3.5 (Krackan Point) "gfx1103": "gfx110X-all", "gfx1102": "gfx110X-all", # RDNA 3 "gfx1101": "gfx110X-all", @@ -738,19 +742,18 @@ def _detect_windows_gfx_arch() -> str | None: # prebuilts / AMD Windows torch indexes support; unknown names return None # (callers then fall back cleanly to CPU). _WIN_GPU_NAME_ARCH_TABLE: "list[tuple[str, str]]" = [ - (r"9070 XT|9080", "gfx1201"), # RDNA 4 (Radeon RX 9070 XT / 9080) - (r"9070|9060", "gfx1200"), # RDNA 4 (Radeon RX 9070 / 9060) + (r"9070|9080", "gfx1201"), # RDNA 4 (Navi 48: Radeon RX 9070 XT / 9070 GRE / 9070 / 9080) + (r"9060", "gfx1200"), # RDNA 4 (Navi 44: Radeon RX 9060 XT / 9060) # RDNA 3.5 (Strix Halo + Gorgon Halo: Radeon 8065S/8060S/8050S/8040S iGPU, Ryzen AI Max / Max+) (r"8065S|8060S|8050S|8040S|Strix Halo|Ryzen AI Max|AI Max", "gfx1151"), - # RDNA 3.5 (Strix/Krackan Point: Radeon 890M/880M iGPU, Ryzen AI 9 HX 370/375) - ( - r"890M|880M|860M|840M|Strix Point|Krackan|HX 37[05]|AI 9 HX|AI 9 36[05]" - r"|AI 7 35[05]|AI 5 34[05]|AI 7 PRO 35|AI 5 33", - "gfx1150", - ), + # RDNA 3.5 (Strix Point: Radeon 890M/880M, Ryzen AI 9 HX 370/375) + (r"890M|880M|Strix Point|HX 37[05]|AI 9 HX|AI 9 36[05]", "gfx1150"), + # RDNA 3.5 (Krackan Point: Radeon 860M/840M, Ryzen AI 7 350 / AI 5 340) + (r"860M|840M|Krackan|AI 7 35[05]|AI 5 34[05]|AI 7 PRO 35|AI 5 33", "gfx1152"), # RDNA 3 desktop / workstation (Navi 31) - (r"RX 7900|RX 7800|RX 7700(?!S)|PRO W7900|PRO W7800|PRO W7700", "gfx1100"), - (r"RX 7600|RX 7700S|RX 7650|PRO W7600|PRO W7500|PRO V710", "gfx1102"), # Navi 33 + (r"RX 7900|PRO W7900|PRO W7800", "gfx1100"), + (r"RX 7800|RX 7700(?!S)|PRO W7700|PRO V710", "gfx1101"), # Navi 32 + (r"RX 7600|RX 7700S|RX 7650|PRO W7600|PRO W7500", "gfx1102"), # Navi 33 # RDNA 3 iGPU (Phoenix / Hawk Point) (r"780M|760M|740M|Phoenix|Hawk Point|Z1 Extreme|Z2 Extreme", "gfx1103"), (r"RX 6900|RX 6800|RX 6750|RX 6700|PRO W6800|PRO W6900", "gfx1030"), # Navi 21 @@ -777,13 +780,12 @@ def _linux_amd_gfx_from_cpuinfo() -> "str | None": return None if re.search(r"Ryzen AI Max|Radeon 80[0-9][05]S|Strix Halo", text, re.IGNORECASE): return "gfx1151" - if re.search( - r"890M|880M|860M|840M|Strix Point|Krackan|HX 37[05]|AI 9 HX|AI 9 36[05]" - r"|AI 7 35[05]|AI 5 34[05]|AI 7 PRO 35|AI 5 33", - text, - re.IGNORECASE, - ): + if re.search(r"890M|880M|Strix Point|HX 37[05]|AI 9 HX|AI 9 36[05]", text, re.IGNORECASE): return "gfx1150" + if re.search( + r"860M|840M|Krackan|AI 7 35[05]|AI 5 34[05]|AI 7 PRO 35|AI 5 33", text, re.IGNORECASE + ): + return "gfx1152" return None @@ -1896,7 +1898,7 @@ def _ensure_rocm_torch() -> None: # An explicit ROCm pin is authoritative: never auto-reroute it. if _strix_needs_amd_arch_index(ver) and _explicit_rocm_torch_index_url() is None: gfx_codes = _detect_amd_gfx_codes() - _strix_gfx = {"gfx1151", "gfx1150"} + _strix_gfx = {"gfx1151", "gfx1150", "gfx1152"} _detected_strix = _strix_gfx.intersection(gfx_codes) if _detected_strix: # Runtime-visible GPU (HIP_VISIBLE_DEVICES index into gfx_codes, else first); diff --git a/studio/setup.ps1 b/studio/setup.ps1 index c6932121c9..6a8499b195 100644 --- a/studio/setup.ps1 +++ b/studio/setup.ps1 @@ -484,7 +484,7 @@ function Redact-InstallOutput { # the install-spec path below and the other installers; other leaves ship <2.11 and stay default. function Test-RocmGfx211Leaf { param([string]$Leaf) - return @('gfx120x-all', 'gfx1151', 'gfx1150') -contains $Leaf + return @('gfx120x-all', 'gfx1151', 'gfx1150', 'gfx1152') -contains $Leaf } # rocmX.Y versions KNOWN to ship torch 2.11: rocm7.2 only today. Do NOT floor an unknown newer @@ -1496,12 +1496,14 @@ if (-not $HasNvidiaSmi) { # (gfx120X/110X/1151/1150/103X); unknown names fall back cleanly to CPU. elseif ($ROCmGpuLabel) { $nameArchTable = @( - @{ P = "9070 XT|9080"; A = "gfx1201" } # RDNA 4 (Radeon RX 9070 XT / 9080) - @{ P = "9070|9060"; A = "gfx1200" } # RDNA 4 (Radeon RX 9070 / 9060) + @{ P = "9070|9080"; A = "gfx1201" } # RDNA 4 (Navi 48: Radeon RX 9070 XT / 9070 GRE / 9070 / 9080) + @{ P = "9060"; A = "gfx1200" } # RDNA 4 (Navi 44: Radeon RX 9060 XT / 9060) @{ P = "8065S|8060S|8050S|8040S|Strix Halo|Ryzen AI Max|AI Max"; A = "gfx1151" } # RDNA 3.5 (Strix Halo + Gorgon Halo: Radeon 8065S/8060S/8050S/8040S iGPU, Ryzen AI Max / Max+) - @{ P = "890M|880M|860M|840M|Strix Point|Krackan|HX 37[05]|AI 9 HX|AI 9 36[05]|AI 7 35[05]|AI 5 34[05]|AI 7 PRO 35|AI 5 33"; A = "gfx1150" } # RDNA 3.5 (Strix/Krackan Point: Radeon 890M/880M iGPU, Ryzen AI 9 HX 370/375) - @{ P = "RX 7900|RX 7800|RX 7700(?!S)|PRO W7900|PRO W7800|PRO W7700"; A = "gfx1100" } # RDNA 3 desktop / workstation (Navi 31) - @{ P = "RX 7600|RX 7700S|RX 7650|PRO W7600|PRO W7500|PRO V710"; A = "gfx1102" } # RDNA 3 (Navi 33) + @{ P = "890M|880M|Strix Point|HX 37[05]|AI 9 HX|AI 9 36[05]"; A = "gfx1150" } # RDNA 3.5 (Strix Point: Radeon 890M/880M, Ryzen AI 9 HX 370/375) + @{ P = "860M|840M|Krackan|AI 7 35[05]|AI 5 34[05]|AI 7 PRO 35|AI 5 33"; A = "gfx1152" } # RDNA 3.5 (Krackan Point: Radeon 860M/840M, Ryzen AI 7 350 / AI 5 340) + @{ P = "RX 7900|PRO W7900|PRO W7800"; A = "gfx1100" } # RDNA 3 desktop / workstation (Navi 31) + @{ P = "RX 7800|RX 7700(?!S)|PRO W7700|PRO V710"; A = "gfx1101" } # RDNA 3 (Navi 32) + @{ P = "RX 7600|RX 7700S|RX 7650|PRO W7600|PRO W7500"; A = "gfx1102" } # RDNA 3 (Navi 33) @{ P = "780M|760M|740M|Phoenix|Hawk Point|Z1 Extreme|Z2 Extreme"; A = "gfx1103" } # RDNA 3 iGPU (Phoenix / Hawk Point) @{ P = "RX 6900|RX 6800|RX 6750|RX 6700|PRO W6800|PRO W6900"; A = "gfx1030" } # RDNA 2 (Navi 21) -- gfx103X family @{ P = "RX 6650|RX 6600|PRO W6600|PRO W6650"; A = "gfx1032" } # RDNA 2 (Navi 23) -- gfx103X family @@ -2773,7 +2775,7 @@ if ((Test-Path -LiteralPath $VenvDir -PathType Container) -and -not $NoTorchMode # for those or a correct CPU venv rebuilds every update. $_rocmWheelArches = @( "gfx1201", "gfx1200", # RDNA 4 - "gfx1151", "gfx1150", # RDNA 3.5 (Strix Halo/Point) + "gfx1151", "gfx1150", "gfx1152", # RDNA 3.5 (Strix Halo/Point, Krackan Point) "gfx1103", "gfx1102", "gfx1101", "gfx1100", # RDNA 3 "gfx1036", "gfx1035", "gfx1034", "gfx1033", "gfx1032", "gfx1031", "gfx1030", # RDNA 2 (RX 6000) "gfx90a", "gfx908" # MI200 / MI100 @@ -3064,6 +3066,7 @@ if (-not $TorchIndexPinned -and ($HasROCm -or $ROCmGfxArch) -and $CuTag -eq "cpu $archFamilyMap = @{ "gfx1201" = "gfx120X-all"; "gfx1200" = "gfx120X-all" # RDNA 4 "gfx1151" = "gfx1151"; "gfx1150" = "gfx1150" # RDNA 3.5 (Strix Halo/Point) + "gfx1152" = "gfx1152" # RDNA 3.5 (Krackan Point) "gfx1103" = "gfx110X-all"; "gfx1102" = "gfx110X-all" # RDNA 3 "gfx1101" = "gfx110X-all"; "gfx1100" = "gfx110X-all" "gfx1036" = "gfx103X-all"; "gfx1035" = "gfx103X-all" # RDNA 2 (RX 6000) @@ -3078,6 +3081,7 @@ if (-not $TorchIndexPinned -and ($HasROCm -or $ROCmGfxArch) -and $CuTag -eq "cpu $torchFloorMap = @{ "gfx1201" = "torch>=2.11.0,<2.12.0"; "gfx1200" = "torch>=2.11.0,<2.12.0" "gfx1151" = "torch>=2.11.0,<2.12.0"; "gfx1150" = "torch>=2.11.0,<2.12.0" + "gfx1152" = "torch>=2.11.0,<2.12.0" } # Companion ranges for torchvision/torchaudio -- must stay in sync with the # torch ceiling so pip can always find a consistent trio on AMD's per-arch @@ -3089,10 +3093,12 @@ if (-not $TorchIndexPinned -and ($HasROCm -or $ROCmGfxArch) -and $CuTag -eq "cpu $torchvisionFloorMap = @{ "gfx1201" = "torchvision>=0.26.0,<0.27.0"; "gfx1200" = "torchvision>=0.26.0,<0.27.0" "gfx1151" = "torchvision>=0.26.0,<0.27.0"; "gfx1150" = "torchvision>=0.26.0,<0.27.0" + "gfx1152" = "torchvision>=0.26.0,<0.27.0" } $torchaudioFloorMap = @{ "gfx1201" = "torchaudio>=2.11.0,<2.12.0"; "gfx1200" = "torchaudio>=2.11.0,<2.12.0" "gfx1151" = "torchaudio>=2.11.0,<2.12.0"; "gfx1150" = "torchaudio>=2.11.0,<2.12.0" + "gfx1152" = "torchaudio>=2.11.0,<2.12.0" } $archFamily = if ($ROCmGfxArch -and $archFamilyMap.ContainsKey($ROCmGfxArch)) { $archFamilyMap[$ROCmGfxArch] } else { $null } $ROCmTorchSpec = if ($ROCmGfxArch -and $torchFloorMap.ContainsKey($ROCmGfxArch)) { $torchFloorMap[$ROCmGfxArch] } else { "torch" } @@ -3104,7 +3110,7 @@ if (-not $TorchIndexPinned -and ($HasROCm -or $ROCmGfxArch) -and $CuTag -eq "cpu # GPU arch detected but not in the supported wheel map — warn explicitly # so the user knows why they are getting CPU PyTorch instead of ROCm. substep "[WARN] AMD GPU ($ROCmGfxArch) not in supported arch list -- falling back to CPU-only PyTorch" "Yellow" - substep " Supported: gfx1200/1201 (RDNA 4), gfx1150/1151 (RDNA 3.5), gfx1100-1103 (RDNA 3), gfx1030-1036 (RDNA 2), gfx90a, gfx908" "Yellow" + substep " Supported: gfx1200/1201 (RDNA 4), gfx1150/1151/1152 (RDNA 3.5), gfx1100-1103 (RDNA 3), gfx1030-1036 (RDNA 2), gfx90a, gfx908" "Yellow" } else { # HIP SDK present ($HasROCm=true via amd-smi) but gcnArchName was not # readable — warn rather than silently falling back to CPU PyTorch. diff --git a/studio/setup.sh b/studio/setup.sh index f6a6bc346b..37d8154e59 100755 --- a/studio/setup.sh +++ b/studio/setup.sh @@ -1167,12 +1167,14 @@ elif [ "$_setup_amd_detected" = true ]; then # gfx1102 matched BEFORE gfx1100 so the spaceless "RX 7700S" lands on # gfx1102 (bash case has no negative lookahead like the PS tables). case "$_setup_mkt" in - *"9070 XT"*|*9080*) _setup_gfx="gfx1201" ;; # RDNA 4 - *9070*|*9060*) _setup_gfx="gfx1200" ;; # RDNA 4 + *9070*|*9080*) _setup_gfx="gfx1201" ;; # RDNA 4 (Navi 48) + *9060*) _setup_gfx="gfx1200" ;; # RDNA 4 (Navi 44) *"8065S"*|*"8060S"*|*"8050S"*|*"8040S"*|*"Strix Halo"*|*"Ryzen AI Max"*|*"AI Max"*) _setup_gfx="gfx1151" ;; # RDNA 3.5 (Strix Halo + Gorgon Halo: Radeon 8065S/8060S/8050S/8040S iGPU, Ryzen AI Max / Max+) - *"890M"*|*"880M"*|*"860M"*|*"840M"*|*"Strix Point"*|*"Krackan"*|*"HX 37"*|*"AI 9 HX"*|*"AI 9 36"*|*"AI 7 35"*|*"AI 5 34"*|*"AI 7 PRO 35"*|*"AI 5 33"*) _setup_gfx="gfx1150" ;; # RDNA 3.5 (Strix/Krackan Point: Radeon 890M/880M iGPU, Ryzen AI 9 HX 370/375) - *"RX 7600"*|*"RX 7700S"*|*"RX 7650"*|*"PRO W7600"*|*"PRO W7500"*|*"PRO V710"*) _setup_gfx="gfx1102" ;; # RDNA 3 (Navi 33) - *"RX 7900"*|*"RX 7800"*|*"RX 7700"*|*"PRO W7900"*|*"PRO W7800"*|*"PRO W7700"*) _setup_gfx="gfx1100" ;; # RDNA 3 desktop / workstation (Navi 31) + *"890M"*|*"880M"*|*"Strix Point"*|*"HX 37"*|*"AI 9 HX"*|*"AI 9 36"*) _setup_gfx="gfx1150" ;; # RDNA 3.5 (Strix Point: Radeon 890M/880M, Ryzen AI 9 HX 370/375) + *"860M"*|*"840M"*|*"Krackan"*|*"AI 7 35"*|*"AI 5 34"*|*"AI 7 PRO 35"*|*"AI 5 33"*) _setup_gfx="gfx1152" ;; # RDNA 3.5 (Krackan Point: Radeon 860M/840M, Ryzen AI 7 350 / AI 5 340) + *"RX 7600"*|*"RX 7700S"*|*"RX 7650"*|*"PRO W7600"*|*"PRO W7500"*) _setup_gfx="gfx1102" ;; # RDNA 3 (Navi 33) + *"RX 7800"*|*"RX 7700"*|*"PRO W7700"*|*"PRO V710"*) _setup_gfx="gfx1101" ;; # RDNA 3 (Navi 32) + *"RX 7900"*|*"PRO W7900"*|*"PRO W7800"*) _setup_gfx="gfx1100" ;; # RDNA 3 desktop / workstation (Navi 31) *"780M"*|*"760M"*|*"740M"*|*"Phoenix"*|*"Hawk Point"*|*"Z1 Extreme"*|*"Z2 Extreme"*) _setup_gfx="gfx1103" ;; # RDNA 3 iGPU (Phoenix / Hawk Point) *"RX 6900"*|*"RX 6800"*|*"RX 6750"*|*"RX 6700"*|*"PRO W6800"*|*"PRO W6900"*) _setup_gfx="gfx1030" ;; # RDNA 2 (Navi 21) *"RX 6650"*|*"RX 6600"*|*"PRO W6600"*|*"PRO W6650"*) _setup_gfx="gfx1032" ;; # RDNA 2 (Navi 23) diff --git a/tests/_zoo_rocm_spoof.py b/tests/_zoo_rocm_spoof.py index 050191e9d1..f5dd090148 100644 --- a/tests/_zoo_rocm_spoof.py +++ b/tests/_zoo_rocm_spoof.py @@ -25,6 +25,7 @@ _PROFILES: dict[str, tuple[str, tuple[int, int], str]] = { "gfx1101": ("AMD Radeon RX 7800 XT", (11, 0), "6.4.43483"), "gfx1102": ("AMD Radeon RX 7600", (11, 0), "7.2.1"), "gfx1150": ("AMD Radeon 890M", (11, 5), "7.2.1"), # RDNA3.5 APU + "gfx1152": ("AMD Radeon 860M", (11, 5), "7.2.1"), "gfx1151": ("AMD Radeon 8060S", (11, 5), "7.2.1"), "gfx1200": ("AMD Radeon RX 9060 XT", (12, 0), "7.2.1"), # RDNA4 "gfx1201": ("AMD Radeon RX 9070 XT", (12, 0), "7.2.1"), @@ -73,7 +74,7 @@ def apply(gfx: str = "gfx1100", device_count: int = 1) -> None: _p.total_memory = 16 * 1024**3 _p.multi_processor_count = 40 _p.warp_size = 32 # RDNA wavefront (CDNA is 64) - _p.is_integrated = gfx in ("gfx1150", "gfx1151") + _p.is_integrated = gfx in ("gfx1150", "gfx1151", "gfx1152") _p.is_multi_gpu_board = False torch.cuda.get_device_properties = lambda *a, **k: _p diff --git a/tests/python/test_cross_platform_parity.py b/tests/python/test_cross_platform_parity.py index b3a9b99c55..b0a5c763d4 100644 --- a/tests/python/test_cross_platform_parity.py +++ b/tests/python/test_cross_platform_parity.py @@ -249,22 +249,42 @@ class TestTorchIndexOverrideParity: class TestGfx211AllowlistParity: - """The gfx per-arch 2.11-floor leaves (gfx120X-all / gfx1151 / gfx1150) must be the - SAME set in every installer and its stale/mismatch check. When they diverged, a - pinned gfx110X-all / gfx90a / gfx908 wheel (<2.11) was force-reinstalled every update.""" + """The gfx per-arch 2.11-floor leaves must be the SAME set in every installer + and its stale/mismatch check. When they diverged, a pinned gfx110X-all / + gfx90a / gfx908 wheel (<2.11) was force-reinstalled every update. - EXPECTED = {"gfx120x-all", "gfx1151", "gfx1150"} + Each test extracts the set each installer actually holds and compares it + against EXPECTED, rather than matching one hardcoded ordering. Order and + spacing are free; membership is not. The earlier literal-string form had to + be edited in four places whenever a leaf was added, which is how adding + gfx1152 (Krackan Point) turned this class red without any installer + actually disagreeing with another.""" + + EXPECTED = {"gfx120x-all", "gfx1151", "gfx1150", "gfx1152"} + + @staticmethod + def _leaves(blob: str) -> set[str]: + """The gfx leaves named in an allowlist literal, quoting-agnostic.""" + return set(re.findall(r"gfx[0-9a-z-]+", blob.lower())) def test_install_sh_allowlist(self): text = INSTALL_SH.read_text(encoding = "utf-8").lower() - # install.sh: the TORCH_CONSTRAINT case (rocm7.2|gfx120x-all|gfx1151|gfx1150). - m = re.search(r"rocm7\.2\|gfx120x-all\|gfx1151\|gfx1150", text) + # install.sh: the TORCH_CONSTRAINT case (rocm7.2|gfx...|gfx...). + m = re.search(r"^\s*(rocm7\.2\|[a-z0-9|.\-]*)\)", text, re.MULTILINE) assert m, "install.sh gfx-2.11 allowlist case not found / changed" + assert self._leaves(m.group(1)) == self.EXPECTED, ( + f"install.sh gfx-2.11 allowlist is {sorted(self._leaves(m.group(1)))}, " + f"expected {sorted(self.EXPECTED)}" + ) def test_install_ps1_allowlist(self): text = INSTALL_PS1.read_text(encoding = "utf-8").lower() - m = re.search(r"@\('gfx120x-all',\s*'gfx1151',\s*'gfx1150'\)", text) + m = re.search(r"\$_pingfx211\s*=\s*@\(([^)]*)\)", text) assert m, "install.ps1 $_pinGfx211 allowlist not found / changed" + assert self._leaves(m.group(1)) == self.EXPECTED, ( + f"install.ps1 $_pinGfx211 is {sorted(self._leaves(m.group(1)))}, " + f"expected {sorted(self.EXPECTED)}" + ) def test_setup_ps1_defines_single_allowlist_helper(self): # setup.ps1 must define the allowlist once (Test-RocmGfx211Leaf) and reuse it, so @@ -273,9 +293,12 @@ class TestGfx211AllowlistParity: assert ( "function Test-RocmGfx211Leaf" in text ), "setup.ps1 should define a single Test-RocmGfx211Leaf allowlist helper" - assert re.search( - r"@\('gfx120x-all',\s*'gfx1151',\s*'gfx1150'\)", text.lower() - ), "Test-RocmGfx211Leaf should hold the gfx-2.11 allowlist" + m = re.search(r"function test-rocmgfx211leaf[\s\S]{0,400}?@\(([^)]*)\)", text.lower()) + assert m, "Test-RocmGfx211Leaf should hold the gfx-2.11 allowlist" + assert self._leaves(m.group(1)) == self.EXPECTED, ( + f"Test-RocmGfx211Leaf holds {sorted(self._leaves(m.group(1)))}, " + f"expected {sorted(self.EXPECTED)}" + ) assert "$_pinGfx211 = Test-RocmGfx211Leaf" in text, ( "setup.ps1 install-spec path should reuse Test-RocmGfx211Leaf, not " "re-hardcode the allowlist (they must not diverge)" @@ -283,9 +306,12 @@ class TestGfx211AllowlistParity: def test_stack_py_allowlist(self): text = STACK_PY.read_text(encoding = "utf-8").lower() - assert ( - '"gfx120x-all", "gfx1151", "gfx1150"' in text - ), "install_python_stack.py _ROCM_GFX_TORCH211_LEAVES not found / changed" + m = re.search(r"_rocm_gfx_torch211_leaves[^=]*=\s*frozenset\(\s*\{([^}]*)\}", text) + assert m, "install_python_stack.py _ROCM_GFX_TORCH211_LEAVES not found / changed" + assert self._leaves(m.group(1)) == self.EXPECTED, ( + f"_ROCM_GFX_TORCH211_LEAVES is {sorted(self._leaves(m.group(1)))}, " + f"expected {sorted(self.EXPECTED)}" + ) class TestCudaLeafDigitParity: @@ -351,15 +377,21 @@ class TestCudaLeafDigitParity: class TestKnown211SetParity: """The KNOWN-2.11 rocm/gfx set must be identical across all four installers: - exactly {rocm7.2} plus the gfx allowlist {gfx120x-all, gfx1151, gfx1150}. + exactly {rocm7.2} plus TestGfx211AllowlistParity.EXPECTED. rocm7.3 / torch 2.12 do not exist, so no side may floor them speculatively.""" def test_install_sh_known_211_leaf_is_rocm72_and_gfx_allowlist(self): text = INSTALL_SH.read_text(encoding = "utf-8") - # The 2.11 floor case matches exactly rocm7.2 + the three gfx leaves. - assert re.search( - r"rocm7\.2\|gfx120x-all\|gfx1151\|gfx1150\)", text - ), "install.sh 2.11 floor must be exactly rocm7.2|gfx120x-all|gfx1151|gfx1150" + # The 2.11 floor case matches exactly rocm7.2 + the gfx allowlist, in + # any order: it is the same set as TestGfx211AllowlistParity.EXPECTED, + # asserted here so the rocm-version half cannot drift on its own. + m = re.search(r"^\s*(rocm7\.2\|[a-zA-Z0-9|.\-]*)\)", text, re.MULTILINE) + assert m, "install.sh 2.11 floor case (rocm7.2|gfx...) not found / changed" + alternatives = set(m.group(1).lower().split("|")) + assert alternatives == {"rocm7.2"} | TestGfx211AllowlistParity.EXPECTED, ( + f"install.sh 2.11 floor is {sorted(alternatives)}, expected " + f"{sorted({'rocm7.2'} | TestGfx211AllowlistParity.EXPECTED)}" + ) # No speculative rocm7.3 anywhere. assert "rocm7.3" not in text, "install.sh must not reference a non-existent rocm7.3" diff --git a/tests/run_all.sh b/tests/run_all.sh index 6eccffc75f..2497ba8fe7 100755 --- a/tests/run_all.sh +++ b/tests/run_all.sh @@ -7,18 +7,27 @@ set -e TESTS_DIR="$(cd "$(dirname "$0")" && pwd)" echo "=== Bash tests ===" -sh "$TESTS_DIR/sh/test_get_torch_index_url.sh" -sh "$TESTS_DIR/sh/test_mac_intel_compat.sh" -sh "$TESTS_DIR/sh/test_torch_constraint.sh" -sh "$TESTS_DIR/sh/test_nvcc_meets_llama_minimum.sh" -sh "$TESTS_DIR/sh/test_resolve_cuda_archs.sh" -sh "$TESTS_DIR/sh/test_staged_validation_enabled.sh" -sh "$TESTS_DIR/sh/test_strixhalo_wsl_reroute.sh" -sh "$TESTS_DIR/sh/test_uninstall_shared_icon.sh" -sh "$TESTS_DIR/sh/test_torch_flavor.sh" -sh "$TESTS_DIR/sh/test_redact_install_output.sh" -sh "$TESTS_DIR/sh/test_install_uv_override_space.sh" -sh "$TESTS_DIR/sh/test_install_rollback_lifecycle.sh" +# Discovered, not listed: a hand-maintained list drifts (this one had fallen +# eight files behind sh/, and the Backend CI copy of it had fallen seven). +# Backend CI discovers the same directory and skips the same file, plus +# test_install_rollback_lifecycle.sh which cross-platform-parity-ci.yml already +# runs on both platforms. tests/studio/test_ci_shell_suite_coverage.py fails if +# either side stops discovering, or skips something undocumented. +# test_install_host_defaults.sh: asserts an install.ps1 layout that has +# drifted (separate followup). +SH_SKIP="test_install_host_defaults.sh" +for _t in "$TESTS_DIR"/sh/test_*.sh; do + case " $SH_SKIP " in + *" $(basename "$_t") "*) echo "skipping $(basename "$_t")"; continue ;; + esac + # bash, not sh: every file under sh/ declares a bash shebang, and three of + # them fail on bashisms under dash, which is /bin/sh on Debian and Ubuntu + # (test_apt_distro_prompt, test_studio_home_node_dir, and + # test_with_llama_cpp_dir_link_behavior). The old hand-written list happened + # to name only dash-clean files, so discovering the directory is what + # exposed this. Backend CI already invokes them with bash. + bash "$_t" +done echo "" echo "=== Python tests ===" diff --git a/tests/studio/install/test_rocm_arch_table_parity.py b/tests/studio/install/test_rocm_arch_table_parity.py new file mode 100644 index 0000000000..40f6858f7e --- /dev/null +++ b/tests/studio/install/test_rocm_arch_table_parity.py @@ -0,0 +1,651 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Drift guards for the AMD gfx tables that are duplicated across the installers. + +The same three tables are hand-copied into up to seven places each: + + gfx -> AMD index family install.sh (_amd_arch_index_family_for_gfx) + install.ps1 ($archFamilyMap) + studio/setup.ps1 ($archFamilyMap) + studio/install_python_stack.py (_GFX_TO_AMD_INDEX_ARCH) + + GPU name -> gfx install.sh (_infer_amd_gfx_arch_from_gpu_name) + install.sh (case "$_gpu_disp_mkt", detection banner + env tip) + studio/setup.sh (case "$_setup_mkt") + install.ps1 ($nameArchTable) + studio/setup.ps1 ($nameArchTable) + studio/install_python_stack.py (_WIN_GPU_NAME_ARCH_TABLE) + tests/_zoo_rocm_spoof.py (_PROFILES, inverted gfx -> name) + + torch>=2.11 pin allowlist install.sh (case "$_torch_index_leaf") + install.ps1 ($_pinGfx211) + studio/setup.ps1 (Test-RocmPinLeaf211) + +Every copy carries a "kept in sync with" comment and nothing enforced it, which is +how the routing family of bugs kept recurring: #7264 / #7280 (Strix left on the +generic rocm7.2 index), #7293 / #7300 (fixed in one installer at a time) and #7277 +(RDNA2 gfx1030-1036 added to install.ps1 / setup.ps1 / install_python_stack.py -- +install.sh had to follow separately). Half-applied edits are invisible until an AMD +user on the missed path gets CPU-only PyTorch. + +These tests parse each copy out of its source file and compare them, so a table +edited in one place fails CI naming the file that was missed. + +Counting the copies by hand is itself unreliable -- the in-code "kept in sync +with" comments claimed four when there were seven -- so TestNoUnregisteredArchTable +below rediscovers them by scanning the repo instead of trusting this list. +""" + +import ast +import fnmatch +import importlib.util +import re +import sys +from pathlib import Path + +import pytest + + +PACKAGE_ROOT = Path(__file__).resolve().parents[3] + +_INSTALL_SH = PACKAGE_ROOT / "install.sh" +_INSTALL_PS1 = PACKAGE_ROOT / "install.ps1" +_SETUP_SH = PACKAGE_ROOT / "studio" / "setup.sh" +_SETUP_PS1 = PACKAGE_ROOT / "studio" / "setup.ps1" +_STACK_PY = PACKAGE_ROOT / "studio" / "install_python_stack.py" +_SPOOF_PY = PACKAGE_ROOT / "tests" / "_zoo_rocm_spoof.py" + + +def _load_stack_module(): + spec = importlib.util.spec_from_file_location("studio_install_python_stack_parity", _STACK_PY) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = mod + spec.loader.exec_module(mod) + return mod + + +stack_mod = _load_stack_module() + + +# ── Source extraction helpers ──────────────────────────────────────────────── + + +def _sh_function_body(source: str, name: str) -> str: + """Return a POSIX-shell function body by brace matching (same idea as + _extract_sh_function_body in test_rocm_support.py, kept local so this file + stands alone).""" + needle = f"{name}() {{" + start = source.find(needle) + assert start != -1, f"{name}() not found" + depth = 0 + i = start + len(needle) - 1 + while i < len(source): + if source[i] == "{": + depth += 1 + elif source[i] == "}": + depth -= 1 + if depth == 0: + return source[start : i + 1] + i += 1 + raise AssertionError(f"unterminated {name}()") + + +def _sh_case_block(source: str, subject: str) -> str: + """Return the body of `case in ... esac` (first match).""" + start = source.find(f"case {subject} in") + assert start != -1, f"case {subject} in ... not found" + end = source.find("esac", start) + assert end != -1, f"unterminated case {subject}" + return source[start:end] + + +def _ps_block(source: str, header: str, open_ch: str, close_ch: str) -> str: + """Return the balanced `header ... ` block from a PowerShell file.""" + start = source.find(header) + assert start != -1, f"{header} not found" + i = source.find(open_ch, start) + assert i != -1 + depth = 0 + while i < len(source): + if source[i] == open_ch: + depth += 1 + elif source[i] == close_ch: + depth -= 1 + if depth == 0: + return source[start : i + 1] + i += 1 + raise AssertionError(f"unterminated {header}") + + +def _strip_sh_comment(line: str) -> str: + """Drop a trailing `# ...` comment. Safe here: no table line contains a '#' + inside a pattern.""" + return line.split("#", 1)[0] + + +# ── Table 1: gfx -> AMD index family ───────────────────────────────────────── + + +def _gfx_family_map_sh() -> dict[str, str]: + body = _sh_function_body( + _INSTALL_SH.read_text(encoding = "utf-8"), "_amd_arch_index_family_for_gfx" + ) + out: dict[str, str] = {} + for line in body.splitlines(): + m = re.match(r"\s*(gfx[^)]*)\)\s*echo\s+(\S+)\s*;;", _strip_sh_comment(line)) + if not m: + continue + for arch in m.group(1).split("|"): + out[arch.strip()] = m.group(2).strip() + return out + + +def _gfx_family_map_ps(path: Path) -> dict[str, str]: + block = _ps_block(path.read_text(encoding = "utf-8"), "$archFamilyMap = @{", "{", "}") + out: dict[str, str] = {} + for line in block.splitlines(): + for m in re.finditer(r'"(gfx[0-9a-z]+)"\s*=\s*"([A-Za-z0-9-]+)"', _strip_sh_comment(line)): + out[m.group(1)] = m.group(2) + return out + + +def _gfx_family_maps() -> dict[str, dict[str, str]]: + return { + "studio/install_python_stack.py": dict(stack_mod._GFX_TO_AMD_INDEX_ARCH), + "install.sh": _gfx_family_map_sh(), + "install.ps1": _gfx_family_map_ps(_INSTALL_PS1), + "studio/setup.ps1": _gfx_family_map_ps(_SETUP_PS1), + } + + +class TestGfxIndexFamilyParity: + """All four gfx -> AMD index family maps must agree, entry for entry.""" + + def test_every_copy_is_non_empty(self): + for where, table in _gfx_family_maps().items(): + assert ( + table + ), f"{where}: parsed an empty gfx -> index family map (table moved or renamed?)" + + def test_all_copies_identical(self): + maps = _gfx_family_maps() + reference_name = "studio/install_python_stack.py" + reference = maps[reference_name] + for where, table in maps.items(): + if where == reference_name: + continue + missing = {k: v for k, v in reference.items() if k not in table} + extra = {k: v for k, v in table.items() if k not in reference} + wrong = { + k: (v, reference[k]) + for k, v in table.items() + if k in reference and v != reference[k] + } + assert ( + not missing + ), f"{where} is missing {sorted(missing)} (present in {reference_name})" + assert not extra, f"{where} has {sorted(extra)} that {reference_name} does not" + assert not wrong, f"{where} maps {wrong} (value, expected)" + + def test_rdna2_family_present_everywhere(self): + """#7277 added gfx1030-1036 to three files; install.sh followed later. + Pin the whole RDNA2 range so the next family lands everywhere at once.""" + for where, table in _gfx_family_maps().items(): + for arch in ( + "gfx1030", + "gfx1031", + "gfx1032", + "gfx1033", + "gfx1034", + "gfx1035", + "gfx1036", + ): + assert table.get(arch) == "gfx103X-all", f"{where}: {arch} -> {table.get(arch)!r}" + + +class TestSupportedWheelArchList: + """setup.ps1's $_rocmWheelArches decides whether a detected arch gets ROCm torch + at all. An arch present in the family map but absent here silently installs + CPU-only PyTorch (the 'not in supported arch list' report from r/unsloth).""" + + def test_wheel_arch_list_covers_every_mapped_arch(self): + block = _ps_block( + _SETUP_PS1.read_text(encoding = "utf-8"), "$_rocmWheelArches = @(", "(", ")" + ) + listed = set(re.findall(r'"(gfx[0-9a-z]+)"', block)) + assert listed, "could not parse $_rocmWheelArches" + mapped = set(stack_mod._GFX_TO_AMD_INDEX_ARCH) + assert mapped - listed == set(), ( + f"studio/setup.ps1 $_rocmWheelArches is missing {sorted(mapped - listed)}: " + "those arches map to an AMD index but would still fall back to CPU torch" + ) + + +# ── Table 2: GPU marketing name -> gfx ─────────────────────────────────────── +# +# Each copy is an ordered, first-match-wins table. The shell copies use case +# globs (case-sensitive); the PowerShell copies use -match regexes +# (case-insensitive, and the only place a negative lookahead is available). +# Rather than diff the patterns -- which legitimately differ in syntax -- run +# every copy against the same real GPU names and require the same answer. + + +def _name_table_sh_function(source: str, name: str) -> list[tuple[list[str], str]]: + body = _sh_function_body(source, name) + rows: list[tuple[list[str], str]] = [] + for line in body.splitlines(): + m = re.match(r"\s*(\*.*?)\)\s*echo\s+(gfx[0-9a-z]+)\s*;;", _strip_sh_comment(line)) + if m: + rows.append(([p.strip() for p in m.group(1).split("|")], m.group(2))) + return rows + + +def _name_table_sh_case(source: str, subject: str, var: str) -> list[tuple[list[str], str]]: + """A bare `case ... in` table that assigns to a variable rather than echoing.""" + block = _sh_case_block(source, subject) + rows: list[tuple[list[str], str]] = [] + for line in block.splitlines(): + m = re.match( + rf'\s*(\*.*?)\)\s*{re.escape(var)}="(gfx[0-9a-z]+)"\s*;;', _strip_sh_comment(line) + ) + if m: + rows.append(([p.strip() for p in m.group(1).split("|")], m.group(2))) + return rows + + +def _name_table_ps(path: Path) -> list[tuple[str, str]]: + block = _ps_block(path.read_text(encoding = "utf-8"), "$nameArchTable = @(", "(", ")") + return re.findall(r'@\{\s*P\s*=\s*"([^"]+)"\s*;\s*A\s*=\s*"(gfx[0-9a-z]+)"\s*\}', block) + + +def _match_sh(rows: list[tuple[list[str], str]], gpu_name: str) -> str | None: + """Evaluate a shell `case` table: first arm whose glob matches wins.""" + for patterns, arch in rows: + for pattern in patterns: + # Shell case globs quote literal segments: *"RX 7900"* -> *RX 7900* + if fnmatch.fnmatchcase(gpu_name, pattern.replace('"', "")): + return arch + return None + + +def _match_ps(rows: list[tuple[str, str]], gpu_name: str) -> str | None: + """Evaluate a PowerShell -match table: first arm whose regex matches wins. + -match is case-insensitive; .NET and Python agree on these patterns + (alternation plus one negative lookahead).""" + for pattern, arch in rows: + if re.search(pattern, gpu_name, re.IGNORECASE): + return arch + return None + + +# Real strings as amd-smi / rocm-smi / WMI report them, including the two +# ordering traps: "RX 9070 XT" must beat the bare "9070" arm, and "RX 7700S" +# must beat the "RX 7700" arm. +# +# The expectation is the *AMD pip index leaf*, not the gfx id. The leaf is what +# the tables exist to produce -- it picks the wheel -- and it is what a wrong +# answer actually costs the user. Exact gfx ids are pinned separately in +# _AMD_DOCUMENTED_ARCH, sourced from AMD rather than from these tables. +_GPU_NAME_LEAF_CASES = [ + ("AMD Radeon RX 9070 XT", "gfx120X-all"), + ("AMD Radeon RX 9070", "gfx120X-all"), + ("AMD Radeon RX 9060 XT", "gfx120X-all"), + ("AMD Radeon 8060S Graphics", "gfx1151"), + ("AMD Ryzen AI Max+ 395 w/ Radeon 8060S Graphics", "gfx1151"), + ("AMD Radeon 890M Graphics", "gfx1150"), + ("AMD Radeon 880M Graphics", "gfx1150"), + ("AMD Radeon 860M Graphics", "gfx1152"), + ("AMD Radeon 840M Graphics", "gfx1152"), + ("AMD Ryzen AI 7 350 w/ Radeon 860M", "gfx1152"), + ("AMD Radeon RX 7900 XTX", "gfx110X-all"), + ("AMD Radeon RX 7800 XT", "gfx110X-all"), + ("AMD Radeon PRO W7900", "gfx110X-all"), + ("AMD Radeon RX 7700S", "gfx110X-all"), + ("AMD Radeon RX 7600 XT", "gfx110X-all"), + ("AMD Radeon 780M Graphics", "gfx110X-all"), + ("AMD Radeon RX 6900 XT", "gfx103X-all"), + ("AMD Radeon RX 6700 XT", "gfx103X-all"), + ("AMD Radeon RX 6600 XT", "gfx103X-all"), + ("AMD Radeon RX 6500 XT", "gfx103X-all"), +] + +# Exact gfx ids, transcribed from AMD's ROCm compatibility matrix (the "Radeon +# GPU" list at rocm.docs.amd.com/en/latest/compatibility/compatibility-matrix.html), +# NOT from the installer tables. This is the ground truth the tables are supposed +# to reproduce, so it has to come from outside them. +# +# Three of these were wrong until the commit that added this table: RX 9070 +# (non-XT) said gfx1200, RX 7800 XT / 7700 XT / PRO W7700 said gfx1100, and PRO +# V710 said gfx1102. Nobody was misrouted, because each wrong id happened to +# share an index leaf with the right one, which is exactly why it went unnoticed +# through five copies of the table. The leaf assertions above cannot catch that +# class of error; only an external source can. +# +# The APU rows were added after that: Krackan Point (860M / 840M) said gfx1150 +# but is gfx1152, and unlike the three above that one DID change the wheel, +# since gfx1150 and gfx1152 are separate index leaves on repo.amd.com. +_AMD_DOCUMENTED_ARCH = { + # RDNA 4 -- Navi 48 is gfx1201, Navi 44 is gfx1200. + "AMD Radeon RX 9070 XT": "gfx1201", + "AMD Radeon RX 9070 GRE": "gfx1201", + "AMD Radeon RX 9070": "gfx1201", + "AMD Radeon RX 9060 XT": "gfx1200", + "AMD Radeon RX 9060": "gfx1200", + # RDNA 3 -- Navi 31 / 32 / 33. + "AMD Radeon RX 7900 XTX": "gfx1100", + "AMD Radeon PRO W7900": "gfx1100", + "AMD Radeon PRO W7800": "gfx1100", + "AMD Radeon RX 7800 XT": "gfx1101", + "AMD Radeon RX 7700 XT": "gfx1101", + "AMD Radeon PRO W7700": "gfx1101", + "AMD Radeon PRO V710": "gfx1101", + "AMD Radeon RX 7600 XT": "gfx1102", + "AMD Radeon RX 7700S": "gfx1102", + "AMD Radeon PRO W7600": "gfx1102", + # RDNA 3.5 APUs -- Strix Point is gfx1150, Krackan Point (860M/840M) is + # gfx1152, per AMD's own lemonade GPU table (src/cpp/server/system_info.cpp). + "AMD Radeon 8060S Graphics": "gfx1151", + "AMD Radeon 890M Graphics": "gfx1150", + "AMD Radeon 880M Graphics": "gfx1150", + "AMD Radeon 860M Graphics": "gfx1152", + "AMD Radeon 840M Graphics": "gfx1152", +} + + +def _name_tables() -> dict[str, object]: + install_sh = _INSTALL_SH.read_text(encoding = "utf-8") + return { + "install.sh:_infer_amd_gfx_arch_from_gpu_name": _name_table_sh_function( + install_sh, "_infer_amd_gfx_arch_from_gpu_name" + ), + # install.sh carries the table TWICE. The second copy drives the detection + # banner and, more importantly, the "Tip: set UNSLOTH_ROCM_GFX_ARCH=" + # line, so a wrong id there gets pasted into a user's environment where it + # becomes authoritative. Neither this copy nor the two below were in this + # parity check until the arch-id fix went looking for every place the + # table lives -- six, not four. + "install.sh:_gpu_disp_gfx": _name_table_sh_case( + install_sh, '"$_gpu_disp_mkt"', "_gpu_disp_gfx" + ), + "studio/setup.sh": _name_table_sh_case( + _SETUP_SH.read_text(encoding = "utf-8"), '"$_setup_mkt"', "_setup_gfx" + ), + "install.ps1": _name_table_ps(_INSTALL_PS1), + "studio/setup.ps1": _name_table_ps(_SETUP_PS1), + "studio/install_python_stack.py": list(stack_mod._WIN_GPU_NAME_ARCH_TABLE), + } + + +def _spoof_profiles() -> dict[str, str]: + """gfx -> marketing name out of tests/_zoo_rocm_spoof.py::_PROFILES. + + Parsed with ast rather than imported: that module spoofs torch.cuda and the + AMD identity as an import side effect, which would poison every test sharing + the process.""" + tree = ast.parse(_SPOOF_PY.read_text(encoding = "utf-8")) + for node in tree.body: + target = node.target if isinstance(node, ast.AnnAssign) else None + if target is not None and getattr(target, "id", "") == "_PROFILES": + return {gfx: value[0] for gfx, value in ast.literal_eval(node.value).items()} + raise AssertionError("_PROFILES not found in tests/_zoo_rocm_spoof.py") + + +# The spoof fixture states the mapping backwards (gfx -> the name torch should +# report), so it is the one copy written from the hardware's point of view +# instead of the installer's. That makes it a useful independent witness: it had +# gfx1101 -> "RX 7800 XT" and gfx1201 -> "RX 9070 XT" correct while all six +# installer copies were wrong, and nothing compared the two. +# +# RX 6700 XT is a known, deliberate divergence rather than drift. AMD's +# compatibility matrix documents no consumer RX 6000 card and no gfx1031 at all +# (only "AMD Radeon PRO W6800 (gfx1030)"), the installer arm is commented +# "gfx103X family", and no code consumes the exact id -- gfx1031 appears only as +# a key in the index-family maps, never as a value any name table emits. With no +# external source to correct it against, changing shipped behaviour here would be +# guesswork, so the divergence is pinned instead of silently normalised. +_SPOOF_DIVERGENCES = { + "gfx1031": "installers group Navi 22 into the gfx1030 arm; see comment above", +} + + +def _resolve(where: str, rows, gpu_name: str) -> str | None: + """Shell copies are case globs; the PowerShell and Python copies are both + ordered first-match regex tables evaluated case-insensitively, so _match_ps + models either one. `where` may be ":" for the files that carry + the table more than once.""" + return ( + _match_sh(rows, gpu_name) + if where.split(":")[0].endswith(".sh") + else _match_ps(rows, gpu_name) + ) + + +class TestGpuNameArchParity: + """All four name -> gfx tables must resolve the same GPU the same way.""" + + def test_every_copy_is_non_empty(self): + for where, rows in _name_tables().items(): + assert rows, f"{where}: parsed an empty name -> gfx table (table moved or renamed?)" + + @pytest.mark.parametrize("gpu_name", [name for name, _ in _GPU_NAME_LEAF_CASES]) + def test_all_copies_return_the_same_arch(self, gpu_name): + """The drift guard proper: no expected value, just agreement. This is what + catches a table edited in one installer and not the other three, and it + stays honest even where the shipped gfx id is itself wrong.""" + answers = {where: _resolve(where, rows, gpu_name) for where, rows in _name_tables().items()} + distinct = set(answers.values()) + assert len(distinct) == 1, f"{gpu_name!r} resolves inconsistently: {answers}" + assert distinct != {None}, f"{gpu_name!r} is not matched by any copy of the table" + + @pytest.mark.parametrize("gpu_name,expected_leaf", _GPU_NAME_LEAF_CASES) + def test_every_copy_routes_to_the_right_wheel_index(self, gpu_name, expected_leaf): + """What the tables are for. A wrong leaf is the user-visible failure: + CPU-only torch, or a wheel built for the wrong ISA.""" + families = stack_mod._GFX_TO_AMD_INDEX_ARCH + for where, rows in _name_tables().items(): + arch = _resolve(where, rows, gpu_name) + assert arch is not None, f"{where}: {gpu_name!r} matched nothing" + assert ( + families.get(arch) == expected_leaf + ), f"{where}: {gpu_name!r} -> {arch} -> {families.get(arch)!r}, expected {expected_leaf!r}" + + @pytest.mark.parametrize("gpu_name,expected_arch", sorted(_AMD_DOCUMENTED_ARCH.items())) + def test_every_copy_matches_amds_documented_arch(self, gpu_name, expected_arch): + """The gfx id itself, against AMD's matrix rather than against a sibling + copy of the same table. Agreement between five copies proves nothing if + all five were transcribed from the same mistake.""" + for where, rows in _name_tables().items(): + arch = _resolve(where, rows, gpu_name) + assert ( + arch == expected_arch + ), f"{where}: {gpu_name!r} -> {arch!r}, AMD documents {expected_arch!r}" + + def test_unknown_name_matches_nothing_anywhere(self): + """An unrecognised card must fall through to the CPU path in every copy, + never onto a neighbouring arm.""" + for where, rows in _name_tables().items(): + got = _resolve(where, rows, "NVIDIA GeForce RTX 4090") + assert got is None, f"{where}: RTX 4090 matched {got!r}" + + def test_inferred_arch_always_has_an_index_family(self): + """Every arch a name table can produce must be routable to an AMD wheel + index, else detection succeeds and the install still lands on CPU torch.""" + families = stack_mod._GFX_TO_AMD_INDEX_ARCH + for where, rows in _name_tables().items(): + for arch in {arch for _, arch in rows}: + assert arch in families, f"{where}: {arch} has no entry in _GFX_TO_AMD_INDEX_ARCH" + + def test_every_documented_gpu_resolves_somewhere(self): + """The reverse of the AMD check above. That one asks "do the tables get + the documented cards right"; this asks "is a documented card missing + entirely", which is a silent CPU fallback rather than a wrong id. + + This cannot notice a GPU AMD shipped that nobody transcribed into + _AMD_DOCUMENTED_ARCH -- doing that honestly would mean fetching AMD's + matrix at test time, which makes the suite non-hermetic and offline + runners fail. It does catch a card added to the ground-truth list, or to + one installer, without the tables being completed.""" + for gpu_name in sorted(_AMD_DOCUMENTED_ARCH): + for where, rows in _name_tables().items(): + assert ( + _resolve(where, rows, gpu_name) is not None + ), f"{where}: {gpu_name!r} matches no arm, so this card gets CPU-only torch" + + +class TestSpoofFixtureParity: + """tests/_zoo_rocm_spoof.py is the seventh copy of the name/gfx mapping and + was outside every drift guard. It is the fixture other ROCm tests build their + fake AMD host from, so if it and the installers disagree, those tests exercise + a machine that cannot exist.""" + + def test_spoof_profiles_parse(self): + profiles = _spoof_profiles() + assert profiles, "parsed an empty _PROFILES (renamed or restructured?)" + assert all(gfx.startswith("gfx") for gfx in profiles), profiles + + def test_spoof_names_resolve_back_to_their_own_arch(self): + """Round-trip: feed each spoofed marketing name through the installer + tables and the answer must be the gfx the spoof claims to be emulating.""" + tables = _name_tables() + for gfx, gpu_name in sorted(_spoof_profiles().items()): + if gfx in _SPOOF_DIVERGENCES: + continue + for where, rows in tables.items(): + got = _resolve(where, rows, gpu_name) + assert ( + got == gfx + ), f"{where}: spoof says {gfx} is {gpu_name!r}, installer says {got!r}" + + def test_divergences_are_real_and_still_diverging(self): + """Keeps the exception list from going stale: if the installers are + corrected later, this fails and the entry has to be removed rather than + quietly suppressing a check that now passes.""" + tables = _name_tables() + profiles = _spoof_profiles() + for gfx in _SPOOF_DIVERGENCES: + assert gfx in profiles, f"{gfx} is exempted but no longer in the spoof" + answers = {_resolve(w, r, profiles[gfx]) for w, r in tables.items()} + assert answers != {gfx}, f"{gfx} now agrees everywhere; drop it from _SPOOF_DIVERGENCES" + + +# ── The meta-guard: find copies nobody registered ──────────────────────────── + + +# A table line names a card and gives its arch. Matching both on one line is what +# separates a real table from the many files that merely mention a gfx id (kernel +# dispatch, OOM guards, doc comments). +_MKT_NAME = re.compile(r"(RX\s*\d{4}|PRO\s*[WV]\d{3,4}|\b90[5-8]0\b)", re.IGNORECASE) +_GFX_ID = re.compile(r"gfx1[0-2][0-9a-z]{1,2}") + +# Skip dirs of third-party or generated code; scanning them is slow and any hit +# would not be ours to fix. +_SCAN_SKIP_DIRS = {".git", "node_modules", ".venv", "venv", "build", "dist", "__pycache__"} + +# Every file allowed to carry a name/arch table, as a repo-relative posix path. +# Adding a copy means adding it here AND wiring it into a parity check above; +# that is the point of the guard. +_REGISTERED_TABLE_FILES = { + "install.sh", + "install.ps1", + "studio/setup.sh", + "studio/setup.ps1", + "studio/install_python_stack.py", + "tests/_zoo_rocm_spoof.py", +} + +# Three or more such lines means a table. One or two means prose: the two known +# single-line hits are comments ("Verified on gfx1151 (Radeon 8060S)" in +# scripts/install_rocm_wsl_strixhalo.sh, and a parenthetical in +# studio/install_llama_prebuilt.py). Real tables score 9 to 17, so the gap is +# wide and the threshold is not load-bearing. +_TABLE_LINE_THRESHOLD = 3 + + +def _files_carrying_a_name_arch_table() -> dict[str, int]: + found: dict[str, int] = {} + for path in PACKAGE_ROOT.rglob("*"): + if path.suffix not in {".sh", ".ps1", ".py"} or not path.is_file(): + continue + rel = path.relative_to(PACKAGE_ROOT).as_posix() + if any(part in _SCAN_SKIP_DIRS for part in path.relative_to(PACKAGE_ROOT).parts): + continue + # Tests that *assert* on the tables quote card names next to gfx ids by + # nature. Fixtures like _zoo_rocm_spoof.py do not start with test_ and so + # stay in scope, which is how the seventh copy surfaced. + if path.name.startswith("test_"): + continue + try: + text = path.read_text(encoding = "utf-8", errors = "ignore") + except OSError: + continue + hits = sum( + 1 for line in text.splitlines() if _MKT_NAME.search(line) and _GFX_ID.search(line) + ) + if hits >= _TABLE_LINE_THRESHOLD: + found[rel] = hits + return found + + +class TestNoUnregisteredArchTable: + """The failure this whole file exists for is a copy of the table that nobody + knew about. Enumerating the copies by hand is the same manual step that let + them drift, so this rediscovers them from the source tree.""" + + def test_scan_still_finds_the_known_copies(self): + """Guards the guard: if the heuristic stops matching (patterns reformatted + onto multiple lines, say), it would silently find nothing and pass.""" + found = _files_carrying_a_name_arch_table() + missing = _REGISTERED_TABLE_FILES - set(found) + assert not missing, f"scan no longer detects known tables in {sorted(missing)}" + + def test_no_unregistered_copies(self): + found = _files_carrying_a_name_arch_table() + extra = {rel: n for rel, n in found.items() if rel not in _REGISTERED_TABLE_FILES} + assert not extra, ( + f"unregistered GPU-name/arch table(s): {extra}. Wire each into " + f"_name_tables() (or the spoof check) and add it to " + f"_REGISTERED_TABLE_FILES, so drift there fails CI too." + ) + + +# ── Table 3: the torch>=2.11 pin allowlist ─────────────────────────────────── + + +class TestTorch211PinAllowlistParity: + """gfx120X-all / gfx1151 / gfx1150 / gfx1152 (and rocm7.2) ship the null + _grouped_mm kernel below torch 2.11, so all three installers must raise the + same floor. A leaf missing from one copy reintroduces the crash there.""" + + _EXPECTED = {"gfx120x-all", "gfx1151", "gfx1150", "gfx1152"} + + def test_install_sh_pins_the_same_leaves(self): + source = _INSTALL_SH.read_text(encoding = "utf-8") + idx = source.find('case "$_torch_index_leaf" in') + assert idx != -1 + arm = re.search(r"\n\s*(rocm7\.2\|[^)]*)\)", source[idx:]) + assert arm, "torch 2.11 pin arm not found in install.sh" + leaves = {leaf.strip() for leaf in arm.group(1).split("|")} + assert ( + self._EXPECTED <= leaves + ), f"install.sh pin arm missing {sorted(self._EXPECTED - leaves)}" + assert "rocm7.2" in leaves + + def test_install_ps1_pins_the_same_leaves(self): + source = _INSTALL_PS1.read_text(encoding = "utf-8") + m = re.search(r"\$_pinGfx211\s*=\s*@\(([^)]*)\)", source) + assert m, "$_pinGfx211 not found in install.ps1" + leaves = set(re.findall(r"'([^']+)'", m.group(1))) + assert leaves == self._EXPECTED, f"install.ps1 pins {sorted(leaves)}" + + def test_setup_ps1_pins_the_same_leaves(self): + source = _SETUP_PS1.read_text(encoding = "utf-8") + m = re.search(r"return\s+@\(([^)]*)\)\s*-contains\s*\$Leaf", source) + assert m, "the 2.11 pin allowlist helper was not found in studio/setup.ps1" + leaves = set(re.findall(r"'([^']+)'", m.group(1))) + assert leaves == self._EXPECTED, f"studio/setup.ps1 pins {sorted(leaves)}" + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/studio/install/test_rocm_native_linux_lib_dirs.py b/tests/studio/install/test_rocm_native_linux_lib_dirs.py new file mode 100644 index 0000000000..9b95af88b0 --- /dev/null +++ b/tests/studio/install/test_rocm_native_linux_lib_dirs.py @@ -0,0 +1,438 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Coverage for the native-Linux system-ROCm library prepend (PR #7233). + +#7233 fixed the segfault-on-launch class of AMD reports (#7208, #7310, #6276 and +the native-Linux half of #7307): a prebuilt llama.cpp ships its own libggml-hip / +HIP runtime, and on a bare-metal ROCm box that bundled runtime can disagree with +the host amdkfd driver, so the server dies the moment a model is loaded. The fix +prepends the *system* ROCm lib dirs ahead of the bundle on LD_LIBRARY_PATH. + +It landed as two hand-copied helpers, one in the installer (validation-time) and +one in the serve-time launcher: + + studio/install_llama_prebuilt.py _bundled_hip_present / _native_linux_system_rocm_lib_dirs + studio/backend/core/inference/llama_cpp.py same two, "mirrors" comment only + +and shipped with no tests at all: the WSL sibling helper added earlier has +TestWslSystemRocmLibDirs / TestBinaryEnvWslOrdering / TestLlamaCppRuntimeWslOrdering, +the native-Linux one has nothing. Every gate here is a false-positive risk that +would silently reorder LD_LIBRARY_PATH for users the fix was never meant to touch +(WSL, NVIDIA hosts, macOS, containers without /dev/kfd), so each gate gets a test, +and both copies are run against the same fake host and required to agree. + +llama_cpp.py cannot be imported from the test suite (module-level structlog / +backend imports), so its two helpers are lifted out with ast and exec'd standalone. +""" + +import ast +import importlib.util +import os +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest + + +PACKAGE_ROOT = Path(__file__).resolve().parents[3] + +_PREBUILT_PATH = PACKAGE_ROOT / "studio" / "install_llama_prebuilt.py" +_LLAMA_CPP_PATH = PACKAGE_ROOT / "studio" / "backend" / "core" / "inference" / "llama_cpp.py" + +_HELPERS = ("_bundled_hip_present", "_native_linux_system_rocm_lib_dirs") + + +def _load_prebuilt_module(): + spec = importlib.util.spec_from_file_location( + "studio_install_llama_prebuilt_native", _PREBUILT_PATH + ) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = mod + spec.loader.exec_module(mod) + return mod + + +def _extract_functions(path: Path, names) -> dict: + """exec just the named top-level functions out of a module that is too + heavy to import.""" + tree = ast.parse(path.read_text(encoding = "utf-8")) + wanted = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in names] + found = {n.name for n in wanted} + assert found == set(names), f"{path.name}: missing {sorted(set(names) - found)}" + module = ast.Module(body = wanted, type_ignores = []) + ns: dict = {"os": os, "sys": sys, "Path": Path} + exec(compile(module, str(path), "exec"), ns) + return ns + + +prebuilt_mod = _load_prebuilt_module() +llama_ns = _extract_functions(_LLAMA_CPP_PATH, _HELPERS) + + +def _impls(): + """The two copies of the helper, by the file they live in.""" + return { + "studio/install_llama_prebuilt.py": prebuilt_mod._native_linux_system_rocm_lib_dirs, + "studio/backend/core/inference/llama_cpp.py": llama_ns[ + "_native_linux_system_rocm_lib_dirs" + ], + } + + +def _norm(paths): + """os.path.join emits '\\' on the Windows test host; compare POSIX-style.""" + return [str(p).replace("\\", "/") for p in paths] + + +def _fake_exists(present): + """os.path.exists stub over a set of POSIX paths.""" + present = {p.replace("\\", "/") for p in present} + + def _exists(p): + return str(p).replace("\\", "/") in present + + return _exists + + +@pytest.fixture +def bundle_dir(tmp_path): + """A prebuilt directory that does contain a bundled HIP runtime.""" + d = tmp_path / "bundle" + d.mkdir() + (d / "libggml-hip.so").write_text("") + return d + + +@pytest.fixture(autouse = True) +def _clean_rocm_env(monkeypatch): + for var in ("UNSLOTH_LLAMA_NO_SYSTEM_ROCM", "HIP_PATH", "HIP_PATH_57", "ROCM_PATH"): + monkeypatch.delenv(var, raising = False) + + +def _call( + impl, + bundle, + present, + platform = "linux", +): + """Run one copy of the helper against a fake host. + + sys.platform is patched inside the call rather than in a fixture: pytest's own + tmp_path factory branches on it, so a session-wide patch breaks the fixture on + a Windows test host.""" + with patch.object(sys, "platform", platform): + with patch("os.path.exists", _fake_exists(present)): + return _norm(impl(str(bundle))) + + +class TestBundledHipPresent: + """The prepend only makes sense when the prebuilt actually bundles HIP; a + CPU or CUDA build must be left alone.""" + + @pytest.mark.parametrize( + "where", ["studio/install_llama_prebuilt.py", "studio/backend/core/inference/llama_cpp.py"] + ) + def test_detects_versioned_and_plain_sonames(self, tmp_path, where): + impl = ( + prebuilt_mod._bundled_hip_present + if where == "studio/install_llama_prebuilt.py" + else llama_ns["_bundled_hip_present"] + ) + plain = tmp_path / "plain" + plain.mkdir() + (plain / "libggml-hip.so").write_text("") + versioned = tmp_path / "versioned" + versioned.mkdir() + (versioned / "libggml-hip.so.0.0.1").write_text("") + cpu_only = tmp_path / "cpu" + cpu_only.mkdir() + (cpu_only / "libggml-cpu.so").write_text("") + assert impl(str(plain)) is True, f"{where}: plain soname not detected" + assert impl(str(versioned)) is True, f"{where}: versioned soname not detected" + assert impl(str(cpu_only)) is False, f"{where}: CPU-only build treated as HIP" + assert impl("") is False, f"{where}: empty binary_dir must be falsy" + assert ( + impl(str(tmp_path / "does-not-exist")) is False + ), f"{where}: missing dir must be falsy" + + +class TestNativeLinuxGates: + """Each gate, on both copies. A gate that stops working silently reorders + LD_LIBRARY_PATH for a platform the fix was never aimed at.""" + + _ROCM_LIB = "/opt/rocm/lib" + _HOST = {"/dev/kfd", "/opt/rocm/lib/libhsa-runtime64.so"} + + _run = staticmethod(_call) + + def test_returns_system_rocm_lib_on_a_native_rocm_host(self, bundle_dir): + for where, impl in _impls().items(): + assert self._run(impl, bundle_dir, self._HOST) == [self._ROCM_LIB], where + + def test_accepts_versioned_hsa_runtime_soname(self, bundle_dir): + present = {"/dev/kfd", "/opt/rocm/lib/libhsa-runtime64.so.1"} + for where, impl in _impls().items(): + assert self._run(impl, bundle_dir, present) == [self._ROCM_LIB], where + + @pytest.mark.parametrize("platform", ["win32", "darwin"]) + def test_no_op_off_linux(self, bundle_dir, platform): + for where, impl in _impls().items(): + assert self._run(impl, bundle_dir, self._HOST, platform = platform) == [], where + + def test_no_op_on_wsl(self, bundle_dir): + """WSL has its own ordering path (plus HSA_ENABLE_DXG_DETECTION); /dev/dxg + must hand off to it, not double-prepend here.""" + for where, impl in _impls().items(): + assert self._run(impl, bundle_dir, self._HOST | {"/dev/dxg"}) == [], where + + def test_no_op_without_amdkfd(self, bundle_dir): + """No /dev/kfd: NVIDIA host, CPU host, or a container without the AMD + device node. Prepending system ROCm there would be pure breakage.""" + present = {"/opt/rocm/lib/libhsa-runtime64.so"} + for where, impl in _impls().items(): + assert self._run(impl, bundle_dir, present) == [], where + + def test_no_op_when_prebuilt_bundles_no_hip(self, tmp_path): + cpu_bundle = tmp_path / "cpu-bundle" + cpu_bundle.mkdir() + for where, impl in _impls().items(): + assert self._run(impl, cpu_bundle, self._HOST) == [], where + + def test_no_op_when_system_rocm_has_no_hsa_runtime(self, bundle_dir): + """ROCm dir exists but is not a usable runtime install.""" + for where, impl in _impls().items(): + assert self._run(impl, bundle_dir, {"/dev/kfd"}) == [], where + + def test_opt_out_env_wins_over_everything(self, bundle_dir, monkeypatch): + monkeypatch.setenv("UNSLOTH_LLAMA_NO_SYSTEM_ROCM", "1") + for where, impl in _impls().items(): + assert self._run(impl, bundle_dir, self._HOST) == [], where + + def test_opt_out_env_only_honours_exactly_one(self, bundle_dir, monkeypatch): + """Documented switch is =1; "0"/"" must not disable the fix.""" + for value in ("0", "", "false"): + monkeypatch.setenv("UNSLOTH_LLAMA_NO_SYSTEM_ROCM", value) + for where, impl in _impls().items(): + assert self._run(impl, bundle_dir, self._HOST) == [ + self._ROCM_LIB + ], f"{where} ({value!r})" + + +class TestNativeLinuxRootResolution: + """Which ROCm roots are searched, in what order.""" + + _run = staticmethod(_call) + + def test_env_roots_take_precedence_over_opt_rocm(self, bundle_dir, monkeypatch): + """A user with a side-by-side ROCm (HIP_PATH) must get theirs first: the + one matching their driver, not whatever /opt/rocm happens to be.""" + monkeypatch.setenv("HIP_PATH", "/usr/local/rocm7") + present = { + "/dev/kfd", + "/usr/local/rocm7/lib/libhsa-runtime64.so", + "/opt/rocm/lib/libhsa-runtime64.so", + } + for where, impl in _impls().items(): + assert self._run(impl, bundle_dir, present) == [ + "/usr/local/rocm7/lib", + "/opt/rocm/lib", + ], where + + def test_all_three_env_roots_are_consulted_in_order(self, bundle_dir, monkeypatch): + monkeypatch.setenv("HIP_PATH", "/a") + monkeypatch.setenv("HIP_PATH_57", "/b") + monkeypatch.setenv("ROCM_PATH", "/c") + present = { + "/dev/kfd", + "/a/lib/libhsa-runtime64.so", + "/b/lib/libhsa-runtime64.so", + "/c/lib/libhsa-runtime64.so", + } + for where, impl in _impls().items(): + assert self._run(impl, bundle_dir, present) == ["/a/lib", "/b/lib", "/c/lib"], where + + def test_lib64_layout_is_found(self, bundle_dir): + """RHEL / SUSE ROCm packages install to lib64.""" + present = {"/dev/kfd", "/opt/rocm/lib64/libhsa-runtime64.so"} + for where, impl in _impls().items(): + assert self._run(impl, bundle_dir, present) == ["/opt/rocm/lib64"], where + + def test_lib_precedes_lib64_when_both_exist(self, bundle_dir): + present = { + "/dev/kfd", + "/opt/rocm/lib/libhsa-runtime64.so", + "/opt/rocm/lib64/libhsa-runtime64.so", + } + for where, impl in _impls().items(): + assert self._run(impl, bundle_dir, present) == [ + "/opt/rocm/lib", + "/opt/rocm/lib64", + ], where + + def test_duplicate_roots_are_deduped(self, bundle_dir, monkeypatch): + """ROCM_PATH=/opt/rocm is the common setup; it must not emit the dir twice.""" + monkeypatch.setenv("ROCM_PATH", "/opt/rocm") + present = {"/dev/kfd", "/opt/rocm/lib/libhsa-runtime64.so"} + for where, impl in _impls().items(): + assert self._run(impl, bundle_dir, present) == ["/opt/rocm/lib"], where + + def test_empty_env_var_is_ignored(self, bundle_dir, monkeypatch): + monkeypatch.setenv("HIP_PATH", "") + present = {"/dev/kfd", "/opt/rocm/lib/libhsa-runtime64.so"} + for where, impl in _impls().items(): + assert self._run(impl, bundle_dir, present) == ["/opt/rocm/lib"], where + + +class TestHelperParity: + """The two copies carry a "mirrors ..." comment and nothing enforced it.""" + + @pytest.mark.parametrize("name", _HELPERS) + def test_bodies_are_identical(self, name): + a = _function_ast(_PREBUILT_PATH, name) + b = _function_ast(_LLAMA_CPP_PATH, name) + assert ast.dump(a) == ast.dump(b), ( + f"{name} has drifted between install_llama_prebuilt.py and llama_cpp.py; " + "the install-time and serve-time launchers must resolve the same lib dirs" + ) + + +def _function_ast(path: Path, name: str) -> ast.FunctionDef: + """The function's executable body, with docstring and type annotations + stripped: llama_cpp.py quotes its annotations ('list[str]') for the + older-typing lint and documents itself as mirroring the installer. Neither is + drift; the code is.""" + tree = ast.parse(path.read_text(encoding = "utf-8")) + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name == name: + for child in ast.walk(node): + if isinstance(child, (ast.AnnAssign, ast.arg)): + child.annotation = None + elif isinstance(child, ast.FunctionDef): + child.returns = None + if ( + node.body + and isinstance(node.body[0], ast.Expr) + and isinstance(node.body[0].value, ast.Constant) + and isinstance(node.body[0].value.value, str) + ): + node.body = node.body[1:] + return node + raise AssertionError(f"{name} not found in {path.name}") + + +class TestBinaryEnvNativeOrdering: + """install-time validation launches the binary through binary_env.""" + + @staticmethod + def _linux_host(): + return prebuilt_mod.HostInfo( + system = "Linux", + machine = "x86_64", + is_windows = False, + is_linux = True, + is_macos = False, + is_x86_64 = True, + is_arm64 = False, + nvidia_smi = None, + driver_cuda_version = None, + compute_caps = [], + visible_cuda_devices = None, + has_physical_nvidia = False, + has_usable_nvidia = False, + has_rocm = True, + ) + + def test_system_rocm_precedes_bundle_dir(self, tmp_path): + binary = tmp_path / "bundle" / "llama-server" + binary.parent.mkdir(parents = True) + binary.write_text("") + sys_rocm = tmp_path / "sysrocm" # dedupe_existing_dirs drops missing dirs + sys_rocm.mkdir() + with patch.object(prebuilt_mod, "_wsl_system_rocm_lib_dirs", return_value = []): + with patch.object( + prebuilt_mod, "_native_linux_system_rocm_lib_dirs", return_value = [str(sys_rocm)] + ): + with patch.dict(os.environ, {}, clear = True): + env = prebuilt_mod.binary_env(binary, tmp_path, self._linux_host()) + ld = [str(Path(p).resolve()) for p in env["LD_LIBRARY_PATH"].split(os.pathsep)] + assert ld.index(str(sys_rocm.resolve())) < ld.index(str(binary.parent.resolve())) + + def test_native_path_does_not_enable_dxg_detection(self, tmp_path): + """HSA_ENABLE_DXG_DETECTION belongs to the WSL branch only; setting it on + bare metal changes HSA agent enumeration for every native AMD user.""" + binary = tmp_path / "bundle" / "llama-server" + binary.parent.mkdir(parents = True) + binary.write_text("") + sys_rocm = tmp_path / "sysrocm" + sys_rocm.mkdir() + with patch.object(prebuilt_mod, "_wsl_system_rocm_lib_dirs", return_value = []): + with patch.object( + prebuilt_mod, "_native_linux_system_rocm_lib_dirs", return_value = [str(sys_rocm)] + ): + with patch.dict(os.environ, {}, clear = True): + env = prebuilt_mod.binary_env(binary, tmp_path, self._linux_host()) + assert "HSA_ENABLE_DXG_DETECTION" not in env + + def test_helper_is_asked_about_the_binary_dir_not_the_install_dir(self, tmp_path): + """_bundled_hip_present globs the directory it is handed; passing + install_dir would look for libggml-hip.so in the wrong place and no-op.""" + binary = tmp_path / "bundle" / "llama-server" + binary.parent.mkdir(parents = True) + binary.write_text("") + seen = [] + + def _spy(binary_dir = ""): + seen.append(binary_dir) + return [] + + with patch.object(prebuilt_mod, "_wsl_system_rocm_lib_dirs", return_value = []): + with patch.object(prebuilt_mod, "_native_linux_system_rocm_lib_dirs", _spy): + with patch.dict(os.environ, {}, clear = True): + prebuilt_mod.binary_env(binary, tmp_path, self._linux_host()) + assert seen == [str(binary.parent)] + + def test_no_prepend_leaves_bundle_dir_first(self, tmp_path): + binary = tmp_path / "bundle" / "llama-server" + binary.parent.mkdir(parents = True) + binary.write_text("") + with patch.object(prebuilt_mod, "_wsl_system_rocm_lib_dirs", return_value = []): + with patch.object(prebuilt_mod, "_native_linux_system_rocm_lib_dirs", return_value = []): + with patch.dict(os.environ, {}, clear = True): + env = prebuilt_mod.binary_env(binary, tmp_path, self._linux_host()) + assert env["LD_LIBRARY_PATH"].split(os.pathsep)[0] == str(binary.parent) + + +class TestLlamaCppRuntimeNativeOrdering: + """The serve-time launcher builds LD_LIBRARY_PATH inline inside a large + function, so this half stays a source check (as the WSL sibling does).""" + + def test_prepends_before_binary_dir(self): + source = _LLAMA_CPP_PATH.read_text(encoding = "utf-8") + idx_helper = source.find("lib_dirs.extend(_native_linux_system_rocm_lib_dirs(binary_dir))") + idx_binary = source.find("lib_dirs.append(binary_dir)") + assert ( + idx_helper != -1 + ), "serve-time launcher must call the native-Linux helper with binary_dir" + assert idx_binary != -1 + assert ( + idx_helper < idx_binary + ), "system ROCm must be searched before the bundled HIP runtime" + + def test_dxg_detection_stays_on_the_wsl_branch(self): + """HSA_ENABLE_DXG_DETECTION must be set from the WSL helper's result only.""" + source = _LLAMA_CPP_PATH.read_text(encoding = "utf-8") + idx_wsl = source.find("lib_dirs.extend(_wsl_system_rocm_lib_dirs())") + idx_dxg = source.find('env.setdefault("HSA_ENABLE_DXG_DETECTION", "1")', idx_wsl) + idx_native = source.find("lib_dirs.extend(_native_linux_system_rocm_lib_dirs(binary_dir))") + assert idx_wsl != -1 and idx_dxg != -1 and idx_native != -1 + assert idx_wsl < idx_dxg < idx_native, ( + "HSA_ENABLE_DXG_DETECTION must be decided from the WSL dirs alone, before the " + "native-Linux dirs are appended to lib_dirs" + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py index b919206120..4f65857a3e 100644 --- a/tests/studio/install/test_rocm_support.py +++ b/tests/studio/install/test_rocm_support.py @@ -2695,9 +2695,11 @@ class TestGfxArchNameFallback: ("AMD Radeon(TM) 890M", "gfx1150"), ("AMD Ryzen AI 9 HX 370 w/ Radeon 890M", "gfx1150"), ("AMD Radeon RX 9070 XT", "gfx1201"), - ("AMD Radeon RX 9070", "gfx1200"), - ("AMD Radeon RX 7700S", "gfx1102"), # (?!S) lookahead must not hit gfx1100 - ("AMD Radeon RX 7700 XT", "gfx1100"), + ("AMD Radeon RX 9070", "gfx1201"), # Navi 48 like the XT, not Navi 44 + ("AMD Radeon RX 9060 XT", "gfx1200"), # Navi 44 + ("AMD Radeon RX 7700S", "gfx1102"), # (?!S) lookahead must not hit gfx1101 + ("AMD Radeon RX 7700 XT", "gfx1101"), # Navi 32 + ("AMD Radeon RX 7900 XTX", "gfx1100"), # Navi 31 ("AMD Radeon(TM) 780M", "gfx1103"), ("NVIDIA GeForce RTX 4090", None), ("Microsoft Basic Display Adapter", None), @@ -3845,7 +3847,7 @@ class TestStrixRocm71Override: assert m._infer_linux_amd_gfx_arch() == "gfx1151" def test_install_sh_cpuinfo_inference_requires_pci_evidence(self): - """install.sh mirror of the VM/container guard: both cpuinfo greps must be + """install.sh mirror of the VM/container guard: every cpuinfo grep must be gated on _gpu_evidence (AMD PCI display device via _amd_gpu_present_via_pci, or the WSL librocdxg gate), and the gate must sit before the first grep.""" source = _INSTALL_SH_PATH.read_text(encoding = "utf-8") @@ -3855,9 +3857,9 @@ class TestStrixRocm71Override: infer = body.find("grep -qiE 'Ryzen AI Max") assert pci >= 0 and infer >= 0 assert pci < infer, "the PCI evidence check must run before the cpuinfo inference" - assert ( - body.count('[ -n "$_gpu_evidence" ] && grep -qiE') == 2 - ), "both cpuinfo greps (gfx1151 and gfx1150) must be gated on _gpu_evidence" + assert body.count("grep -qiE") == body.count( + '[ -n "$_gpu_evidence" ] && grep -qiE' + ), "every cpuinfo grep (gfx1151/gfx1150/gfx1152) must be gated on _gpu_evidence" def test_lspci_scan_covers_all_display_controllers(self): """The lspci fallback must scan every display-class line, not just the @@ -4211,11 +4213,14 @@ class TestStrixRocm71Override: source = _INSTALL_SH_PATH.read_text(encoding = "utf-8") # The 2.11 constraint block must switch on $_torch_index_leaf, not the full # $TORCH_INDEX_URL (a */gfx* match false-positives on a mirror base path). Only the - # _grouped_mm-bug gfx families (gfx120X-all / gfx1151 / gfx1150) are pushed to 2.11; + # _grouped_mm-bug gfx families (gfx120X-all / gfx1151 / gfx1150 / gfx1152) go to 2.11; # a bare gfx* would also floor gfx110X-all/gfx90a/gfx908, left bare on purpose. - assert 'case "$_torch_index_leaf" in\n rocm7.2|gfx120x-all|gfx1151|gfx1150)' in source, ( + assert ( + 'case "$_torch_index_leaf" in\n rocm7.2|gfx120x-all|gfx1151|gfx1150|gfx1152)' + in source + ), ( "the torch>=2.11 constraint must match the specific gfx leaves that need " - "it (rocm7.2|gfx120x-all|gfx1151|gfx1150), not a bare gfx* or the whole URL" + "it (rocm7.2|gfx120x-all|gfx1151|gfx1150|gfx1152), not a bare gfx* or the URL" ) def test_amd_rocm_mirror_env_var_respected(self): diff --git a/tests/studio/test_ci_shell_suite_coverage.py b/tests/studio/test_ci_shell_suite_coverage.py new file mode 100644 index 0000000000..50ed22f9de --- /dev/null +++ b/tests/studio/test_ci_shell_suite_coverage.py @@ -0,0 +1,197 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Guards that the installer test suites actually run on a PR. + +Two ways coverage went missing without anyone noticing: + +1. Backend CI ran a hardcoded list of tests/sh/*.sh files. New tests were added + to the directory and never to the list, so by the time this was written the + list was seven files behind -- including test_strixhalo_wsl_reroute.sh, the + only shell coverage of the ROCm WSL reroute, which had never run on a PR. + tests/run_all.sh, the local entrypoint, had drifted the other way. + +2. Backend CI's path filter did not include install.sh / install.ps1, while a + large share of the suites it runs (tests/sh/*, tests/studio/install/*) assert + against exactly those two files. An install-only change -- the shape most + AMD/ROCm routing fixes take, e.g. #7277 / #7293 / #7300 -- skipped the + workflow that tests it. + +Both are now discovery-based. These tests fail if either reverts to a list, if a +shell test lands somewhere the discovery cannot see it, or if a skip is added +without a reason next to it. +""" + +import re +from pathlib import Path + +import pytest +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] + +_WORKFLOWS = REPO_ROOT / ".github" / "workflows" +_BACKEND_CI = _WORKFLOWS / "studio-backend-ci.yml" +_PARITY_CI = _WORKFLOWS / "cross-platform-parity-ci.yml" +_RUN_ALL = REPO_ROOT / "tests" / "run_all.sh" +_SH_DIR = REPO_ROOT / "tests" / "sh" + +# Files deliberately not run by the auto-discovered Backend CI step. Each needs +# a reason here AND in the workflow; anything else in tests/sh must run. +_EXPECTED_CI_SKIPS = { + "test_install_host_defaults.sh": "asserts an install.ps1 layout that has drifted", + "test_install_rollback_lifecycle.sh": "runs on both platforms in cross-platform-parity-ci.yml", +} + + +def _backend_ci() -> dict: + return yaml.safe_load(_BACKEND_CI.read_text(encoding = "utf-8")) + + +def _shell_step_script() -> str: + """The `run:` body of the shell-installer step, located by name through the + parsed YAML rather than by slicing the raw file.""" + for job in _backend_ci()["jobs"].values(): + for step in job.get("steps", []): + if step.get("name") == "Shell installer tests": + return step["run"] + raise AssertionError("Backend CI has no 'Shell installer tests' step") + + +def _shell_test_files(): + files = sorted(p.name for p in _SH_DIR.glob("test_*.sh")) + assert files, "tests/sh has no test_*.sh files -- did the directory move?" + return files + + +def _skip_list(source: str) -> set[str]: + """The skip= / SH_SKIP= line from a discovery loop.""" + m = re.search(r"^\s*(?:skip|SH_SKIP)=\"([^\"]*)\"", source, re.MULTILINE) + assert m, "no skip list found; the discovery loop must declare one (even if empty)" + return {name for name in m.group(1).split() if name} + + +class TestBackendCiRunsEveryShellTest: + def test_step_discovers_the_directory_instead_of_listing_files(self): + """Matched against the parsed step script, and on the glob rather than a + verbatim line, so reformatting the loop does not turn CI red -- only + going back to a hardcoded list does.""" + script = _shell_step_script() + assert re.search(r"for\s+\w+\s+in\s+tests/sh/test_\*\.sh", script), ( + "Backend CI must glob tests/sh; a hardcoded list is how the ROCm WSL " + f"suite went unrun for months. Step script was:\n{script}" + ) + listed = re.findall(r"tests/sh/test_[a-z0-9_]+\.sh", script) + assert not listed, f"Backend CI still names individual shell tests: {sorted(set(listed))}" + + def test_step_fails_loudly_if_discovery_finds_nothing(self): + """A moved directory must break the build, not pass vacuously.""" + assert "no shell tests discovered under tests/sh" in _shell_step_script() + + def test_every_shell_test_runs_or_is_a_known_skip(self): + skips = _skip_list(_shell_step_script()) + unexpected = skips - set(_EXPECTED_CI_SKIPS) + assert not unexpected, ( + f"Backend CI skips {sorted(unexpected)} without a reason recorded in " + "_EXPECTED_CI_SKIPS; add one or stop skipping it" + ) + # Everything else in the directory is covered by the glob. + for name in _shell_test_files(): + assert name not in skips or name in _EXPECTED_CI_SKIPS, name + + def test_skip_entries_are_not_stale(self): + """A skip for a deleted file quietly widens next time a name is reused.""" + existing = set(_shell_test_files()) + for name in _skip_list(_shell_step_script()): + assert name in existing, f"{name} is skipped but no longer exists in tests/sh" + + def test_each_skip_is_documented_in_the_workflow(self): + source = _BACKEND_CI.read_text(encoding = "utf-8") + for name in _EXPECTED_CI_SKIPS: + assert ( + source.count(name) >= 2 + ), f"{name} is skipped in Backend CI without a comment explaining why" + + def test_rollback_lifecycle_really_does_run_elsewhere(self): + """The one skip justified by 'another workflow covers it' must be true.""" + assert "tests/sh/test_install_rollback_lifecycle.sh" in _PARITY_CI.read_text( + encoding = "utf-8" + ) + + def test_rocm_shell_suite_is_in_scope(self): + """The suite whose absence prompted this file: it must exist and be + picked up (i.e. not skipped).""" + assert "test_strixhalo_wsl_reroute.sh" in _shell_test_files() + assert "test_strixhalo_wsl_reroute.sh" not in _skip_list(_shell_step_script()) + + +class TestRunAllMatchesCi: + """tests/run_all.sh is what a contributor runs before pushing. If it and CI + disagree, one of them is lying about the state of the tree.""" + + def test_run_all_discovers_the_directory(self): + source = _RUN_ALL.read_text(encoding = "utf-8") + assert 'for _t in "$TESTS_DIR"/sh/test_*.sh; do' in source + + def test_run_all_invokes_the_tests_with_bash(self): + """Both runners must use the interpreter the tests declare. Every file + under tests/sh/ has a bash shebang, and on Debian/Ubuntu /bin/sh is + dash, under which three of them fail on bashisms. Running them with sh + would fail the suite locally for reasons CI never reproduces.""" + source = _RUN_ALL.read_text(encoding = "utf-8") + assert 'bash "$_t"' in source, "tests/run_all.sh must run tests/sh/ with bash" + assert 'sh "$_t"' not in source.replace( + 'bash "$_t"', "" + ), "tests/run_all.sh still invokes a discovered test with sh" + assert 'bash "$s"' in _shell_step_script(), "Backend CI must run tests/sh/ with bash" + + def test_run_all_skips_are_a_subset_of_ci_skips(self): + local = _skip_list(_RUN_ALL.read_text(encoding = "utf-8")) + unexpected = local - set(_EXPECTED_CI_SKIPS) + assert not unexpected, ( + f"tests/run_all.sh skips {sorted(unexpected)} that CI still runs: a " + "contributor would see green locally and red on the PR" + ) + + +class TestBackendCiPathFilters: + """The workflow has to fire on the files its tests assert against.""" + + def _paths(self) -> set[str]: + """Read the real trigger through the YAML parser. `on:` is a YAML 1.1 + boolean, so pyyaml keys it as True.""" + wf = _backend_ci() + triggers = wf.get("on", wf.get(True)) + assert triggers, "Backend CI has no trigger block" + paths = triggers["pull_request"]["paths"] + assert paths, "Backend CI pull_request trigger has no paths filter" + return set(paths) + + @pytest.mark.parametrize( + "path,why", + [ + ("install.sh", "tests/sh/* and tests/studio/install/* assert against it"), + ("install.ps1", "the Windows/ROCm arch tables and pin allowlist live here"), + ("studio/**", "covers studio/setup.sh, studio/setup.ps1, install_python_stack.py"), + ("tests/**", "test-only changes must run the tests they touch"), + ], + ) + def test_trigger_covers(self, path, why): + assert path in self._paths(), f"Backend CI does not run when {path} changes ({why})" + + def test_installer_change_would_trigger_the_workflow(self): + """End to end: the exact filenames the ROCm fixes edit.""" + paths = self._paths() + for changed in ("install.sh", "install.ps1"): + assert changed in paths + for changed in ("studio/setup.ps1", "studio/setup.sh", "studio/install_python_stack.py"): + assert any( + changed.startswith(pattern.rstrip("*").rstrip("/")) + for pattern in paths + if pattern.endswith("/**") + ), f"nothing in the path filter matches {changed}" + + +if __name__ == "__main__": + pytest.main([__file__, "-v"])