diff --git a/install.sh b/install.sh index adec5f2d6e..e0f57c198b 100755 --- a/install.sh +++ b/install.sh @@ -2127,6 +2127,23 @@ _has_amd_rocm_gpu() { return 1 } +# Returns 0 if an AMD display GPU is on the PCI bus even when ROCm can't use it +# (e.g. a Strix Halo iGPU with no /dev/kfd). Only sharpens the "no GPU detected" +# hint. vendor 0x1002 = AMD/ATI; class 0x03* = display controller. +_amd_gpu_present_via_pci() { + [ -d /sys/bus/pci/devices ] || return 1 + for _pci_vendor in /sys/bus/pci/devices/*/vendor; do + [ -r "$_pci_vendor" ] || continue + read -r _v < "$_pci_vendor" 2>/dev/null || continue + [ "$_v" = "0x1002" ] || continue + _cls="${_pci_vendor%vendor}class" + [ -r "$_cls" ] || continue + read -r _c < "$_cls" 2>/dev/null || continue + case "$_c" in 0x03*) return 0 ;; esac + done + return 1 +} + # ── Detect GPU and choose PyTorch index URL ── # Mirrors Get-TorchIndexUrl in install.ps1. # On CPU-only machines this returns the cpu index, avoiding the solver @@ -2818,29 +2835,45 @@ case "$TORCH_INDEX_URL" in fi ;; esac -# ── Strix Halo / Strix Point: force rocm7.2 wheels, bypass Radeon repo ─────── -# gfx1151 (Strix Halo) and gfx1150 (Strix Point) have a ROCm 7.1 driver bug -# that causes a segfault in torch._grouped_mm (moe_utils.py line 167). -# The Radeon repo now ships cp313 wheels for rocm-rel-7.1, so when -# _amd_gpu_radeon=true the installer silently lands on the broken combo. -# Detect these GPUs when TORCH_INDEX_URL is rocm7.1 and override to rocm7.2. -case "$TORCH_INDEX_URL" in - */rocm7.1|*/rocm7.1.*) +# 0 when a rocmX.Y index leaf ($1, the final path segment) is older than floor +# $2.$3 (int compare, so rocm7.2 < rocm7.13). Non-rocm leaves (gfx*, cu*, cpu) and +# non-numeric versions return 1. Leaf-based (like $_torch_index_leaf) so a mirror +# base holding its own rocm token compares the family leaf, not the base path. +_rocm_leaf_below() { + case "$1" in rocm[0-9]*.[0-9]*) : ;; *) return 1 ;; esac + _rb=${1#rocm}; _maj=${_rb%%.*}; _min=${_rb#*.}; _min=${_min%%.*} + case "$_maj$_min" in *[!0-9]*) return 1 ;; esac + if [ "$_maj" -lt "$2" ]; then return 0; fi + if [ "$_maj" -eq "$2" ] && [ "$_min" -lt "$3" ]; then return 0; fi + return 1 +} +# ── Strix Halo / Strix Point: route to the AMD arch-specific index ─────────── +# gfx1151/gfx1150 need torch 2.11+rocm7.13 from repo.amd.com/rocm/whl/gfx/, +# which carries AMD's real fixes (the rocm7.1 _grouped_mm segfault, moe_utils.py:167, +# and later Strix kernel bugs). Every generic pytorch.org index below rocm7.13 lacks +# them (and the Radeon repo can be offline, unslothai#7264), so reroute a detected +# Strix GPU whenever the picked index is older than the arch build -- covers today's +# rocm6.0-7.2 and any future 7.x < 7.13; rocm7.13+ already has the fixes, so leave it. +case "$_torch_index_leaf" in + rocm[0-9]*) # Collect every gfx token in rocminfo / amd-smi enumeration order # (skip duplicates), then index by HIP_VISIBLE_DEVICES / # ROCR_VISIBLE_DEVICES so a mixed Strix iGPU + non-Strix dGPU box # where the user selected the dGPU does NOT get rerouted to the # Strix per-gfx index. + # || true on each probe: no gfx match makes grep exit 1, which under + # set -euo pipefail would abort the installer before the next fallback + # runs (now that the case matches every rocm* index, not just rocm7.1). _gfx_all="" if command -v rocminfo >/dev/null 2>&1; then - _gfx_all=$(rocminfo 2>/dev/null | grep -oE 'gfx[1-9][0-9a-z]{2,3}') + _gfx_all=$(rocminfo 2>/dev/null | grep -oE 'gfx[1-9][0-9a-z]{2,3}' || true) fi if [ -z "$_gfx_all" ] && command -v amd-smi >/dev/null 2>&1; then - _gfx_all=$(amd-smi list 2>/dev/null | grep -oE 'gfx[1-9][0-9a-z]{2,3}') + _gfx_all=$(amd-smi list 2>/dev/null | grep -oE 'gfx[1-9][0-9a-z]{2,3}' || true) # PowerShell paths also probe `amd-smi static --asic`; mirror it # so a host with hipinfo-less amd-smi reports the gfx target. if [ -z "$_gfx_all" ]; then - _gfx_all=$(amd-smi static --asic 2>/dev/null | grep -oE 'gfx[1-9][0-9a-z]{2,3}') + _gfx_all=$(amd-smi static --asic 2>/dev/null | grep -oE 'gfx[1-9][0-9a-z]{2,3}' || true) fi fi _runtime_gfx="" @@ -2865,13 +2898,14 @@ case "$TORCH_INDEX_URL" in case "$_runtime_gfx" in gfx1151|gfx1150) _strix_gfx="$_runtime_gfx" ;; esac - if [ -n "$_strix_gfx" ]; then + # Skip rocm7.13+ generic indexes: they already ship the fixes, so the + # arch build (rocm7.13) would be a downgrade rather than a rescue. + if [ -n "$_strix_gfx" ] && _rocm_leaf_below "$_torch_index_leaf" 7 13; then echo "" >&2 - echo " [WARN] $_strix_gfx (Strix) + ROCm 7.1 detected -- known _grouped_mm segfault" >&2 - echo " [WARN] ROCm 7.1 wheels are broken for gfx1150/gfx1151 (moe_utils.py:167)" >&2 - echo " [WARN] Routing to AMD arch-specific index (torch 2.11+rocm7.13 has the real fix)" >&2 - echo " [WARN] Upgrade ROCm to 7.2+ to use the standard index:" >&2 - echo " [WARN] https://rocm.docs.amd.com/en/latest/deploy/linux/index.html" >&2 + echo " [WARN] $_strix_gfx (Strix) detected -- routing to the AMD arch-specific index" >&2 + echo " [WARN] torch 2.11+rocm7.13 has AMD's real gfx1150/gfx1151 fixes (the ROCm 7.1" >&2 + echo " [WARN] _grouped_mm segfault, moe_utils.py:167, and later Strix kernel bugs)," >&2 + echo " [WARN] and is more reliable than the rocm7.2 index or an offline Radeon repo." >&2 echo "" >&2 # AMD's arch-specific index serves torch 2.11.0+rocm7.13.0 which has AMD's # actual fix for the gfx1151/gfx1150 _grouped_mm kernel bug -- preferred @@ -3031,6 +3065,13 @@ case "$TORCH_INDEX_URL" in substep " driver is current; or run unsloth/scripts/install_rocm_wsl_strixhalo.sh yourself." else substep "AMD ROCm users: see https://docs.unsloth.ai/get-started/install-and-update/amd" + # Only when ROCm truly can't see the GPU: a detected-but-too-old + # ROCm (rocminfo works, wheels need 6.0+) has its own guidance. + if ! _has_amd_rocm_gpu && _amd_gpu_present_via_pci; then + substep "An AMD GPU is on the PCI bus but ROCm cannot see it (no /dev/kfd," "$C_WARN" + substep " rocminfo, or amd-smi). Install the ROCm kernel stack so /dev/kfd exists;" + substep " Strix Halo (gfx1151/gfx1150) needs a recent kernel (6.11+) and ROCm 7.x." + fi fi substep "Re-run with --no-torch for GGUF-only (faster, no PyTorch):" substep " curl -fsSL https://unsloth.ai/install.sh | sh -s -- --no-torch" diff --git a/tests/sh/test_previous_torch_pin.sh b/tests/sh/test_previous_torch_pin.sh index 1bc0d1f27f..f910affc2f 100644 --- a/tests/sh/test_previous_torch_pin.sh +++ b/tests/sh/test_previous_torch_pin.sh @@ -98,7 +98,7 @@ assert_eq "probe before venv replacement" "yes" "$([ -n "$_probe_line" ] && [ -n # The pin must be evaluated AFTER the last index/constraint decision (the Strix # reroute raises the floor), so a raised floor rejects an older kept release. _pin_line=$(grep -n '_prev_pin=\$(_previous_torch_pin' "$INSTALL_SH" | head -1 | cut -d: -f1) -_strix_line=$(grep -n 'Strix Halo / Strix Point: force rocm7.2 wheels' "$INSTALL_SH" | head -1 | cut -d: -f1) +_strix_line=$(grep -n 'Strix Halo / Strix Point:' "$INSTALL_SH" | head -1 | cut -d: -f1) assert_eq "pin evaluated after the Strix reroute" "yes" "$([ -n "$_pin_line" ] && [ -n "$_strix_line" ] && [ "$_pin_line" -gt "$_strix_line" ] && echo yes)" # A kept release that vanished from the index must fall back to the supported range. assert_eq "resolve-failure fallback wired" "yes" "$(grep -q 'TORCH_CONSTRAINT="\$_PREV_FALLBACK_CONSTRAINT"' "$INSTALL_SH" && echo yes)" diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py index f7dcd79d10..5825bbe31f 100644 --- a/tests/studio/install/test_rocm_support.py +++ b/tests/studio/install/test_rocm_support.py @@ -3,8 +3,11 @@ import importlib.util import json import os +import re +import shutil import subprocess import sys +import tempfile from pathlib import Path from unittest.mock import MagicMock, mock_open, patch, PropertyMock @@ -3191,13 +3194,64 @@ class TestStrixRocm71Override: source = _INSTALL_SH_PATH.read_text(encoding = "utf-8") assert "moe_utils" in source or "_grouped_mm" in source - def test_strix_override_only_fires_on_rocm71(self): - """install.sh must scope the Strix override to rocm7.1 only (not rocm7.2+).""" + def test_strix_override_scoped_below_arch_floor(self): + """Strix reroute must fire for rocm leaves BELOW the arch floor (7.13) and + NOT at/above it. Executed via _rocm_leaf_below so it verifies the actual + version comparison, not a text match that a comment could satisfy.""" source = _INSTALL_SH_PATH.read_text(encoding = "utf-8") - strix_idx = source.find("_strix_gfx") - assert strix_idx != -1 - context_before = source[max(0, strix_idx - 2400) : strix_idx] - assert "rocm7.1" in context_before + # Selector + gate must switch on the index LEAF, not the whole URL (a mirror + # base path with its own rocm token would false-positive otherwise). + assert 'case "$_torch_index_leaf" in' in source + assert '_rocm_leaf_below "$_torch_index_leaf" 7 13' in source + shell = shutil.which("sh") or shutil.which("bash") + if not shell: + pytest.skip("no POSIX shell to execute _rocm_leaf_below") + match = re.search(r"^_rocm_leaf_below\(\) \{.*?^\}", source, re.S | re.M) + assert match, "could not extract _rocm_leaf_below from install.sh" + fn = match.group(0) + + def below(leaf): + return ( + subprocess.run( + [shell, "-c", f'{fn}\n_rocm_leaf_below "$1" 7 13', "_", leaf] + ).returncode + == 0 + ) + + for leaf in ("rocm6.0", "rocm7.0", "rocm7.1", "rocm7.2", "rocm7.12"): + assert below(leaf), f"{leaf} must reroute (below arch floor 7.13)" + for leaf in ("rocm7.13", "rocm7.14", "rocm8.0", "gfx1151", "cu128", "cpu"): + assert not below(leaf), f"{leaf} must NOT reroute (>= floor or non-rocm)" + + def test_gfx_probe_survives_no_match_under_set_e(self): + """A gfx probe whose grep finds no match must not abort install.sh under + set -euo pipefail before the amd-smi fallback runs. The reroute case now + matches every rocm* index, so this would break ordinary 6.x/7.2 installs + with a flaky rocminfo. Executed with shimmed tools, not a text match.""" + shell = shutil.which("bash") + if not shell: + pytest.skip("bash needed to execute the probe block") + source = _INSTALL_SH_PATH.read_text(encoding = "utf-8") + block = re.search( + r'^ _gfx_all=""\n.*?(?=^ _strix_gfx="")', source, re.S | re.M + ) + assert block, "could not extract the gfx-detection block" + with tempfile.TemporaryDirectory() as d: + # rocminfo emits no gfx token; amd-smi supplies gfx1151 (the fallback) + for name, out in (("rocminfo", "no gpu here"), ("amd-smi", "GPU: gfx1151")): + p = os.path.join(d, name) + with open(p, "w", encoding = "utf-8") as f: + f.write(f'#!/bin/sh\ncat <<"EOT"\n{out}\nEOT\n') + os.chmod(p, 0o755) + script = ( + 'set -euo pipefail\nHIP_VISIBLE_DEVICES=""\nROCR_VISIBLE_DEVICES=""\n' + + block.group(0) + + '\nprintf "OK:%s\\n" "$_gfx_all"\n' + ) + env = dict(os.environ, PATH = d + os.pathsep + os.environ.get("PATH", "")) + r = subprocess.run([shell, "-c", script], env = env, capture_output = True, text = True) + assert r.returncode == 0, f"probe aborted under set -e: {r.stderr}" + assert "OK:gfx1151" in r.stdout, f"amd-smi fallback not reached: {r.stdout!r}" def test_torch_constraint_updated_for_strix_amd_index(self): """install.sh must set TORCH_CONSTRAINT>=2.11 when routing Strix to AMD index."""