fix: use partial hipinfo output on crash to avoid CPU fallback (RDNA 4 / gfx1200) (#6292)
* fix: use partial hipinfo output on crash to avoid CPU fallback (#6043) `hipinfo.exe` on some RDNA 4 hosts (e.g. RX 9060 XT / gfx1200) exits with STATUS_ACCESS_VIOLATION (0xC0000005) after printing the gcnArchName line. The previous guard `$LASTEXITCODE -eq 0` in studio/setup.ps1 and `if result.returncode == 0` in install_python_stack.py discarded this partial-but-valid output, causing the installer to fall through to WMI name inference which sets HasROCm=false and installs CPU PyTorch instead of the ROCm wheel. Fix: check for gcnArchName in stdout first; accept the arch regardless of exit code. Only fall through to the amd-smi / WMI path when no gcnArchName is present at all (crash before any output, or a genuine "no device" error). A cyan INFO substep is emitted when the arch is recovered from a crashed hipinfo run so users can see what happened. Adds a regression test covering the crash-with-valid-output path. Fixes #6043 * Fix/adjust hipinfo crash fallback for PR #6292 --------- Co-authored-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Co-authored-by: wasimysaid <wasimysdev@gmail.com> Co-authored-by: Lee Jackson <130007945+Imagineer99@users.noreply.github.com>
This commit is contained in:
Matt Van Horn
GitHub
parent
a8c2012401
commit
08c3878919
4 changed files with 70 additions and 16 deletions
11
install.ps1
11
install.ps1
|
|
@ -1553,7 +1553,9 @@ shell.Run cmd, 0, False
|
|||
$HipSdkInstalled = $true # binary found → SDK is installed regardless of device state
|
||||
try {
|
||||
$hipOut = & $hipinfoExe.Source 2>&1 | Out-String
|
||||
if ($LASTEXITCODE -eq 0 -and $hipOut -match "(?i)gcnArchName") {
|
||||
if ($hipOut -match "(?i)gcnArchName") {
|
||||
# hipinfo can crash after printing gcnArchName (#6043).
|
||||
# Once the arch is printed, keep the ROCm wheel path.
|
||||
$HasROCm = $true
|
||||
$_hipAllArches = @([regex]::Matches($hipOut, "(?im)^\s*gcnArchName\s*:\s*(\S+)") | ForEach-Object { ($_.Groups[1].Value -split ':')[0].Trim().ToLower() })
|
||||
$_hipVisIdx = if ($env:HIP_VISIBLE_DEVICES -match '^\d') { [int]($env:HIP_VISIBLE_DEVICES -split ',')[0] } elseif ($env:ROCR_VISIBLE_DEVICES -match '^\d') { [int]($env:ROCR_VISIBLE_DEVICES -split ',')[0] } else { 0 }
|
||||
|
|
@ -1563,8 +1565,13 @@ shell.Run cmd, 0, False
|
|||
} else {
|
||||
$ROCmGpuLabel = "AMD ROCm"
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Host " [INFO] hipinfo exited with code $LASTEXITCODE but reported gcnArchName -- treating as ROCm-capable (see #6043)" -ForegroundColor Cyan
|
||||
}
|
||||
} elseif ($LASTEXITCODE -ne 0) {
|
||||
# hipinfo ran but returned a HIP runtime error (e.g. "no ROCm-capable device detected")
|
||||
# hipinfo ran but returned a HIP runtime error without any gcnArchName
|
||||
# output (e.g. "no ROCm-capable device detected"), or crashed before
|
||||
# printing device info.
|
||||
$firstLine = ($hipOut -split '\r?\n' | Where-Object { $_.Trim() } | Select-Object -First 1)
|
||||
Write-Host " [WARN] hipinfo returned a HIP runtime error (exit $LASTEXITCODE)" -ForegroundColor Yellow
|
||||
Write-Host " $firstLine" -ForegroundColor Yellow
|
||||
|
|
|
|||
|
|
@ -354,16 +354,20 @@ def _detect_windows_gfx_arch() -> str | None:
|
|||
stderr = subprocess.DEVNULL,
|
||||
timeout = 10,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
text = result.stdout.decode(errors = "replace")
|
||||
# findall gets every gcnArchName line so multi-GPU hosts are
|
||||
# enumerable and HIP_VISIBLE_DEVICES selects correctly.
|
||||
_tokens = [
|
||||
t.strip().lower() for t in re.findall(r"(?im)^\s*gcnArchName\s*:\s*(\S+)", text)
|
||||
]
|
||||
_pick = _dedup_pick(_tokens)
|
||||
if _pick:
|
||||
return _pick
|
||||
# Accept partial output even when hipinfo crashes (e.g. exit code
|
||||
# 0xC0000005 / STATUS_ACCESS_VIOLATION on some RDNA 4 hosts): if
|
||||
# gcnArchName is present in stdout the device was enumerated before
|
||||
# the crash, so the arch is trustworthy. Ignoring it causes a
|
||||
# silent CPU PyTorch fallback (issue #6043).
|
||||
text = result.stdout.decode(errors = "replace")
|
||||
# findall gets every gcnArchName line so multi-GPU hosts are
|
||||
# enumerable and HIP_VISIBLE_DEVICES selects correctly.
|
||||
_tokens = [
|
||||
t.strip().lower() for t in re.findall(r"(?im)^\s*gcnArchName\s*:\s*(\S+)", text)
|
||||
]
|
||||
_pick = _dedup_pick(_tokens)
|
||||
if _pick:
|
||||
return _pick
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -823,7 +823,9 @@ if (-not $HasNvidiaSmi) {
|
|||
$HipSdkInstalled = $true # binary found → SDK is installed regardless of device state
|
||||
try {
|
||||
$hipOut = & $hipinfoExe.Source 2>&1 | Out-String
|
||||
if ($LASTEXITCODE -eq 0 -and $hipOut -match "(?i)gcnArchName") {
|
||||
if ($hipOut -match "(?i)gcnArchName") {
|
||||
# hipinfo can crash after printing gcnArchName (#6043).
|
||||
# Once the arch is printed, keep the ROCm wheel path.
|
||||
$HasROCm = $true
|
||||
$_hipAllArches = @([regex]::Matches($hipOut, "(?im)^\s*gcnArchName\s*:\s*(\S+)") | ForEach-Object { ($_.Groups[1].Value -split ':')[0].Trim().ToLower() })
|
||||
$_hipVisIdx = if ($env:HIP_VISIBLE_DEVICES -match '^\d') { [int]($env:HIP_VISIBLE_DEVICES -split ',')[0] } elseif ($env:ROCR_VISIBLE_DEVICES -match '^\d') { [int]($env:ROCR_VISIBLE_DEVICES -split ',')[0] } else { 0 }
|
||||
|
|
@ -833,8 +835,13 @@ if (-not $HasNvidiaSmi) {
|
|||
} else {
|
||||
$ROCmGpuLabel = "AMD ROCm"
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
substep "[INFO] hipinfo exited with code $LASTEXITCODE but reported gcnArchName -- treating as ROCm-capable (see #6043)" "Cyan"
|
||||
}
|
||||
} elseif ($LASTEXITCODE -ne 0) {
|
||||
# hipinfo ran but returned a HIP runtime error (e.g. "no ROCm-capable device detected")
|
||||
# hipinfo ran but returned a HIP runtime error without any gcnArchName
|
||||
# output (e.g. "no ROCm-capable device detected"), or crashed before
|
||||
# printing device info.
|
||||
$firstLine = ($hipOut -split '\r?\n' | Where-Object { $_.Trim() } | Select-Object -First 1)
|
||||
substep "[WARN] hipinfo returned a HIP runtime error (exit $LASTEXITCODE)" "Yellow"
|
||||
substep " $firstLine" "Yellow"
|
||||
|
|
|
|||
|
|
@ -1820,10 +1820,27 @@ class TestDetectWindowsGfxArch:
|
|||
result = stack_mod._detect_windows_gfx_arch()
|
||||
assert result == "gfx1200"
|
||||
|
||||
def test_returns_none_on_nonzero_returncode(self):
|
||||
def test_returns_arch_on_crash_with_gcnarchname_in_output(self):
|
||||
# Regression test for issue #6043: hipinfo may exit with a non-zero
|
||||
# code (e.g. 0xC0000005 / STATUS_ACCESS_VIOLATION on RDNA 4 hosts)
|
||||
# while still printing the gcnArchName line before crashing. The
|
||||
# previous guard `if result.returncode == 0` discarded this output,
|
||||
# causing a CPU PyTorch fallback. The fix: accept the arch whenever
|
||||
# gcnArchName is present in stdout regardless of exit code.
|
||||
mock_result = MagicMock()
|
||||
mock_result.returncode = -1073741819 # 0xC0000005 STATUS_ACCESS_VIOLATION
|
||||
mock_result.stdout = b"gcnArchName : gfx1200\nsome other line\n"
|
||||
with patch("shutil.which", return_value = "/usr/bin/hipinfo"):
|
||||
with patch("subprocess.run", return_value = mock_result):
|
||||
result = stack_mod._detect_windows_gfx_arch()
|
||||
assert result == "gfx1200"
|
||||
|
||||
def test_returns_none_on_nonzero_returncode_without_gcnarchname(self):
|
||||
# Non-zero exit without any gcnArchName output (e.g. no device detected)
|
||||
# must still return None and fall through to amd-smi / WMI.
|
||||
mock_result = MagicMock()
|
||||
mock_result.returncode = 1
|
||||
mock_result.stdout = b"gcnArchName : gfx1200\n"
|
||||
mock_result.stdout = b"HIP runtime error: no device detected\n"
|
||||
with patch("shutil.which", return_value = "/usr/bin/hipinfo"):
|
||||
with patch("subprocess.run", return_value = mock_result):
|
||||
result = stack_mod._detect_windows_gfx_arch()
|
||||
|
|
@ -2673,6 +2690,15 @@ class TestHipSdkEnvPathResolution:
|
|||
"""Verify that both install scripts resolve hipinfo/hipconfig via HIP_PATH
|
||||
and ROCM_PATH when the tools are not on $PATH, and emit explicit warnings."""
|
||||
|
||||
@staticmethod
|
||||
def _assert_accepts_partial_hipinfo_output(source: str):
|
||||
hipout_idx = source.find("$hipOut = & $hipinfoExe.Source")
|
||||
assert hipout_idx != -1
|
||||
hipinfo_block = source[hipout_idx : hipout_idx + 1600]
|
||||
assert 'if ($hipOut -match "(?i)gcnArchName")' in hipinfo_block
|
||||
assert "$LASTEXITCODE -eq 0 -and $hipOut -match" not in hipinfo_block
|
||||
assert "but reported gcnArchName" in hipinfo_block
|
||||
|
||||
# ── hipinfo resolution ────────────────────────────────────────────────────
|
||||
|
||||
def test_setup_checks_hip_path_for_hipinfo(self):
|
||||
|
|
@ -2747,6 +2773,16 @@ class TestHipSdkEnvPathResolution:
|
|||
source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
|
||||
assert "HIP runtime error" in source or "runtime error" in source.lower()
|
||||
|
||||
def test_setup_accepts_hipinfo_gcnarchname_on_nonzero_exit(self):
|
||||
"""setup.ps1 must accept partial hipinfo output from the #6043 crash path."""
|
||||
source = _SETUP_PS1_PATH.read_text(encoding = "utf-8")
|
||||
self._assert_accepts_partial_hipinfo_output(source)
|
||||
|
||||
def test_install_accepts_hipinfo_gcnarchname_on_nonzero_exit(self):
|
||||
"""install.ps1 must accept partial hipinfo output from the #6043 crash path."""
|
||||
source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8")
|
||||
self._assert_accepts_partial_hipinfo_output(source)
|
||||
|
||||
# ── hipconfig resolution ──────────────────────────────────────────────────
|
||||
|
||||
def test_setup_resolves_hipconfig_via_bin_subdir(self):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue