diff --git a/install.ps1 b/install.ps1 index dccd8e4fc1..1073ab5726 100644 --- a/install.ps1 +++ b/install.ps1 @@ -1553,7 +1553,9 @@ shell.Run cmd, 0, False $HipSdkInstalled = $true # binary found → SDK is installed regardless of device state try { $hipOut = & $hipinfoExe.Source 2>&1 | Out-String - if ($LASTEXITCODE -eq 0 -and $hipOut -match "(?i)gcnArchName") { + if ($hipOut -match "(?i)gcnArchName") { + # hipinfo can crash after printing gcnArchName (#6043). + # Once the arch is printed, keep the ROCm wheel path. $HasROCm = $true $_hipAllArches = @([regex]::Matches($hipOut, "(?im)^\s*gcnArchName\s*:\s*(\S+)") | ForEach-Object { ($_.Groups[1].Value -split ':')[0].Trim().ToLower() }) $_hipVisIdx = if ($env:HIP_VISIBLE_DEVICES -match '^\d') { [int]($env:HIP_VISIBLE_DEVICES -split ',')[0] } elseif ($env:ROCR_VISIBLE_DEVICES -match '^\d') { [int]($env:ROCR_VISIBLE_DEVICES -split ',')[0] } else { 0 } @@ -1563,8 +1565,13 @@ shell.Run cmd, 0, False } else { $ROCmGpuLabel = "AMD ROCm" } + if ($LASTEXITCODE -ne 0) { + Write-Host " [INFO] hipinfo exited with code $LASTEXITCODE but reported gcnArchName -- treating as ROCm-capable (see #6043)" -ForegroundColor Cyan + } } elseif ($LASTEXITCODE -ne 0) { - # hipinfo ran but returned a HIP runtime error (e.g. "no ROCm-capable device detected") + # hipinfo ran but returned a HIP runtime error without any gcnArchName + # output (e.g. "no ROCm-capable device detected"), or crashed before + # printing device info. $firstLine = ($hipOut -split '\r?\n' | Where-Object { $_.Trim() } | Select-Object -First 1) Write-Host " [WARN] hipinfo returned a HIP runtime error (exit $LASTEXITCODE)" -ForegroundColor Yellow Write-Host " $firstLine" -ForegroundColor Yellow diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py index d404041ca7..a8bd73ad9b 100644 --- a/studio/install_python_stack.py +++ b/studio/install_python_stack.py @@ -354,16 +354,20 @@ def _detect_windows_gfx_arch() -> str | None: stderr = subprocess.DEVNULL, timeout = 10, ) - if result.returncode == 0: - text = result.stdout.decode(errors = "replace") - # findall gets every gcnArchName line so multi-GPU hosts are - # enumerable and HIP_VISIBLE_DEVICES selects correctly. - _tokens = [ - t.strip().lower() for t in re.findall(r"(?im)^\s*gcnArchName\s*:\s*(\S+)", text) - ] - _pick = _dedup_pick(_tokens) - if _pick: - return _pick + # Accept partial output even when hipinfo crashes (e.g. exit code + # 0xC0000005 / STATUS_ACCESS_VIOLATION on some RDNA 4 hosts): if + # gcnArchName is present in stdout the device was enumerated before + # the crash, so the arch is trustworthy. Ignoring it causes a + # silent CPU PyTorch fallback (issue #6043). + text = result.stdout.decode(errors = "replace") + # findall gets every gcnArchName line so multi-GPU hosts are + # enumerable and HIP_VISIBLE_DEVICES selects correctly. + _tokens = [ + t.strip().lower() for t in re.findall(r"(?im)^\s*gcnArchName\s*:\s*(\S+)", text) + ] + _pick = _dedup_pick(_tokens) + if _pick: + return _pick except Exception: pass diff --git a/studio/setup.ps1 b/studio/setup.ps1 index acc7e92875..f02ed586be 100644 --- a/studio/setup.ps1 +++ b/studio/setup.ps1 @@ -823,7 +823,9 @@ if (-not $HasNvidiaSmi) { $HipSdkInstalled = $true # binary found → SDK is installed regardless of device state try { $hipOut = & $hipinfoExe.Source 2>&1 | Out-String - if ($LASTEXITCODE -eq 0 -and $hipOut -match "(?i)gcnArchName") { + if ($hipOut -match "(?i)gcnArchName") { + # hipinfo can crash after printing gcnArchName (#6043). + # Once the arch is printed, keep the ROCm wheel path. $HasROCm = $true $_hipAllArches = @([regex]::Matches($hipOut, "(?im)^\s*gcnArchName\s*:\s*(\S+)") | ForEach-Object { ($_.Groups[1].Value -split ':')[0].Trim().ToLower() }) $_hipVisIdx = if ($env:HIP_VISIBLE_DEVICES -match '^\d') { [int]($env:HIP_VISIBLE_DEVICES -split ',')[0] } elseif ($env:ROCR_VISIBLE_DEVICES -match '^\d') { [int]($env:ROCR_VISIBLE_DEVICES -split ',')[0] } else { 0 } @@ -833,8 +835,13 @@ if (-not $HasNvidiaSmi) { } else { $ROCmGpuLabel = "AMD ROCm" } + if ($LASTEXITCODE -ne 0) { + substep "[INFO] hipinfo exited with code $LASTEXITCODE but reported gcnArchName -- treating as ROCm-capable (see #6043)" "Cyan" + } } elseif ($LASTEXITCODE -ne 0) { - # hipinfo ran but returned a HIP runtime error (e.g. "no ROCm-capable device detected") + # hipinfo ran but returned a HIP runtime error without any gcnArchName + # output (e.g. "no ROCm-capable device detected"), or crashed before + # printing device info. $firstLine = ($hipOut -split '\r?\n' | Where-Object { $_.Trim() } | Select-Object -First 1) substep "[WARN] hipinfo returned a HIP runtime error (exit $LASTEXITCODE)" "Yellow" substep " $firstLine" "Yellow" diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py index a3c9555d0b..b0abaaeacd 100644 --- a/tests/studio/install/test_rocm_support.py +++ b/tests/studio/install/test_rocm_support.py @@ -1820,10 +1820,27 @@ class TestDetectWindowsGfxArch: result = stack_mod._detect_windows_gfx_arch() assert result == "gfx1200" - def test_returns_none_on_nonzero_returncode(self): + def test_returns_arch_on_crash_with_gcnarchname_in_output(self): + # Regression test for issue #6043: hipinfo may exit with a non-zero + # code (e.g. 0xC0000005 / STATUS_ACCESS_VIOLATION on RDNA 4 hosts) + # while still printing the gcnArchName line before crashing. The + # previous guard `if result.returncode == 0` discarded this output, + # causing a CPU PyTorch fallback. The fix: accept the arch whenever + # gcnArchName is present in stdout regardless of exit code. + mock_result = MagicMock() + mock_result.returncode = -1073741819 # 0xC0000005 STATUS_ACCESS_VIOLATION + mock_result.stdout = b"gcnArchName : gfx1200\nsome other line\n" + with patch("shutil.which", return_value = "/usr/bin/hipinfo"): + with patch("subprocess.run", return_value = mock_result): + result = stack_mod._detect_windows_gfx_arch() + assert result == "gfx1200" + + def test_returns_none_on_nonzero_returncode_without_gcnarchname(self): + # Non-zero exit without any gcnArchName output (e.g. no device detected) + # must still return None and fall through to amd-smi / WMI. mock_result = MagicMock() mock_result.returncode = 1 - mock_result.stdout = b"gcnArchName : gfx1200\n" + mock_result.stdout = b"HIP runtime error: no device detected\n" with patch("shutil.which", return_value = "/usr/bin/hipinfo"): with patch("subprocess.run", return_value = mock_result): result = stack_mod._detect_windows_gfx_arch() @@ -2673,6 +2690,15 @@ class TestHipSdkEnvPathResolution: """Verify that both install scripts resolve hipinfo/hipconfig via HIP_PATH and ROCM_PATH when the tools are not on $PATH, and emit explicit warnings.""" + @staticmethod + def _assert_accepts_partial_hipinfo_output(source: str): + hipout_idx = source.find("$hipOut = & $hipinfoExe.Source") + assert hipout_idx != -1 + hipinfo_block = source[hipout_idx : hipout_idx + 1600] + assert 'if ($hipOut -match "(?i)gcnArchName")' in hipinfo_block + assert "$LASTEXITCODE -eq 0 -and $hipOut -match" not in hipinfo_block + assert "but reported gcnArchName" in hipinfo_block + # ── hipinfo resolution ──────────────────────────────────────────────────── def test_setup_checks_hip_path_for_hipinfo(self): @@ -2747,6 +2773,16 @@ class TestHipSdkEnvPathResolution: source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8") assert "HIP runtime error" in source or "runtime error" in source.lower() + def test_setup_accepts_hipinfo_gcnarchname_on_nonzero_exit(self): + """setup.ps1 must accept partial hipinfo output from the #6043 crash path.""" + source = _SETUP_PS1_PATH.read_text(encoding = "utf-8") + self._assert_accepts_partial_hipinfo_output(source) + + def test_install_accepts_hipinfo_gcnarchname_on_nonzero_exit(self): + """install.ps1 must accept partial hipinfo output from the #6043 crash path.""" + source = _INSTALL_PS1_PATH.read_text(encoding = "utf-8") + self._assert_accepts_partial_hipinfo_output(source) + # ── hipconfig resolution ────────────────────────────────────────────────── def test_setup_resolves_hipconfig_via_bin_subdir(self):