Harden ROCm support: probe error handling, version cap, validation
Address review findings from 8 independent reviewers: - Wrap _ensure_rocm_torch() torch probe in try/except for TimeoutExpired and OSError so a hung or broken torch import does not crash the installer (8/8 reviewers flagged this) - Add torch>=2.4,<2.11.0 version cap to the ROCm reinstall path to prevent installing unsupported torch 2.11.0 from the rocm7.1 index - Use with-statement for file reads in _detect_rocm_version() to avoid resource leaks - Handle ROCM_PATH="" correctly (use `or "/opt/rocm"` instead of default parameter to avoid relative path resolution) - Strengthen shell validation guard from rocm[0-9] to rocm[1-9] to reject rocm0.x tags that would produce nonexistent PyTorch index URLs - Switch shell version cap from blocklist to allowlist (rocm6.*|rocm7.0* |rocm7.1* pass through, everything else caps to rocm7.1) so future ROCm 10+ does not fall through to a nonexistent index - Add sorted() to _ROCM_TORCH_INDEX lookup for defensive ordering - Fix test_probe_timeout_handled: replace zero-assertion test with proper assertions verifying reinstall proceeds after timeout
This commit is contained in:
parent
7290199c26
commit
2cb0b52be7
3 changed files with 41 additions and 34 deletions
|
|
@ -47,13 +47,14 @@ _PYTORCH_WHL_BASE = "https://download.pytorch.org/whl"
|
|||
def _detect_rocm_version() -> tuple[int, int] | None:
|
||||
"""Return (major, minor) of the installed ROCm stack, or None."""
|
||||
# Check /opt/rocm/.info/version or ROCM_PATH equivalent
|
||||
rocm_root = os.environ.get("ROCM_PATH", "/opt/rocm")
|
||||
rocm_root = os.environ.get("ROCM_PATH") or "/opt/rocm"
|
||||
for path in (
|
||||
os.path.join(rocm_root, ".info", "version"),
|
||||
os.path.join(rocm_root, "lib", "rocm_version"),
|
||||
):
|
||||
try:
|
||||
parts = open(path).read().strip().split("-")[0].split(".")
|
||||
with open(path) as fh:
|
||||
parts = fh.read().strip().split("-")[0].split(".")
|
||||
return int(parts[0]), int(parts[1])
|
||||
except Exception:
|
||||
pass
|
||||
|
|
@ -86,7 +87,7 @@ def _ensure_rocm_torch() -> None:
|
|||
links against HIP (ROCm) or CUDA (NVIDIA). Skips on Windows/macOS.
|
||||
Uses pip_install() to respect uv, constraints, and --python targeting.
|
||||
"""
|
||||
rocm_root = os.environ.get("ROCM_PATH", "/opt/rocm")
|
||||
rocm_root = os.environ.get("ROCM_PATH") or "/opt/rocm"
|
||||
if not os.path.isdir(rocm_root) and not shutil.which("hipcc"):
|
||||
return # no ROCm toolchain
|
||||
|
||||
|
|
@ -96,22 +97,29 @@ def _ensure_rocm_torch() -> None:
|
|||
return
|
||||
|
||||
# Skip if torch is already GPU-enabled (HIP or CUDA)
|
||||
probe = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
"import torch; print(torch.version.hip or torch.version.cuda or '')",
|
||||
],
|
||||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.DEVNULL,
|
||||
timeout = 30,
|
||||
)
|
||||
if probe.returncode == 0 and probe.stdout.decode().strip():
|
||||
try:
|
||||
probe = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
"import torch; print(torch.version.hip or torch.version.cuda or '')",
|
||||
],
|
||||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.DEVNULL,
|
||||
timeout = 30,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
probe = None
|
||||
if probe is not None and probe.returncode == 0 and probe.stdout.decode().strip():
|
||||
return # torch already GPU-enabled
|
||||
|
||||
# Select best matching wheel tag (newest ROCm version <= installed)
|
||||
tag = next(
|
||||
(t for (maj, mn), t in _ROCM_TORCH_INDEX.items() if ver >= (maj, mn)),
|
||||
(
|
||||
t
|
||||
for (maj, mn), t in sorted(_ROCM_TORCH_INDEX.items(), reverse = True)
|
||||
if ver >= (maj, mn)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if tag is None:
|
||||
|
|
@ -124,7 +132,7 @@ def _ensure_rocm_torch() -> None:
|
|||
f"ROCm torch ({tag})",
|
||||
"--force-reinstall",
|
||||
"--no-cache-dir",
|
||||
"torch",
|
||||
"torch>=2.4,<2.11.0",
|
||||
"torchvision",
|
||||
"torchaudio",
|
||||
"--index-url",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue