Harden ROCm detection, fix VRAM heuristic, and expand RDNA2 coverage
- Windows ROCm detection: validate actual GPU presence via hipinfo/amd-smi
output markers instead of just checking tool existence on PATH
- _ensure_rocm_torch: validate nvidia-smi actually reports a GPU before
giving NVIDIA precedence (fixes AMD-only hosts with stale NVIDIA tools)
- amd.py _parse_numeric: handle dict-shaped metric objects from newer
amd-smi versions ({"value": 10, "unit": "W"}) and strip MiB/GiB units
- amd.py VRAM heuristic: raise threshold from 100k to 10M to correctly
handle MI300X (192 GB = 196608 MB) and other high-VRAM GPUs
- amd.py visible GPU: use AMD-reported GPU IDs instead of enumerate index
so non-dense sets like CUDA_VISIBLE_DEVICES=1,3 report correctly
- install.sh: add ROCm <6.0 minimum version guard (no PyTorch wheels
exist for older versions); fix rocm7.1* glob to not match rocm7.10+
- is_rdna: add gfx1033-1036 for RDNA2 mobile GPUs (RX 6600M etc.)
- worker.py: increase ROCm source build timeout from 600s to 1800s;
fix success log message for ROCm source builds
- Tests: update mocks for _has_usable_nvidia_gpu, add RDNA2 target asserts
This commit is contained in:
parent
14823265d2
commit
134638dd0e
7 changed files with 108 additions and 41 deletions
|
|
@ -109,6 +109,24 @@ def _has_rocm_gpu() -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def _has_usable_nvidia_gpu() -> bool:
|
||||
"""Return True only when nvidia-smi exists AND reports at least one GPU."""
|
||||
exe = shutil.which("nvidia-smi")
|
||||
if not exe:
|
||||
return False
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[exe, "-L"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
return result.returncode == 0 and "GPU " in result.stdout
|
||||
|
||||
|
||||
def _ensure_rocm_torch() -> None:
|
||||
"""Reinstall torch with ROCm wheels when the venv received CPU-only torch.
|
||||
|
||||
|
|
@ -118,8 +136,8 @@ def _ensure_rocm_torch() -> None:
|
|||
(NVIDIA takes precedence).
|
||||
Uses pip_install() to respect uv, constraints, and --python targeting.
|
||||
"""
|
||||
# NVIDIA takes precedence on mixed hosts
|
||||
if shutil.which("nvidia-smi"):
|
||||
# NVIDIA takes precedence on mixed hosts -- but only if an actual GPU is usable
|
||||
if _has_usable_nvidia_gpu():
|
||||
return
|
||||
rocm_root = os.environ.get("ROCM_PATH") or "/opt/rocm"
|
||||
if not os.path.isdir(rocm_root) and not shutil.which("hipcc"):
|
||||
|
|
@ -707,7 +725,7 @@ def install_python_stack() -> int:
|
|||
|
||||
# Windows + AMD GPU: PyTorch does not publish ROCm wheels for Windows.
|
||||
# Detect and warn so users know manual steps are needed for GPU training.
|
||||
if IS_WINDOWS and not NO_TORCH:
|
||||
if IS_WINDOWS and not NO_TORCH and not _has_usable_nvidia_gpu():
|
||||
if shutil.which("hipinfo") or shutil.which("amd-smi"):
|
||||
_safe_print(
|
||||
_dim(" Note:"),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue