diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py index 7292fa185f..893d0364f7 100644 --- a/studio/backend/utils/hardware/hardware.py +++ b/studio/backend/utils/hardware/hardware.py @@ -1818,7 +1818,13 @@ def apply_gpu_ids(gpu_ids) -> None: ) if _is_rocm: os.environ["HIP_VISIBLE_DEVICES"] = value - os.environ["ROCR_VISIBLE_DEVICES"] = value + # ROCR_VISIBLE_DEVICES operates at the HSA agent level and uses + # different indexing semantics to HIP_VISIBLE_DEVICES. Setting it + # to a physical GPU index breaks multi-GPU ROCm systems where the + # parent already set ROCR_VISIBLE_DEVICES (e.g. "0,1"): narrowing + # to "1" causes torch.cuda.is_available() to return False in the + # worker subprocess. HIP_VISIBLE_DEVICES is sufficient for GPU + # selection on ROCm -- leave ROCR_VISIBLE_DEVICES inherited. _visible_gpu_count = None if _is_rocm: logger.info("Applied gpu_ids: CUDA_VISIBLE_DEVICES='%s' (rocm)", value) diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py index 7d1ae535fd..9aa06b6ff7 100644 --- a/tests/studio/install/test_rocm_support.py +++ b/tests/studio/install/test_rocm_support.py @@ -1614,14 +1614,18 @@ class TestApplyGpuIdsRocmFallback: func_body = source[func_start : source.find("\ndef ", func_start + 1)] assert 'getattr(_torch.version, "hip", None)' in func_body - def test_apply_gpu_ids_sets_hip_and_rocr_visible_devices(self): - """apply_gpu_ids should set both HIP_VISIBLE_DEVICES and ROCR_VISIBLE_DEVICES on ROCm.""" + def test_apply_gpu_ids_sets_hip_but_not_rocr_visible_devices(self): + """apply_gpu_ids should set HIP_VISIBLE_DEVICES but leave ROCR_VISIBLE_DEVICES inherited. + + ROCR_VISIBLE_DEVICES uses HSA agent-level indexing, not physical GPU indices. + Overwriting it breaks multi-GPU ROCm systems (see issue #6118). + """ hw_path = PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py" source = hw_path.read_text(encoding = "utf-8") func_start = source.find("def apply_gpu_ids") func_body = source[func_start : source.find("\ndef ", func_start + 1)] assert 'os.environ["HIP_VISIBLE_DEVICES"] = value' in func_body - assert 'os.environ["ROCR_VISIBLE_DEVICES"] = value' in func_body + assert 'os.environ["ROCR_VISIBLE_DEVICES"] = value' not in func_body def test_apply_gpu_ids_rocm_fallback_is_guarded_by_try_except(self): """torch import in apply_gpu_ids must be wrapped in try/except so a missing torch never crashes."""