diff --git a/studio/backend/tests/test_utils.py b/studio/backend/tests/test_utils.py index b34618cd4d..cc7d04ca0a 100644 --- a/studio/backend/tests/test_utils.py +++ b/studio/backend/tests/test_utils.py @@ -88,6 +88,19 @@ class TestGetDevice: ): assert _reset_and_detect() == DeviceType.CUDA + @needs_torch + def test_detect_survives_device0_probe_failure(self, capsys): + # is_available() True but the device-0 name probe raises: startup must + # still resolve CUDA rather than crash. + with ( + patch("utils.hardware.hardware._has_torch", return_value = True), + patch("torch.cuda.is_available", return_value = True), + patch("torch.cuda.device_count", return_value = 1), + patch("torch.cuda.get_device_properties", side_effect = RuntimeError("probe")), + ): + assert _reset_and_detect() == DeviceType.CUDA + assert "" in capsys.readouterr().out + @needs_mlx def test_returns_mlx_when_on_apple_silicon_with_mlx(self): with ( @@ -383,6 +396,22 @@ class TestPrintCudaDeviceList: _hw_module._print_cuda_device_list(is_rocm = False) assert capsys.readouterr().out == "" + @needs_torch + def test_rocm_label_omits_cuda_device_order(self, capsys): + # CUDA_DEVICE_ORDER governs CUDA only, so the ROCm listing must not claim it. + props = [MagicMock(), MagicMock()] + props[0].name = "AMD Instinct MI300X" + props[1].name = "AMD Instinct MI300X" + with ( + patch("torch.cuda.device_count", return_value = 2), + patch("torch.cuda.get_device_properties", side_effect = lambda i: props[i]), + ): + _hw_module._print_cuda_device_list(is_rocm = True) + out = capsys.readouterr().out + assert "ROCm devices (2):" in out + assert "CUDA_DEVICE_ORDER" not in out + assert "[0] AMD Instinct MI300X" in out + # ========== format_error_message() ========== diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py index db608c3f53..08a470f11f 100644 --- a/studio/backend/utils/hardware/hardware.py +++ b/studio/backend/utils/hardware/hardware.py @@ -107,12 +107,13 @@ def _has_mlx() -> bool: def _print_cuda_device_list(is_rocm: bool) -> None: - """Print every visible CUDA/ROCm GPU with its index. + """List every visible CUDA/ROCm GPU with its index at startup. The "Hardware detected" banner names only device 0, which hides the other - cards on a multi-GPU host and obscures which physical GPU each index maps to. - Listing all devices in the pinned PCI_BUS_ID order (see CUDA_DEVICE_ORDER at - module top) makes the available set explicit and matches `nvidia-smi -L`. + cards on a multi-GPU host. This lists the full visible set in CUDA-ordinal + order, matching `nvidia-smi -L` when no CUDA_VISIBLE_DEVICES mask is set + (under a mask the indices are visible ordinals, not physical PCI ids). + CUDA_DEVICE_ORDER governs only CUDA, so it is shown for CUDA but not ROCm. No-ops on single-GPU hosts and never raises -- it is purely informational. """ try: @@ -121,13 +122,17 @@ def _print_cuda_device_list(is_rocm: bool) -> None: count = torch.cuda.device_count() if count <= 1: return - label = "ROCm" if is_rocm else "CUDA" - order = os.environ.get("CUDA_DEVICE_ORDER", "default") - lines = [f"{label} devices ({count}, CUDA_DEVICE_ORDER={order}):"] + if is_rocm: + header = f"ROCm devices ({count}):" + else: + order = os.environ.get("CUDA_DEVICE_ORDER", "default") + header = f"CUDA devices ({count}, CUDA_DEVICE_ORDER={order}):" + lines = [header] for i in range(count): try: name = torch.cuda.get_device_properties(i).name - except Exception: + except Exception as e: + logger.debug("CUDA device %d property probe failed: %s", i, e) name = "" lines.append(f" [{i}] {name}") print("\n".join(lines)) @@ -156,7 +161,11 @@ def detect_hardware() -> DeviceType: if torch.cuda.is_available(): DEVICE = DeviceType.CUDA CHAT_ONLY = False - device_name = torch.cuda.get_device_properties(0).name + try: + device_name = torch.cuda.get_device_properties(0).name + except Exception as e: + logger.debug("CUDA device 0 property probe failed: %s", e) + device_name = "" # Distinguish ROCm from CUDA for display only (DeviceType stays CUDA). # AMD SDK wheels don't set torch.version.hip, so fall back to __version__.