From 0539621fa15969d9f3922665f8a7115d2d6856d2 Mon Sep 17 00:00:00 2001 From: LeoBorcherding Date: Tue, 5 May 2026 13:08:38 -0500 Subject: [PATCH 1/5] fix(studio): set HIP_VISIBLE_DEVICES in apply_gpu_ids for ROCm training workers Training workers are spawned via multiprocessing spawn before detect_hardware() runs, so IS_ROCM is still False. If the user never set HIP_VISIBLE_DEVICES in their shell, _inherits_rocm_visibility is also False, leaving the worker with only CUDA_VISIBLE_DEVICES set. On ROCm hosts the HIP runtime honors HIP_VISIBLE_DEVICES over CUDA_VISIBLE_DEVICES, so the worker saw the full device list and torch raised "no usable HIP accelerator" on some setups. Fall back to probing torch.version.hip (a build-time attribute, safe to read before GPU init) to detect ROCm when neither IS_ROCM nor inherited env vars are available. Mirrors the existing fix in llama_cpp.py for llama-server subprocess GPU pinning. Fixes https://github.com/unslothai/unsloth/issues/5180 --- studio/backend/utils/hardware/hardware.py | 15 +++++++++-- tests/studio/install/test_rocm_support.py | 33 +++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py index be31c00a78..7246a0519a 100644 --- a/studio/backend/utils/hardware/hardware.py +++ b/studio/backend/utils/hardware/hardware.py @@ -1391,14 +1391,25 @@ def apply_gpu_ids(gpu_ids) -> None: # parent process already set a ROCm visibility variable -- that # way a downstream ROCm process inherits the narrowed mask even # before Studio's hardware detection has classified the host. + # As a final fallback, probe torch.version.hip directly so spawned + # training workers on AMD hosts where the user never set HIP_VISIBLE_DEVICES + # still get the correct ROCm visibility mask (mirrors the llama_cpp.py + # approach for llama-server subprocess GPU pinning). _inherits_rocm_visibility = ( "HIP_VISIBLE_DEVICES" in os.environ or "ROCR_VISIBLE_DEVICES" in os.environ ) - if IS_ROCM or _inherits_rocm_visibility: + _is_rocm = IS_ROCM or _inherits_rocm_visibility + if not _is_rocm: + try: + import torch as _torch + _is_rocm = bool(getattr(_torch.version, "hip", None)) + except Exception: + pass + if _is_rocm: os.environ["HIP_VISIBLE_DEVICES"] = value os.environ["ROCR_VISIBLE_DEVICES"] = value _visible_gpu_count = None - if IS_ROCM or _inherits_rocm_visibility: + if _is_rocm: logger.info("Applied gpu_ids: CUDA_VISIBLE_DEVICES='%s' (rocm)", value) else: logger.info("Applied gpu_ids: CUDA_VISIBLE_DEVICES='%s'", value) diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py index 48831fd57b..9ce6e53998 100644 --- a/tests/studio/install/test_rocm_support.py +++ b/tests/studio/install/test_rocm_support.py @@ -1250,6 +1250,39 @@ class TestHardwareAmdBranching: assert "amd.get_physical_gpu_count" in func_body +# ============================================================================= +# TEST: hardware.py -- apply_gpu_ids ROCm fallback (issue #5180) +# ============================================================================= + + +class TestApplyGpuIdsRocmFallback: + """Verify apply_gpu_ids sets HIP_VISIBLE_DEVICES on ROCm hosts even when + IS_ROCM is still False (worker subprocess before detect_hardware runs).""" + + def test_apply_gpu_ids_source_checks_torch_version_hip(self): + """apply_gpu_ids should fall back to torch.version.hip when IS_ROCM is False.""" + hw_path = ( + PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py" + ) + source = hw_path.read_text() + func_start = source.find("def apply_gpu_ids") + func_body = source[func_start : source.find("\ndef ", func_start + 1)] + assert 'getattr(_torch.version, "hip", None)' in func_body or \ + "getattr(torch.version, 'hip', None)" in func_body or \ + "torch.version.hip" in func_body + + def test_apply_gpu_ids_source_sets_hip_visible_devices(self): + """apply_gpu_ids should set HIP_VISIBLE_DEVICES and ROCR_VISIBLE_DEVICES.""" + hw_path = ( + PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py" + ) + source = hw_path.read_text() + func_start = source.find("def apply_gpu_ids") + func_body = source[func_start : source.find("\ndef ", func_start + 1)] + assert "HIP_VISIBLE_DEVICES" in func_body + assert "ROCR_VISIBLE_DEVICES" in func_body + + # ============================================================================= # TEST: install_python_stack.py -- Windows AMD warning # ============================================================================= From 14fccdee48bac943b8f12ec504068e5836a1193a Mon Sep 17 00:00:00 2001 From: LeoBorcherding Date: Tue, 5 May 2026 13:30:56 -0500 Subject: [PATCH 2/5] test: tighten apply_gpu_ids ROCm fallback assertions Replace loose OR chain with exact string matches, split into three focused tests, and add a guard check for the try/except wrapper. --- tests/studio/install/test_rocm_support.py | 27 +++++++++++++++-------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py index 9ce6e53998..99bc9c11bc 100644 --- a/tests/studio/install/test_rocm_support.py +++ b/tests/studio/install/test_rocm_support.py @@ -1259,28 +1259,37 @@ class TestApplyGpuIdsRocmFallback: """Verify apply_gpu_ids sets HIP_VISIBLE_DEVICES on ROCm hosts even when IS_ROCM is still False (worker subprocess before detect_hardware runs).""" - def test_apply_gpu_ids_source_checks_torch_version_hip(self): - """apply_gpu_ids should fall back to torch.version.hip when IS_ROCM is False.""" + def test_apply_gpu_ids_falls_back_to_torch_version_hip(self): + """apply_gpu_ids should probe torch.version.hip when IS_ROCM is False and no ROCm env vars are set.""" hw_path = ( PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py" ) source = hw_path.read_text() func_start = source.find("def apply_gpu_ids") func_body = source[func_start : source.find("\ndef ", func_start + 1)] - assert 'getattr(_torch.version, "hip", None)' in func_body or \ - "getattr(torch.version, 'hip', None)" in func_body or \ - "torch.version.hip" in func_body + assert 'getattr(_torch.version, "hip", None)' in func_body - def test_apply_gpu_ids_source_sets_hip_visible_devices(self): - """apply_gpu_ids should set HIP_VISIBLE_DEVICES and ROCR_VISIBLE_DEVICES.""" + def test_apply_gpu_ids_sets_hip_and_rocr_visible_devices(self): + """apply_gpu_ids should set both HIP_VISIBLE_DEVICES and ROCR_VISIBLE_DEVICES on ROCm.""" hw_path = ( PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py" ) source = hw_path.read_text() func_start = source.find("def apply_gpu_ids") func_body = source[func_start : source.find("\ndef ", func_start + 1)] - assert "HIP_VISIBLE_DEVICES" in func_body - assert "ROCR_VISIBLE_DEVICES" in func_body + assert 'os.environ["HIP_VISIBLE_DEVICES"] = value' in func_body + assert 'os.environ["ROCR_VISIBLE_DEVICES"] = value' in func_body + + def test_apply_gpu_ids_rocm_fallback_is_guarded_by_try_except(self): + """torch import in apply_gpu_ids must be wrapped in try/except so a missing torch never crashes.""" + hw_path = ( + PACKAGE_ROOT / "studio" / "backend" / "utils" / "hardware" / "hardware.py" + ) + source = hw_path.read_text() + func_start = source.find("def apply_gpu_ids") + func_body = source[func_start : source.find("\ndef ", func_start + 1)] + assert "import torch as _torch" in func_body + assert "except Exception" in func_body # ============================================================================= From e87c90f769589f6556a1f24475fc58d2661ab031 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 6 May 2026 04:34:52 +0000 Subject: [PATCH 3/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/utils/hardware/hardware.py | 1 + 1 file changed, 1 insertion(+) diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py index 7246a0519a..3b1b9fe95e 100644 --- a/studio/backend/utils/hardware/hardware.py +++ b/studio/backend/utils/hardware/hardware.py @@ -1402,6 +1402,7 @@ def apply_gpu_ids(gpu_ids) -> None: if not _is_rocm: try: import torch as _torch + _is_rocm = bool(getattr(_torch.version, "hip", None)) except Exception: pass From cb0edfc56c4bc653f557639872dbf73da85e32fc Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 6 May 2026 12:29:20 +0000 Subject: [PATCH 4/5] Use 'is not None' and log debug on torch.version.hip probe failures Two small follow-ups to the apply_gpu_ids ROCm fallback: 1. Match detect_hardware()'s 'getattr(torch.version, "hip", None) is not None' form so the entire codebase has one canonical 'this torch was built with HIP' check. On every shipping torch wheel hip is either None or a non-empty version string, so the new form agrees with the old bool() form on every real install. 2. Log the probe failure at debug level instead of swallowing it silently. The broad 'except Exception' is intentional (we never want apply_gpu_ids to crash a worker over a probe), but the silent pass made it impossible to tell whether the fallback was firing or being skipped. --- studio/backend/utils/hardware/hardware.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py index 3b1b9fe95e..1cb985539a 100644 --- a/studio/backend/utils/hardware/hardware.py +++ b/studio/backend/utils/hardware/hardware.py @@ -1400,12 +1400,24 @@ def apply_gpu_ids(gpu_ids) -> None: ) _is_rocm = IS_ROCM or _inherits_rocm_visibility if not _is_rocm: + # Use ``is not None`` here to match the detect_hardware() check at + # module top -- torch ships HIP version as a non-empty string on + # ROCm builds and None on CUDA builds, so the two forms agree on + # every shipping torch wheel; the ``is not None`` form is the one + # the rest of the codebase reads for "this torch was built with + # HIP". Keep the broad ``except`` as a safety net (we never want + # apply_gpu_ids to crash a worker over a probe failure) but log at + # debug level so the skip is observable when needed. try: import torch as _torch - _is_rocm = bool(getattr(_torch.version, "hip", None)) - except Exception: - pass + _is_rocm = getattr(_torch.version, "hip", None) is not None + except Exception as e: + logger.debug( + "apply_gpu_ids: torch.version.hip probe skipped (%s: %s)", + type(e).__name__, + e, + ) if _is_rocm: os.environ["HIP_VISIBLE_DEVICES"] = value os.environ["ROCR_VISIBLE_DEVICES"] = value From 9a83a74bbfdef5233ce1a8850b1709aa2cea6929 Mon Sep 17 00:00:00 2001 From: LeoBorcherding Date: Wed, 6 May 2026 12:49:19 -0500 Subject: [PATCH 5/5] fix(studio): honour HIP_VISIBLE_DEVICES in _get_parent_visible_gpu_spec before IS_ROCM is set When a user has HIP_VISIBLE_DEVICES set in their shell (e.g. "1" to select GPU 1) but detect_hardware() has not yet run in the Studio parent process, IS_ROCM is still False. _get_parent_visible_gpu_spec() was gated on IS_ROCM so it fell through to CUDA_VISIBLE_DEVICES (unset), saw all physical GPUs, and auto-selected index 0. apply_gpu_ids then overwrote HIP_VISIBLE_DEVICES with "0", making the intended GPU invisible to ROCm torch in the worker, which triggered the "no usable HIP accelerator" error (issue #5180). Apply the same _inherits_rocm_visibility pattern already used in apply_gpu_ids: check for HIP_VISIBLE_DEVICES / ROCR_VISIBLE_DEVICES in the environment regardless of IS_ROCM so the correct GPU index is preserved. --- studio/backend/utils/hardware/hardware.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py index 1cb985539a..487ab1e850 100644 --- a/studio/backend/utils/hardware/hardware.py +++ b/studio/backend/utils/hardware/hardware.py @@ -599,7 +599,10 @@ def _get_parent_visible_gpu_spec() -> Dict[str, Any]: # Use explicit None checks (not `or`) so empty string "" is honoured # as "no visible GPUs" rather than falling through to CUDA_VISIBLE_DEVICES. cuda_visible = None - if IS_ROCM: + _is_rocm_spec = IS_ROCM or ( + "HIP_VISIBLE_DEVICES" in os.environ or "ROCR_VISIBLE_DEVICES" in os.environ + ) + if _is_rocm_spec: hip_vis = os.environ.get("HIP_VISIBLE_DEVICES") rocr_vis = os.environ.get("ROCR_VISIBLE_DEVICES") if hip_vis is not None: