From c267895538172d42ce9bc672484adafe487b4e5a Mon Sep 17 00:00:00 2001 From: Guerriero Riccardo <40391857+guerrieroriccardo@users.noreply.github.com> Date: Thu, 23 Jul 2026 03:16:25 +0200 Subject: [PATCH] Studio: mask AMD GPU pins via ROCR so an unsupported iGPU can't crash llama-server (#7272) * Studio: mask AMD GPU pins via ROCR so an unsupported iGPU can't crash llama-server On a mixed AMD host (e.g. a discrete gfx1102 GPU next to a gfx1103 iGPU) the bundled rocm-gfx110X llama.cpp build segfaults during HSA device enumeration on the unsupported iGPU -- before llama-server prints a line, so every model load fails with a bare signal and empty logs. The GPU-subset pin masked visibility with HIP_VISIBLE_DEVICES, but HIP filtering runs only after the HSA runtime has already enumerated (and crashed on) every agent. Mask the subset via ROCR_VISIBLE_DEVICES (the ROCr/HSA layer) instead, so a deselected/unsupported GPU is never enumerated. Exactly one layer is masked (HIP cleared) to avoid the double-mask reindex that would otherwise drop the child to CPU. The whole-set tensor-split path and the CPU-only sentinel keep their existing HIP behavior. Also stop misreporting the resulting startup segfault as a vision projector incompatibility: when the text-only mmproj retry also hard- crashes with a signal, surface a GPU/driver init crash (with the ROCR hint) instead of blaming the projector. Co-Authored-By: Claude Opus 4.8 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Tighten _emit_child_gpu_visibility comments for #7272 Comment and docstring only: condense the ROCR-vs-HIP masking rationale from ~22 to ~14 lines and the call-site note from 5 to 3, keeping every technical point (HSA enumeration segfault, physical ids, the -1 sentinel). Logic is unchanged, verified by an AST compare with docstrings stripped and by exercising _emit_child_gpu_visibility against a torch/HIP stub. * Detect AMD SDK ROCm wheels (hip=None) in _emit_child_gpu_visibility (Codex P2) The ROCm branch gated only on torch.version.hip, but AMD SDK wheels leave that unset while encoding 'rocm' in __version__ (detect_hardware handles this the same way). On such a wheel the masking was skipped entirely, leaving only CUDA_VISIBLE_DEVICES, so on a mixed AMD box the unsupported deselected iGPU still enumerated and could crash llama-server. Now the branch also treats 'rocm' in torch.__version__ as ROCm, mirroring detect_hardware. Adds tests for the hip=None SDK wheel (ROCR + default paths) and a CUDA guard so the version-string check can't false-positive. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Remap CUDA_VISIBLE_DEVICES to post-ROCR ordinals on prefer_rocr for PR #7272 (Codex P1) On the prefer_rocr path _emit_child_gpu_visibility set ROCR_VISIBLE_DEVICES to the physical id and cleared HIP_VISIBLE_DEVICES, but left CUDA_VISIBLE_DEVICES at the physical id. ROCR re-indexes the visible agents from 0 and, with HIP cleared, HIP honours CUDA_VISIBLE_DEVICES -- so a non-zero pick (e.g. GPU 1) pointed out of range, HIP saw 0 devices, and the child fell back to CPU, defeating GPU-picker selections other than physical GPU 0. Remap CUDA to the post-ROCR ordinals (0..N-1); GPU 0 is unchanged, the default (HIP) path and the CPU sentinel are untouched, and non-AMD wheels never enter this branch. * Detect AMD SDK wheels in _resolve_visible_physical_ids for PR #7272 (Codex P2) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Keep the HIP mask on Windows ROCm in prefer_rocr for PR #7272 (Codex P2) * Ignore ROCR_VISIBLE_DEVICES in _resolve_visible_physical_ids on Windows for PR #7272 (Codex P2) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Preserve inherited ROCR masks in the tensor-split pin for PR #7272 (Codex P2) --------- Co-authored-by: Claude Opus 4.8 Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Daniel Han Co-authored-by: Leo Borcherding --- studio/backend/core/inference/llama_cpp.py | 108 ++++++++-- studio/backend/tests/test_gpu_memory_mode.py | 205 ++++++++++++++++++- 2 files changed, 291 insertions(+), 22 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 8651ed9ea8..1c9c76ebe9 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -2912,12 +2912,25 @@ class LlamaCppBackend: on the ordinal->physical mapping.""" try: import torch - is_rocm = getattr(torch.version, "hip", None) is not None + + # Same ROCm detection as _emit_child_gpu_visibility: AMD SDK wheels + # leave version.hip unset but encode "rocm" in __version__. The two + # must agree, else an inherited ROCR mask reads back as "no mask", + # ordinal 0 is labelled physical 0, and the child's new ROCR pin + # re-exposes the GPU the inherited mask was hiding. + is_rocm = ( + getattr(torch.version, "hip", None) is not None + or "rocm" in getattr(torch, "__version__", "").lower() + ) except Exception: is_rocm = False if is_rocm: hip_v = os.environ.get("HIP_VISIBLE_DEVICES") - rocr_v = os.environ.get("ROCR_VISIBLE_DEVICES") + # ROCR_VISIBLE_DEVICES is a Linux ROCr variable; Windows HIP has no + # ROCr layer, so a stray ROCR var there does not mask the runtime and + # must not be read as the ordinal->physical mapping (mirrors the + # Windows gate in _emit_child_gpu_visibility). + rocr_v = None if sys.platform == "win32" else os.environ.get("ROCR_VISIBLE_DEVICES") cvd = ( hip_v if hip_v is not None @@ -2935,20 +2948,52 @@ class LlamaCppBackend: return None @staticmethod - def _emit_child_gpu_visibility(env: dict, pinned: str) -> None: - """Write the child's GPU visibility mask (CUDA, plus the HIP mirror on - ROCm, where narrowing only CUDA_VISIBLE_DEVICES leaves an AMD child - seeing the full set). Do NOT also set ROCR_VISIBLE_DEVICES: ROCR and HIP - mask at different layers, so the same indices apply twice -- ROCR reduces - and re-indexes from 0, then a non-zero HIP pin points out of range, HIP - enumerates 0 devices, and llama.cpp falls back to CPU. The HIP mask alone - narrows correctly; clear any inherited ROCR mask so it can't double up.""" + def _emit_child_gpu_visibility( + env: dict, + pinned: str, + *, + prefer_rocr: bool = False, + ) -> None: + """Write the child's GPU visibility mask: CUDA, plus a ROCm mirror on AMD + (masking only CUDA_VISIBLE_DEVICES leaves an AMD child seeing every GPU). + + Default: HIP_VISIBLE_DEVICES, clearing any inherited ROCR mask so the two + can't stack (ROCR re-indexes from 0, then a non-zero HIP pin points out of + range, HIP sees 0 devices, and llama.cpp falls back to CPU). + + prefer_rocr masks at the ROCr/HSA layer instead (clearing HIP). A HIP mask + filters only AFTER the HSA runtime enumerates every agent, and that + enumeration segfaults at startup on a GPU the build has no kernels for + (e.g. a gfx1103 iGPU under a gfx110X prebuilt), before llama-server logs a + line. ROCR drops the device at the driver layer, consuming physical ids. + The CPU-only sentinel ("-1") has no portable ROCR spelling, so it keeps + the HIP mask. Windows keeps the HIP mask too: ROCR_VISIBLE_DEVICES is a + Linux ROCr variable (Windows HIP has no ROCr layer), so the ROCR pin + would be dead there while the cleared HIP mask stops selecting.""" env["CUDA_VISIBLE_DEVICES"] = pinned try: import torch as _torch - if getattr(_torch.version, "hip", None) is not None: - env["HIP_VISIBLE_DEVICES"] = pinned - env.pop("ROCR_VISIBLE_DEVICES", None) + + # torch.version.hip is set on ROCm, None on CUDA; AMD SDK wheels may + # leave it unset but encode "rocm" in __version__ (mirrors detect_hardware). + if ( + getattr(_torch.version, "hip", None) is not None + or "rocm" in getattr(_torch, "__version__", "").lower() + ): + if prefer_rocr and pinned != "-1" and sys.platform != "win32": + env["ROCR_VISIBLE_DEVICES"] = pinned + env.pop("HIP_VISIBLE_DEVICES", None) + # ROCR re-indexes the visible agents from 0, and with HIP + # cleared HIP honours CUDA_VISIBLE_DEVICES -- so it must carry + # the post-ROCR ordinals (0..N-1), not the physical ids, else a + # non-zero pick points out of range and HIP sees 0 devices (the + # same stacking the default path avoids by clearing ROCR). + env["CUDA_VISIBLE_DEVICES"] = ",".join( + str(i) for i in range(len(pinned.split(","))) + ) + else: + env["HIP_VISIBLE_DEVICES"] = pinned + env.pop("ROCR_VISIBLE_DEVICES", None) except Exception as e: logger.debug("Failed to set ROCm visibility env vars for child: %s", e) @@ -2983,7 +3028,21 @@ class LlamaCppBackend: logger.debug("Could not read reported GPU order for split pin: %s", e) if order is None: order = sorted(inherited) - LlamaCppBackend._emit_child_gpu_visibility(env, ",".join(str(i) for i in order)) + # Re-emit at the layer that produced the mapping. A parent masked only + # via ROCR_VISIBLE_DEVICES hides agents at the driver layer, and the + # default HIP re-emission clears that mask -- HSA then enumerates every + # agent again and can segfault at startup on an unsupported GPU the + # parent was hiding (the crash prefer_rocr exists to avoid). Linux-only, + # mirroring _resolve_visible_physical_ids: on Windows a stray ROCR var + # is dead and was not the mapping's source. + prefer_rocr = ( + sys.platform != "win32" + and env.get("HIP_VISIBLE_DEVICES") is None + and env.get("ROCR_VISIBLE_DEVICES") is not None + ) + LlamaCppBackend._emit_child_gpu_visibility( + env, ",".join(str(i) for i in order), prefer_rocr = prefer_rocr + ) @staticmethod def _amd_apu_wants_unified_memory(gpu_indices = None) -> bool: @@ -7740,7 +7799,12 @@ class LlamaCppBackend: # default FASTEST_FIRST order (#5025). if gpu_ids: env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" - self._emit_child_gpu_visibility(env, ",".join(str(i) for i in gpu_indices)) + # Mask on AMD at the ROCr/HSA layer: HIP-only masking still + # enumerates every agent first, which segfaults on a deselected + # unsupported GPU (e.g. gfx1103 iGPU under a gfx110X prebuilt). + self._emit_child_gpu_visibility( + env, ",".join(str(i) for i in gpu_indices), prefer_rocr = True + ) elif manual_tensor_split_emitted and not is_vulkan_backend: # A manual per-GPU ratio across ALL GPUs (no explicit pick, so # no CUDA_VISIBLE_DEVICES mask above): the UI built the @@ -8102,6 +8166,20 @@ class LlamaCppBackend: # an OS-killed text-only retry still gets the OOM message. _retry_rc = self._process.poll() if self._process is not None else None self._kill_process() + # If the text-only retry ALSO hard-crashed (a signal, not + # OOM/timeout), the vision projector was never the cause: + # llama-server is faulting during GPU/driver init. Say so + # -- with the ROCm fix -- instead of blaming the mmproj. + if self._is_signal_crash(_retry_rc): + raise RuntimeError( + "llama-server crashed at startup on both the vision " + "and text-only attempts -- a GPU driver/runtime " + "initialization crash, not a model or vision-projector " + "problem. This often means an unsupported secondary " + "GPU; on AMD/ROCm, hide it with ROCR_VISIBLE_DEVICES " + "(e.g. ROCR_VISIBLE_DEVICES=0 exposes only the first " + "GPU) before launching Unsloth Studio." + ) raise RuntimeError( "Vision projector incompatible with this llama.cpp " "build, and the text-only retry also failed: " diff --git a/studio/backend/tests/test_gpu_memory_mode.py b/studio/backend/tests/test_gpu_memory_mode.py index b17274197f..19ba9e3e05 100644 --- a/studio/backend/tests/test_gpu_memory_mode.py +++ b/studio/backend/tests/test_gpu_memory_mode.py @@ -733,20 +733,211 @@ def test_split_pin_without_mask_only_sets_pci_order(monkeypatch): def test_split_pin_mirrors_hip_mask_on_rocm(monkeypatch): - # ROCm: the pin must land in HIP_VISIBLE_DEVICES too, and an inherited ROCR - # mask is cleared so the mask can't apply twice (ROCR re-indexes, then HIP - # would index into the already-reduced set). + # ROCm with the mask sourced from HIP: the pin must land in + # HIP_VISIBLE_DEVICES too, and an inherited ROCR mask is cleared so the + # mask can't apply twice (ROCR re-indexes, then HIP would index into the + # already-reduced set). _patch_split_pin_env(monkeypatch, inherited = [3, 1], reported = [1, 3]) - torch_stub = _types.ModuleType("torch") - torch_stub.version = _types.SimpleNamespace(hip = "6.0") - monkeypatch.setitem(sys.modules, "torch", torch_stub) - env = {"CUDA_VISIBLE_DEVICES": "3,1", "ROCR_VISIBLE_DEVICES": "3,1"} + _rocm_torch_stub(monkeypatch) + env = { + "CUDA_VISIBLE_DEVICES": "3,1", + "HIP_VISIBLE_DEVICES": "3,1", + "ROCR_VISIBLE_DEVICES": "3,1", + } LlamaCppBackend._pin_visible_gpu_order_for_split(env) assert env["CUDA_VISIBLE_DEVICES"] == "1,3" assert env["HIP_VISIBLE_DEVICES"] == "1,3" assert "ROCR_VISIBLE_DEVICES" not in env +def test_split_pin_preserves_inherited_rocr_mask(monkeypatch): + # Mask sourced from ROCR alone (e.g. an AMD SDK parent): the pin must + # re-emit at the ROCr layer, not swap to HIP -- clearing ROCR re-exposes + # every agent to HSA enumeration, which can segfault at startup on an + # unsupported GPU the parent mask was hiding (#7272 review). CUDA carries + # the post-ROCR ordinals, mirroring the prefer_rocr emission. + _patch_split_pin_env(monkeypatch, inherited = [3, 1], reported = [1, 3]) + _rocm_torch_stub(monkeypatch) + env = {"ROCR_VISIBLE_DEVICES": "3,1"} + LlamaCppBackend._pin_visible_gpu_order_for_split(env) + assert env["ROCR_VISIBLE_DEVICES"] == "1,3" + assert env["CUDA_VISIBLE_DEVICES"] == "0,1" + assert "HIP_VISIBLE_DEVICES" not in env + + +def test_split_pin_keeps_hip_on_windows_despite_stray_rocr(monkeypatch): + # On Windows the ROCR var is dead (no ROCr layer) and the resolver never + # reads it, so a stray value must not flip the pin to the ROCR emission: + # the HIP mask is the only effective selector there. + _patch_split_pin_env(monkeypatch, inherited = [3, 1], reported = [1, 3]) + torch_stub = _types.ModuleType("torch") + torch_stub.version = _types.SimpleNamespace(hip = "6.0") + monkeypatch.setitem(sys.modules, "torch", torch_stub) + monkeypatch.setattr(sys, "platform", "win32") + env = {"CUDA_VISIBLE_DEVICES": "3,1", "ROCR_VISIBLE_DEVICES": "9"} + LlamaCppBackend._pin_visible_gpu_order_for_split(env) + assert env["CUDA_VISIBLE_DEVICES"] == "1,3" + assert env["HIP_VISIBLE_DEVICES"] == "1,3" + assert "ROCR_VISIBLE_DEVICES" not in env + + +def _rocm_torch_stub(monkeypatch): + torch_stub = _types.ModuleType("torch") + torch_stub.version = _types.SimpleNamespace(hip = "6.0") + monkeypatch.setitem(sys.modules, "torch", torch_stub) + # prefer_rocr is Linux-only (ROCR is an ROCr variable); pin the platform so + # these Linux-behaviour tests also pass on a Windows dev box. + monkeypatch.setattr(sys, "platform", "linux") + + +def test_subset_pin_masks_via_rocr_on_rocm(monkeypatch): + # A GPU-subset pin must exclude the rest at the ROCr/HSA layer: HIP masking + # still enumerates every agent first, which segfaults the build on an + # unsupported deselected GPU (e.g. a gfx1103 iGPU under a gfx110X prebuilt). + # ROCR drops it at the driver layer; only one mask is set (HIP cleared). + _rocm_torch_stub(monkeypatch) + env = {"HIP_VISIBLE_DEVICES": "9"} # stale/inherited HIP mask must not survive + LlamaCppBackend._emit_child_gpu_visibility(env, "0", prefer_rocr = True) + assert env["ROCR_VISIBLE_DEVICES"] == "0" + assert env["CUDA_VISIBLE_DEVICES"] == "0" + assert "HIP_VISIBLE_DEVICES" not in env + + +def test_prefer_rocr_remaps_cuda_to_post_rocr_ordinals(monkeypatch): + # ROCR re-indexes the visible agents from 0, and HIP (cleared here) falls back + # to CUDA_VISIBLE_DEVICES -- so on the prefer_rocr path CUDA must carry the + # post-ROCR ordinals, not the physical ids, else a non-zero pick indexes out + # of range and the child sees no GPU and drops to CPU (#7272 review). + _rocm_torch_stub(monkeypatch) + # Single non-zero GPU: ROCR keeps the physical id, CUDA becomes ordinal 0. + env = {} + LlamaCppBackend._emit_child_gpu_visibility(env, "1", prefer_rocr = True) + assert env["ROCR_VISIBLE_DEVICES"] == "1" + assert env["CUDA_VISIBLE_DEVICES"] == "0" + assert "HIP_VISIBLE_DEVICES" not in env + # Multi-GPU subset: ROCR keeps the physical ids, CUDA is the 0-based ordinals. + env = {} + LlamaCppBackend._emit_child_gpu_visibility(env, "1,3", prefer_rocr = True) + assert env["ROCR_VISIBLE_DEVICES"] == "1,3" + assert env["CUDA_VISIBLE_DEVICES"] == "0,1" + assert "HIP_VISIBLE_DEVICES" not in env + + +def test_subset_pin_default_still_uses_hip_and_clears_rocr(monkeypatch): + # Without prefer_rocr the masking is unchanged: HIP narrows, inherited ROCR + # is cleared so the two can't double-mask. + _rocm_torch_stub(monkeypatch) + env = {"ROCR_VISIBLE_DEVICES": "0,1"} + LlamaCppBackend._emit_child_gpu_visibility(env, "1") + assert env["HIP_VISIBLE_DEVICES"] == "1" + assert "ROCR_VISIBLE_DEVICES" not in env + + +def test_cpu_only_pin_keeps_hip_even_with_prefer_rocr(monkeypatch): + # The CPU-only sentinel never routes through ROCR (no portable "hide all" + # spelling); it hides every GPU via HIP. + _rocm_torch_stub(monkeypatch) + env = {} + LlamaCppBackend._emit_child_gpu_visibility(env, "-1", prefer_rocr = True) + assert env["HIP_VISIBLE_DEVICES"] == "-1" + assert "ROCR_VISIBLE_DEVICES" not in env + + +def _amd_sdk_torch_stub(monkeypatch): + # AMD SDK wheel: torch.version.hip is None but __version__ encodes rocm. + torch_stub = _types.ModuleType("torch") + torch_stub.version = _types.SimpleNamespace(hip = None) + torch_stub.__version__ = "2.9.1+rocm7.2.1" + monkeypatch.setitem(sys.modules, "torch", torch_stub) + monkeypatch.setattr(sys, "platform", "linux") + + +def test_prefer_rocr_falls_back_to_hip_on_windows(monkeypatch): + # ROCR_VISIBLE_DEVICES is a Linux ROCr variable (Windows HIP has no ROCr + # layer), so on Windows ROCm prefer_rocr must keep the HIP mask or a nonzero + # pick loses its only effective selector (#7272 review). + torch_stub = _types.ModuleType("torch") + torch_stub.version = _types.SimpleNamespace(hip = "6.0") + monkeypatch.setitem(sys.modules, "torch", torch_stub) + monkeypatch.setattr(sys, "platform", "win32") + env = {"ROCR_VISIBLE_DEVICES": "9"} + LlamaCppBackend._emit_child_gpu_visibility(env, "1", prefer_rocr = True) + assert env["HIP_VISIBLE_DEVICES"] == "1" + assert env["CUDA_VISIBLE_DEVICES"] == "1" + assert "ROCR_VISIBLE_DEVICES" not in env + + +def test_amd_sdk_wheel_hip_none_still_masks_rocr(monkeypatch): + # An AMD SDK wheel leaves torch.version.hip unset but has "rocm" in __version__. + # It must still get the ROCR mask, else only CUDA_VISIBLE_DEVICES is set and an + # unsupported iGPU keeps enumerating and can crash llama-server. + _amd_sdk_torch_stub(monkeypatch) + env = {"HIP_VISIBLE_DEVICES": "9"} + LlamaCppBackend._emit_child_gpu_visibility(env, "0", prefer_rocr = True) + assert env["ROCR_VISIBLE_DEVICES"] == "0" + assert "HIP_VISIBLE_DEVICES" not in env + + +def test_cuda_wheel_hip_none_gets_no_rocm_mask(monkeypatch): + # A CUDA wheel (hip=None, no "rocm" in __version__) must NOT get a HIP/ROCR mask + # -- only CUDA_VISIBLE_DEVICES -- so the version-string check can't false-positive. + torch_stub = _types.ModuleType("torch") + torch_stub.version = _types.SimpleNamespace(hip = None) + torch_stub.__version__ = "2.9.1+cu124" + monkeypatch.setitem(sys.modules, "torch", torch_stub) + env = {} + LlamaCppBackend._emit_child_gpu_visibility(env, "0", prefer_rocr = True) + assert env["CUDA_VISIBLE_DEVICES"] == "0" + assert "ROCR_VISIBLE_DEVICES" not in env + assert "HIP_VISIBLE_DEVICES" not in env + + +def test_resolve_physical_ids_reads_rocr_on_amd_sdk_wheel(monkeypatch): + # _resolve_visible_physical_ids must use the same ROCm detection as + # _emit_child_gpu_visibility: on an AMD SDK wheel (hip=None, rocm in + # __version__) an inherited ROCR mask IS the ordinal->physical mapping. + # Reading it as "no mask" labels ordinal 0 as physical 0 and the child's + # ROCR pin then re-exposes the GPU the mask was hiding (#7272 review). + _amd_sdk_torch_stub(monkeypatch) + for var in ("HIP_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"): + monkeypatch.delenv(var, raising = False) + monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "1") + assert LlamaCppBackend._resolve_visible_physical_ids() == [1] + + +def test_resolve_physical_ids_ignores_rocr_on_cuda_wheel(monkeypatch): + # A CUDA wheel (hip=None, no "rocm") keeps CUDA-only semantics: a stray + # ROCR var must not be read as the mask. + torch_stub = _types.ModuleType("torch") + torch_stub.version = _types.SimpleNamespace(hip = None) + torch_stub.__version__ = "2.9.1+cu124" + monkeypatch.setitem(sys.modules, "torch", torch_stub) + for var in ("HIP_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"): + monkeypatch.delenv(var, raising = False) + monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "1") + assert LlamaCppBackend._resolve_visible_physical_ids() is None + + +def test_resolve_physical_ids_ignores_rocr_on_windows(monkeypatch): + # ROCR_VISIBLE_DEVICES is a Linux ROCr variable: Windows HIP has no ROCr + # layer, so a stray ROCR var there does not mask the runtime. Reading it as + # the ordinal->physical mapping would label ordinal 0 with a stale ROCR id + # while the runtime still enumerates every adapter, so auto-selection could + # budget one card and pin another (#7272 review). HIP must still be honoured. + torch_stub = _types.ModuleType("torch") + torch_stub.version = _types.SimpleNamespace(hip = None) + torch_stub.__version__ = "2.9.1+rocm7.2.1" # AMD SDK wheel + monkeypatch.setitem(sys.modules, "torch", torch_stub) + monkeypatch.setattr(sys, "platform", "win32") + for var in ("HIP_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"): + monkeypatch.delenv(var, raising = False) + monkeypatch.setenv("ROCR_VISIBLE_DEVICES", "1") + assert LlamaCppBackend._resolve_visible_physical_ids() is None + # HIP precedence is unchanged on Windows. + monkeypatch.setenv("HIP_VISIBLE_DEVICES", "1") + assert LlamaCppBackend._resolve_visible_physical_ids() == [1] + + # ── Diffusion single-device selection ───────────────────────────────────────