From 120daf9d8b93aa737dd32f5f17558ac509ec7142 Mon Sep 17 00:00:00 2001 From: Viktor Ferenczi Date: Thu, 11 Jun 2026 17:21:48 +0200 Subject: [PATCH] fix(studio/rocm): don't stack ROCR_VISIBLE_DEVICES on HIP_VISIBLE_DEVICES (#6176) When pinning GPUs for the llama-server child, the ROCm path set both HIP_VISIBLE_DEVICES and ROCR_VISIBLE_DEVICES to the same physical indices. These masks filter at different layers and stack: ROCR_VISIBLE_DEVICES reduces the visible set at the HSA/ROCr layer and re-indexes from 0, then HIP_VISIBLE_DEVICES indexes into that reduced set. _select_gpus ranks by free VRAM and picks the most-free card, so a single non-zero pin (e.g. "1") becomes out of range at the HIP layer, HIP enumerates 0 devices, and the model silently runs on CPU ("ggml_cuda_init: failed to initialize ROCm: no ROCm-capable device is detected"). Set only HIP_VISIBLE_DEVICES (which narrows correctly on its own) and clear any inherited ROCR mask so it can't double up. Verified on a 2x Radeon AI PRO R9700 (gfx1201) host, ROCm 7.1.1: the same selected=[1] load that fell back to CPU (~7.7 tok/s) now runs on the GPU (~78 tok/s). Fixes #6175 Co-authored-by: Lee Jackson <130007945+Imagineer99@users.noreply.github.com> --- studio/backend/core/inference/llama_cpp.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 2560f9e5ac..b83e4e6961 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -3479,7 +3479,7 @@ class LlamaCppBackend: # Pin to selected GPU(s). On ROCm, narrowing only # CUDA_VISIBLE_DEVICES leaves an AMD child seeing the full - # HIP/ROCR set, so set those too. + # set, so set HIP_VISIBLE_DEVICES too. if gpu_indices is not None: pinned = ",".join(str(i) for i in gpu_indices) env["CUDA_VISIBLE_DEVICES"] = pinned @@ -3487,7 +3487,19 @@ class LlamaCppBackend: import torch as _torch if getattr(_torch.version, "hip", None) is not None: env["HIP_VISIBLE_DEVICES"] = pinned - env["ROCR_VISIBLE_DEVICES"] = pinned + # Do NOT also set ROCR_VISIBLE_DEVICES to the same + # value. ROCR_VISIBLE_DEVICES filters at the HSA/ROCr + # layer and HIP_VISIBLE_DEVICES at the HIP layer, so + # setting both with the same physical indices applies + # the mask twice: ROCR reduces the visible set and + # re-indexes it from 0, then HIP indexes into the + # already-reduced set. A single non-zero pin (e.g. + # "1") then points out of range at the HIP layer, HIP + # enumerates 0 devices, and llama.cpp falls back to + # CPU ("ggml_cuda_init: no ROCm-capable device is + # detected"). The HIP mask alone narrows correctly; + # clear any inherited ROCR mask so it can't double up. + env.pop("ROCR_VISIBLE_DEVICES", None) except Exception as e: logger.debug("Failed to set ROCm visibility env vars for child: %s", e)