From 5603ced75f533a5fd898b3b139a76dcceddd5b0c Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 15 Mar 2026 07:58:20 +0000 Subject: [PATCH] studio: ignore CUDA_VISIBLE_DEVICES in GPU memory query for llama-server _get_gpu_free_memory was filtering by CUDA_VISIBLE_DEVICES, so with CUDA_VISIBLE_DEVICES='0' set by the training env, llama-server only saw 1 GPU and used --fit for CPU offloading instead of spreading across all 8 GPUs. Since llama-server manages its own GPU allocation (the _select_gpus method picks GPUs and sets CUDA_VISIBLE_DEVICES for the subprocess), the query must see ALL physical GPUs to make the right decision. --- studio/backend/core/inference/llama_cpp.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index a57c160a8e..bc3734fde2 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -204,12 +204,11 @@ class LlamaCppBackend: def _get_gpu_free_memory() -> list[tuple[int, int]]: """Query free memory per GPU via nvidia-smi. - Returns list of (gpu_index, free_mib) sorted by index. - Only returns GPUs that are allowed by CUDA_VISIBLE_DEVICES - (if set). Returns empty list if nvidia-smi is not available. + Returns list of (gpu_index, free_mib) for ALL physical GPUs, + ignoring CUDA_VISIBLE_DEVICES. llama-server manages its own GPU + allocation via _select_gpus which sets CUDA_VISIBLE_DEVICES + explicitly for the subprocess. """ - import os - try: result = subprocess.run( [ @@ -224,23 +223,12 @@ class LlamaCppBackend: if result.returncode != 0: return [] - # Parse which GPUs are allowed by existing CUDA_VISIBLE_DEVICES - allowed = None - cvd = os.environ.get("CUDA_VISIBLE_DEVICES") - if cvd is not None and cvd.strip(): - try: - allowed = set(int(x.strip()) for x in cvd.split(",")) - except ValueError: - pass # Non-numeric (e.g., "GPU-uuid"), ignore filter - gpus = [] for line in result.stdout.strip().splitlines(): parts = line.split(",") if len(parts) == 2: idx = int(parts[0].strip()) free_mib = int(parts[1].strip()) - if allowed is not None and idx not in allowed: - continue gpus.append((idx, free_mib)) return gpus except Exception: