From e73a89ff8271bb70e0636fabe84d6c00cc004db9 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 15 Jun 2026 23:06:44 -0700 Subject: [PATCH] Studio: warn when a GPU model silently loaded on CPU (#6339) * Studio: warn when a GPU model silently loaded on CPU llama-server can serve HTTP 200 while running a model entirely on CPU when its GPU backend fails to init, so Studio could run a GGUF on CPU without saying so (#5807 / #5106 / #5830). The silent-CPU warning already exists but stopped firing on current llama.cpp because _classify_gpu_offload keyed only on the dropped 'model buffer size' lines. Add a shared classify_gpu_offload_lines (offloaded N/M counts, GPU model-buffer markers excluding _Host, device_info disconfirm-only) and delegate to it so the warning fires again. Log-only: no install or load behavior changes. Pure classification of already-captured startup log lines, run once after load; no new subprocess, no slowdown. * Studio: key the CPU-offload warning on the main model, not a draft With MTP/speculative decoding llama-server logs 'offloaded N/M layers to GPU' twice: once for the main model and once for the small draft model. The old scan returned True on any non-zero count, so a drafter that fits on GPU while the main GGUF runs on CPU suppressed the warning (the Qwen3.6-27B-MTP case). Decide on the line with the most layers (the main model) instead, so a drafter cannot mask a main model on CPU. --- studio/backend/core/inference/llama_cpp.py | 116 +++++++++++++++--- .../tests/test_llama_cpp_context_fit.py | 109 +++++++++++++++- 2 files changed, 206 insertions(+), 19 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 74e7a8f220..59628c9e46 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -75,6 +75,99 @@ from state.tool_approvals import ( logger = get_logger(__name__) +# llama-server can serve HTTP 200 while running a model entirely on CPU when a +# GPU backend fails to init (#5807 / #5106 / #5830). Classify the startup log so +# Studio can warn. Priority: explicit "offloaded N/M layers to GPU" counts +# (authoritative), then GPU "model buffer size" lines (host-pinned _Host +# excluded), then the "device_info:" device table (disconfirm only). +_GPU_OFFLOAD_MARKERS = ( + "CUDA", + "ROCm", + "ROCM", + "HIP", + "Metal", + "Vulkan", + "OpenCL", + "SYCL", + "MUSA", + "CANN", +) +_OFFLOADED_LAYERS_RE = re.compile( + r"offloaded\s+(\d+)\s*/\s*(\d+)\s+layers?\s+to\s+gpu", re.IGNORECASE +) +_DEVICE_ROW_RE = re.compile( + r"-\s*(CUDA|ROCm|ROCM|HIP|Metal|Vulkan|SYCL|OpenCL|MUSA|CANN|CPU)\w*\s*:", + re.IGNORECASE, +) +_GPU_DEVICE_PREFIXES = ( + "cuda", + "rocm", + "hip", + "metal", + "vulkan", + "sycl", + "opencl", + "musa", + "cann", +) + + +def classify_gpu_offload_lines(lines: "list[str]") -> Optional[bool]: + """True if the model landed on a GPU, False if it stayed on CPU despite GPU + intent, None when the log has no usable signal.""" + # Counted offload is authoritative, keyed on the model with the most layers. + # A separate MTP/draft model logs its own (much smaller) "offloaded N/M" + # line, so decide on the largest-M line: a drafter that fits on GPU must not + # mask a main model running on CPU. N>0 on that model is True, 0 is False. + max_total = -1 + offloaded_at_max = 0 + for line in lines: + match = _OFFLOADED_LAYERS_RE.search(line) + if not match: + continue + offloaded, total = int(match.group(1)), int(match.group(2)) + if total > max_total or (total == max_total and offloaded > offloaded_at_max): + max_total, offloaded_at_max = total, offloaded + if max_total >= 0: + return offloaded_at_max > 0 + + # GPU marker on a *model* buffer; _Host buffers are CPU-pinned, not offload. + # Buffer lines are authoritative: present but none on a GPU means CPU-only, + # so do not let the device table below override that. + saw_model_buffer = False + for line in lines: + if "model buffer size" not in line: + continue + saw_model_buffer = True + if "_Host" not in line and any(m in line for m in _GPU_OFFLOAD_MARKERS): + return True + if saw_model_buffer: + return False + + # device_info: lists *available* devices (printed whenever a GPU backend is + # visible), not where the model loaded, so it can only disconfirm: an + # all-CPU table means no usable GPU. A visible GPU device is not proof the + # model used it, so it does not return True. Rows after the header only. + after_header = False + saw_device_row = False + saw_gpu_device = False + for line in lines: + if "device_info:" in line: + after_header = True + continue + if not after_header: + continue + match = _DEVICE_ROW_RE.search(line) + if not match: + continue + saw_device_row = True + if match.group(1).lower().startswith(_GPU_DEVICE_PREFIXES): + saw_gpu_device = True + if saw_device_row and not saw_gpu_device: + return False + return None + + def _wsl_system_rocm_lib_dirs() -> "list[str]": """System ROCm lib dir(s) to load before a prebuilt's bundled HIP, on WSL. @@ -4958,26 +5051,13 @@ class LlamaCppBackend: def _classify_gpu_offload( self, expected_gpu: bool, detected_gpus: list[tuple[int, int]] ) -> Optional[bool]: - """True if a GPU model buffer was allocated, False if only CPU - buffers landed despite GPU intent, None when there's no signal (no - GPU detected, no buffer-size lines, etc.).""" + """True if the model landed on a GPU, False if only CPU buffers landed + despite GPU intent, None when there's no signal. Delegates to the shared + classifier so it tracks current llama.cpp logs (offloaded-layer counts / + device_info), not just the older "model buffer size" lines.""" if not detected_gpus or not expected_gpu: return None - # llama-server logs one "model buffer size = N MiB" line per backend - # buffer; CUDA/ROCm/Metal/Vulkan/OpenCL/SYCL are GPU, CPU* are not. - gpu_markers = ("CUDA", "ROCm", "Metal", "Vulkan", "OpenCL", "SYCL") - saw_buffer_line = False - saw_gpu_buffer = False - for line in self._stdout_lines: - if "model buffer size" not in line: - continue - saw_buffer_line = True - if any(marker in line for marker in gpu_markers): - saw_gpu_buffer = True - break - if not saw_buffer_line: - return None - return saw_gpu_buffer + return classify_gpu_offload_lines(self._stdout_lines) def load_cancelled(self) -> bool: """True if a load was cancelled (e.g. via unload/_cancel_event) and not diff --git a/studio/backend/tests/test_llama_cpp_context_fit.py b/studio/backend/tests/test_llama_cpp_context_fit.py index 58226f938c..abacc6e953 100644 --- a/studio/backend/tests/test_llama_cpp_context_fit.py +++ b/studio/backend/tests/test_llama_cpp_context_fit.py @@ -67,7 +67,7 @@ _httpx_stub.Client = type( ) sys.modules.setdefault("httpx", _httpx_stub) -from core.inference.llama_cpp import LlamaCppBackend +from core.inference.llama_cpp import LlamaCppBackend, classify_gpu_offload_lines from core.inference.llama_server_args import parse_ctx_override, resolve_requested_ctx @@ -557,3 +557,110 @@ class TestClassifyGpuOffload: ] ) assert inst._classify_gpu_offload(True, [(0, 22805)]) is True + + def test_offloaded_zero_count_returns_false(self): + # Authoritative count overrides any GPU-looking buffer line. + inst = self._backend( + [ + "load_tensors: offloaded 0/33 layers to GPU", + "load_tensors: CUDA0 model buffer size = 21000.0 MiB", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is False + + def test_offloaded_draft_then_main_returns_true(self): + # A small draft model (0/2) does not mask the main model (33/33). + inst = self._backend( + [ + "load_tensors: offloaded 0/2 layers to GPU", + "load_tensors: offloaded 33/33 layers to GPU", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is True + + def test_main_on_cpu_with_draft_on_gpu_returns_false(self): + # MTP: the small drafter fits on GPU (1/1) but the main model is on CPU + # (0/33). Decide on the largest model, so the warning still fires. + inst = self._backend( + [ + "load_tensors: offloaded 0/33 layers to GPU", + "load_tensors: offloaded 1/1 layers to GPU", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is False + + def test_main_on_gpu_with_draft_on_cpu_returns_true(self): + # Reverse: main model on GPU (33/33), drafter on CPU (0/1) -> no warning. + inst = self._backend( + [ + "load_tensors: offloaded 33/33 layers to GPU", + "load_tensors: offloaded 0/1 layers to GPU", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is True + + def test_cuda_host_buffer_excluded_returns_false(self): + # CUDA_Host is CPU-pinned memory, not a model offload. + inst = self._backend( + [ + "load_tensors: CUDA_Host model buffer size = 500.0 MiB", + "load_tensors: CPU model buffer size = 21000.0 MiB", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is False + + def test_device_info_gpu_row_alone_is_inconclusive(self): + # device_info lists available devices, not where the model loaded, so a + # GPU row alone is not proof of offload. + inst = self._backend( + [ + "print_info: device_info:", + " - CUDA0 : 24564 MiB free", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is None + + def test_cpu_buffers_with_gpu_device_row_returns_false(self): + # Definite CPU-only buffers must win over a GPU device-inventory row. + inst = self._backend( + [ + "load_tensors: CPU model buffer size = 21000.0 MiB", + "print_info: device_info:", + " - CUDA0 : 24564 MiB free", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is False + + def test_device_info_cpu_only_returns_false(self): + inst = self._backend( + [ + "print_info: device_info:", + " - CPU : 64000 MiB free", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is False + + def test_system_info_cuda_before_device_info_does_not_count(self): + # A compiled-in backend named in system_info is not proof of offload; + # only the device_info table (here CPU only) decides. + inst = self._backend( + [ + "system_info: CUDA : ARCHS = 890 | n_threads = 8", + "print_info: device_info:", + " - CPU : 64000 MiB free", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is False + + @pytest.mark.parametrize( + "marker", + ["CUDA0", "ROCm0", "HIP0", "Metal", "Vulkan0", "OpenCL0", "SYCL0", "MUSA0", "CANN0"], + ) + def test_all_gpu_buffer_markers_return_true(self, marker): + assert ( + classify_gpu_offload_lines([f"load_tensors: {marker} model buffer size = 8000.0 MiB"]) + is True + ) + + def test_module_level_no_signal_returns_none(self): + assert classify_gpu_offload_lines(["INFO starting server"]) is None