diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 9001396afe..67758b7073 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -90,22 +90,34 @@ def classify_gpu_offload_lines(lines: list[str]) -> Optional[bool]: """True if the model landed on a GPU, False if it stayed on CPU despite GPU intent, None when the log has no usable signal. Priority: buffer-size lines, then offloaded-layers count, then device_info enumeration.""" + # Exclude host-pinned buffers ("CUDA_Host" ...): CPU RAM the GPU backend + # pinned, not device memory, so they must not read as GPU offload. saw_buffer_line = False for line in lines: if "buffer size" not in line: continue - if any(marker in line for marker in _GPU_BUFFER_MARKERS): + if "_Host" not in line and any( + marker in line for marker in _GPU_BUFFER_MARKERS + ): return True if "model buffer size" in line: saw_buffer_line = True + # Accept if any "offloaded N/M" has N>0 (a draft model can log 0/k before + # the main model's 33/33); CPU-only only when every offloaded line is zero. + saw_offloaded = False for line in lines: match = _OFFLOADED_LAYERS_RE.search(line) if match: - return int(match.group(1)) > 0 + saw_offloaded = True + if int(match.group(1)) > 0: + return True + continue low = line.lower() if "offloading" in low and "to gpu" in low: return True + if saw_offloaded: + return False after_device_info = False saw_device_row = False diff --git a/studio/backend/tests/test_llama_cpp_context_fit.py b/studio/backend/tests/test_llama_cpp_context_fit.py index 3d295f711c..0e069684c7 100644 --- a/studio/backend/tests/test_llama_cpp_context_fit.py +++ b/studio/backend/tests/test_llama_cpp_context_fit.py @@ -621,3 +621,22 @@ class TestClassifyGpuOffload: ["system_info: n_threads = 8 | CUDA : ARCHS = 1200 | CPU : AVX2 = 1"] ) assert inst._classify_gpu_offload(True, [(0, 22805)]) is None + + def test_cuda_host_buffer_is_not_gpu(self): + # CUDA_Host is host-pinned CPU RAM; weights on CPU_Mapped means CPU only. + inst = self._backend( + [ + "load_tensors: CUDA_Host model buffer size = 21000.0 MiB", + "load_tensors: CPU_Mapped model buffer size = 0.6 MiB", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is False + + def test_draft_zero_before_main_offload_is_gpu(self): + inst = self._backend( + [ + "load_tensors: offloaded 0/2 layers to GPU", + "load_tensors: offloaded 33/33 layers to GPU", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is True diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index 151c459924..c1c520f5f9 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -5665,26 +5665,39 @@ def server_log_shows_gpu_offload(log_text: str) -> bool | None: # Signal 1: per-backend buffer-size lines. A GPU marker on ANY "buffer # size" line (model / KV / compute) means the GPU holds part of the model - # -- accept. Only the model-buffer location decides CPU-only (KV/compute - # CPU buffers exist even on GPU runs), so the False determination keys on - # "model buffer size" alone. + # -- accept. Exclude host-pinned buffers ("CUDA_Host" / "ROCm_Host" ...): + # those are CPU RAM the GPU backend pinned, not device memory, so a binary + # that pins host memory but offloads no weights must not read as GPU. Only + # the model-buffer location decides CPU-only (KV/compute CPU buffers exist + # even on GPU runs), so the False determination keys on "model buffer size". saw_buffer_line = False for line in lines: if "buffer size" not in line: continue - if any(marker in line for marker in _GPU_MODEL_BUFFER_MARKERS): + if "_Host" not in line and any( + marker in line for marker in _GPU_MODEL_BUFFER_MARKERS + ): return True if "model buffer size" in line: saw_buffer_line = True - # Signal 2: explicit offloaded-layers count. + # Signal 2: explicit offloaded-layers count. Scan every "offloaded N/M" + # line and accept if any has N>0 (a draft/speculative model can log + # "offloaded 0/k" before the main model's "offloaded 33/33"); only when all + # offloaded lines are zero is it CPU-only. + saw_offloaded = False for line in lines: match = _OFFLOADED_LAYERS_RE.search(line) if match: - return int(match.group(1)) > 0 + saw_offloaded = True + if int(match.group(1)) > 0: + return True + continue low = line.lower() if "offloading" in low and "to gpu" in low: return True + if saw_offloaded: + return False # Signal 3: device_info enumeration. Only trust device rows once the # "device_info:" header has appeared, so the compiled-backend system_info diff --git a/tests/studio/install/test_validate_server_gpu_offload.py b/tests/studio/install/test_validate_server_gpu_offload.py index 561ea94ab8..49c6cb8ad9 100644 --- a/tests/studio/install/test_validate_server_gpu_offload.py +++ b/tests/studio/install/test_validate_server_gpu_offload.py @@ -232,6 +232,32 @@ def test_device_row_case_insensitive(): assert server_log_shows_gpu_offload(log) is True +def test_cuda_host_buffer_is_not_gpu_offload(): + # CUDA_Host is host-pinned CPU RAM, not device memory. A binary that pins + # host memory but loads weights on CPU must not pass as GPU offload. + log = ( + "load_tensors: CUDA_Host model buffer size = 21000.0 MiB\n" + "load_tensors: CPU_Mapped model buffer size = 0.6 MiB\n" + ) + assert server_log_shows_gpu_offload(log) is False + # A real device buffer alongside a CUDA_Host line still reads as GPU. + log_ok = ( + "load_tensors: CUDA_Host model buffer size = 100.0 MiB\n" + "load_tensors: CUDA0 model buffer size = 21000.0 MiB\n" + ) + assert server_log_shows_gpu_offload(log_ok) is True + + +def test_draft_offloaded_zero_before_main_offload_is_gpu(): + # Speculative decoding: a draft model logs "offloaded 0/2" before the main + # model's "offloaded 33/33". The N>0 line must win. + log = ( + "load_tensors: offloaded 0/2 layers to GPU\n" + "load_tensors: offloaded 33/33 layers to GPU\n" + ) + assert server_log_shows_gpu_offload(log) is True + + def test_crlf_log_parses_identically(): # Windows logs use CRLF; classification must not change. assert (