Studio: warn when a GPU model silently loaded on CPU (#6339)
* Studio: warn when a GPU model silently loaded on CPU llama-server can serve HTTP 200 while running a model entirely on CPU when its GPU backend fails to init, so Studio could run a GGUF on CPU without saying so (#5807 / #5106 / #5830). The silent-CPU warning already exists but stopped firing on current llama.cpp because _classify_gpu_offload keyed only on the dropped 'model buffer size' lines. Add a shared classify_gpu_offload_lines (offloaded N/M counts, GPU model-buffer markers excluding _Host, device_info disconfirm-only) and delegate to it so the warning fires again. Log-only: no install or load behavior changes. Pure classification of already-captured startup log lines, run once after load; no new subprocess, no slowdown. * Studio: key the CPU-offload warning on the main model, not a draft With MTP/speculative decoding llama-server logs 'offloaded N/M layers to GPU' twice: once for the main model and once for the small draft model. The old scan returned True on any non-zero count, so a drafter that fits on GPU while the main GGUF runs on CPU suppressed the warning (the Qwen3.6-27B-MTP case). Decide on the line with the most layers (the main model) instead, so a drafter cannot mask a main model on CPU.
This commit is contained in:
Daniel Han
GitHub
parent
9fc21b3977
commit
e73a89ff82
2 changed files with 206 additions and 19 deletions
|
|
@ -75,6 +75,99 @@ from state.tool_approvals import (
|
|||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
# llama-server can serve HTTP 200 while running a model entirely on CPU when a
|
||||
# GPU backend fails to init (#5807 / #5106 / #5830). Classify the startup log so
|
||||
# Studio can warn. Priority: explicit "offloaded N/M layers to GPU" counts
|
||||
# (authoritative), then GPU "model buffer size" lines (host-pinned _Host
|
||||
# excluded), then the "device_info:" device table (disconfirm only).
|
||||
_GPU_OFFLOAD_MARKERS = (
|
||||
"CUDA",
|
||||
"ROCm",
|
||||
"ROCM",
|
||||
"HIP",
|
||||
"Metal",
|
||||
"Vulkan",
|
||||
"OpenCL",
|
||||
"SYCL",
|
||||
"MUSA",
|
||||
"CANN",
|
||||
)
|
||||
_OFFLOADED_LAYERS_RE = re.compile(
|
||||
r"offloaded\s+(\d+)\s*/\s*(\d+)\s+layers?\s+to\s+gpu", re.IGNORECASE
|
||||
)
|
||||
_DEVICE_ROW_RE = re.compile(
|
||||
r"-\s*(CUDA|ROCm|ROCM|HIP|Metal|Vulkan|SYCL|OpenCL|MUSA|CANN|CPU)\w*\s*:",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_GPU_DEVICE_PREFIXES = (
|
||||
"cuda",
|
||||
"rocm",
|
||||
"hip",
|
||||
"metal",
|
||||
"vulkan",
|
||||
"sycl",
|
||||
"opencl",
|
||||
"musa",
|
||||
"cann",
|
||||
)
|
||||
|
||||
|
||||
def classify_gpu_offload_lines(lines: "list[str]") -> Optional[bool]:
|
||||
"""True if the model landed on a GPU, False if it stayed on CPU despite GPU
|
||||
intent, None when the log has no usable signal."""
|
||||
# Counted offload is authoritative, keyed on the model with the most layers.
|
||||
# A separate MTP/draft model logs its own (much smaller) "offloaded N/M"
|
||||
# line, so decide on the largest-M line: a drafter that fits on GPU must not
|
||||
# mask a main model running on CPU. N>0 on that model is True, 0 is False.
|
||||
max_total = -1
|
||||
offloaded_at_max = 0
|
||||
for line in lines:
|
||||
match = _OFFLOADED_LAYERS_RE.search(line)
|
||||
if not match:
|
||||
continue
|
||||
offloaded, total = int(match.group(1)), int(match.group(2))
|
||||
if total > max_total or (total == max_total and offloaded > offloaded_at_max):
|
||||
max_total, offloaded_at_max = total, offloaded
|
||||
if max_total >= 0:
|
||||
return offloaded_at_max > 0
|
||||
|
||||
# GPU marker on a *model* buffer; _Host buffers are CPU-pinned, not offload.
|
||||
# Buffer lines are authoritative: present but none on a GPU means CPU-only,
|
||||
# so do not let the device table below override that.
|
||||
saw_model_buffer = False
|
||||
for line in lines:
|
||||
if "model buffer size" not in line:
|
||||
continue
|
||||
saw_model_buffer = True
|
||||
if "_Host" not in line and any(m in line for m in _GPU_OFFLOAD_MARKERS):
|
||||
return True
|
||||
if saw_model_buffer:
|
||||
return False
|
||||
|
||||
# device_info: lists *available* devices (printed whenever a GPU backend is
|
||||
# visible), not where the model loaded, so it can only disconfirm: an
|
||||
# all-CPU table means no usable GPU. A visible GPU device is not proof the
|
||||
# model used it, so it does not return True. Rows after the header only.
|
||||
after_header = False
|
||||
saw_device_row = False
|
||||
saw_gpu_device = False
|
||||
for line in lines:
|
||||
if "device_info:" in line:
|
||||
after_header = True
|
||||
continue
|
||||
if not after_header:
|
||||
continue
|
||||
match = _DEVICE_ROW_RE.search(line)
|
||||
if not match:
|
||||
continue
|
||||
saw_device_row = True
|
||||
if match.group(1).lower().startswith(_GPU_DEVICE_PREFIXES):
|
||||
saw_gpu_device = True
|
||||
if saw_device_row and not saw_gpu_device:
|
||||
return False
|
||||
return None
|
||||
|
||||
|
||||
def _wsl_system_rocm_lib_dirs() -> "list[str]":
|
||||
"""System ROCm lib dir(s) to load before a prebuilt's bundled HIP, on WSL.
|
||||
|
||||
|
|
@ -4958,26 +5051,13 @@ class LlamaCppBackend:
|
|||
def _classify_gpu_offload(
|
||||
self, expected_gpu: bool, detected_gpus: list[tuple[int, int]]
|
||||
) -> Optional[bool]:
|
||||
"""True if a GPU model buffer was allocated, False if only CPU
|
||||
buffers landed despite GPU intent, None when there's no signal (no
|
||||
GPU detected, no buffer-size lines, etc.)."""
|
||||
"""True if the model landed on a GPU, False if only CPU buffers landed
|
||||
despite GPU intent, None when there's no signal. Delegates to the shared
|
||||
classifier so it tracks current llama.cpp logs (offloaded-layer counts /
|
||||
device_info), not just the older "model buffer size" lines."""
|
||||
if not detected_gpus or not expected_gpu:
|
||||
return None
|
||||
# llama-server logs one "model buffer size = N MiB" line per backend
|
||||
# buffer; CUDA/ROCm/Metal/Vulkan/OpenCL/SYCL are GPU, CPU* are not.
|
||||
gpu_markers = ("CUDA", "ROCm", "Metal", "Vulkan", "OpenCL", "SYCL")
|
||||
saw_buffer_line = False
|
||||
saw_gpu_buffer = False
|
||||
for line in self._stdout_lines:
|
||||
if "model buffer size" not in line:
|
||||
continue
|
||||
saw_buffer_line = True
|
||||
if any(marker in line for marker in gpu_markers):
|
||||
saw_gpu_buffer = True
|
||||
break
|
||||
if not saw_buffer_line:
|
||||
return None
|
||||
return saw_gpu_buffer
|
||||
return classify_gpu_offload_lines(self._stdout_lines)
|
||||
|
||||
def load_cancelled(self) -> bool:
|
||||
"""True if a load was cancelled (e.g. via unload/_cancel_event) and not
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ _httpx_stub.Client = type(
|
|||
)
|
||||
sys.modules.setdefault("httpx", _httpx_stub)
|
||||
|
||||
from core.inference.llama_cpp import LlamaCppBackend
|
||||
from core.inference.llama_cpp import LlamaCppBackend, classify_gpu_offload_lines
|
||||
from core.inference.llama_server_args import parse_ctx_override, resolve_requested_ctx
|
||||
|
||||
|
||||
|
|
@ -557,3 +557,110 @@ class TestClassifyGpuOffload:
|
|||
]
|
||||
)
|
||||
assert inst._classify_gpu_offload(True, [(0, 22805)]) is True
|
||||
|
||||
def test_offloaded_zero_count_returns_false(self):
|
||||
# Authoritative count overrides any GPU-looking buffer line.
|
||||
inst = self._backend(
|
||||
[
|
||||
"load_tensors: offloaded 0/33 layers to GPU",
|
||||
"load_tensors: CUDA0 model buffer size = 21000.0 MiB",
|
||||
]
|
||||
)
|
||||
assert inst._classify_gpu_offload(True, [(0, 22805)]) is False
|
||||
|
||||
def test_offloaded_draft_then_main_returns_true(self):
|
||||
# A small draft model (0/2) does not mask the main model (33/33).
|
||||
inst = self._backend(
|
||||
[
|
||||
"load_tensors: offloaded 0/2 layers to GPU",
|
||||
"load_tensors: offloaded 33/33 layers to GPU",
|
||||
]
|
||||
)
|
||||
assert inst._classify_gpu_offload(True, [(0, 22805)]) is True
|
||||
|
||||
def test_main_on_cpu_with_draft_on_gpu_returns_false(self):
|
||||
# MTP: the small drafter fits on GPU (1/1) but the main model is on CPU
|
||||
# (0/33). Decide on the largest model, so the warning still fires.
|
||||
inst = self._backend(
|
||||
[
|
||||
"load_tensors: offloaded 0/33 layers to GPU",
|
||||
"load_tensors: offloaded 1/1 layers to GPU",
|
||||
]
|
||||
)
|
||||
assert inst._classify_gpu_offload(True, [(0, 22805)]) is False
|
||||
|
||||
def test_main_on_gpu_with_draft_on_cpu_returns_true(self):
|
||||
# Reverse: main model on GPU (33/33), drafter on CPU (0/1) -> no warning.
|
||||
inst = self._backend(
|
||||
[
|
||||
"load_tensors: offloaded 33/33 layers to GPU",
|
||||
"load_tensors: offloaded 0/1 layers to GPU",
|
||||
]
|
||||
)
|
||||
assert inst._classify_gpu_offload(True, [(0, 22805)]) is True
|
||||
|
||||
def test_cuda_host_buffer_excluded_returns_false(self):
|
||||
# CUDA_Host is CPU-pinned memory, not a model offload.
|
||||
inst = self._backend(
|
||||
[
|
||||
"load_tensors: CUDA_Host model buffer size = 500.0 MiB",
|
||||
"load_tensors: CPU model buffer size = 21000.0 MiB",
|
||||
]
|
||||
)
|
||||
assert inst._classify_gpu_offload(True, [(0, 22805)]) is False
|
||||
|
||||
def test_device_info_gpu_row_alone_is_inconclusive(self):
|
||||
# device_info lists available devices, not where the model loaded, so a
|
||||
# GPU row alone is not proof of offload.
|
||||
inst = self._backend(
|
||||
[
|
||||
"print_info: device_info:",
|
||||
" - CUDA0 : 24564 MiB free",
|
||||
]
|
||||
)
|
||||
assert inst._classify_gpu_offload(True, [(0, 22805)]) is None
|
||||
|
||||
def test_cpu_buffers_with_gpu_device_row_returns_false(self):
|
||||
# Definite CPU-only buffers must win over a GPU device-inventory row.
|
||||
inst = self._backend(
|
||||
[
|
||||
"load_tensors: CPU model buffer size = 21000.0 MiB",
|
||||
"print_info: device_info:",
|
||||
" - CUDA0 : 24564 MiB free",
|
||||
]
|
||||
)
|
||||
assert inst._classify_gpu_offload(True, [(0, 22805)]) is False
|
||||
|
||||
def test_device_info_cpu_only_returns_false(self):
|
||||
inst = self._backend(
|
||||
[
|
||||
"print_info: device_info:",
|
||||
" - CPU : 64000 MiB free",
|
||||
]
|
||||
)
|
||||
assert inst._classify_gpu_offload(True, [(0, 22805)]) is False
|
||||
|
||||
def test_system_info_cuda_before_device_info_does_not_count(self):
|
||||
# A compiled-in backend named in system_info is not proof of offload;
|
||||
# only the device_info table (here CPU only) decides.
|
||||
inst = self._backend(
|
||||
[
|
||||
"system_info: CUDA : ARCHS = 890 | n_threads = 8",
|
||||
"print_info: device_info:",
|
||||
" - CPU : 64000 MiB free",
|
||||
]
|
||||
)
|
||||
assert inst._classify_gpu_offload(True, [(0, 22805)]) is False
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"marker",
|
||||
["CUDA0", "ROCm0", "HIP0", "Metal", "Vulkan0", "OpenCL0", "SYCL0", "MUSA0", "CANN0"],
|
||||
)
|
||||
def test_all_gpu_buffer_markers_return_true(self, marker):
|
||||
assert (
|
||||
classify_gpu_offload_lines([f"load_tensors: {marker} model buffer size = 8000.0 MiB"])
|
||||
is True
|
||||
)
|
||||
|
||||
def test_module_level_no_signal_returns_none(self):
|
||||
assert classify_gpu_offload_lines(["INFO starting server"]) is None
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue