ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Studio: harden GPU-offload classifier against CUDA_Host and split offload lines

Browse source 
Exclude host-pinned buffers (CUDA_Host etc.) from the GPU buffer-size signal so a
binary that pins host memory but loads weights on CPU is not misread as GPU
offload. Scan every 'offloaded N/M layers to GPU' line and accept if any N>0 so a
speculative draft model logging 0/k before the main model's 33/33 is not flagged
CPU-only. Same fix in the installer and runtime classifiers.
This commit is contained in:
danielhanchen 2026-06-01 16:03:21 +00:00
commit 17070c0886
4 changed files with 78 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

16
studio/backend/core/inference/llama_cpp.py
View file

@ -90,22 +90,34 @@ def classify_gpu_offload_lines(lines: list[str]) -> Optional[bool]:
"""True if the model landed on a GPU, False if it stayed on CPU despite GPU
intent, None when the log has no usable signal. Priority: buffer-size lines,
then offloaded-layers count, then device_info enumeration."""
# Exclude host-pinned buffers ("CUDA_Host" ...): CPU RAM the GPU backend
# pinned, not device memory, so they must not read as GPU offload.
saw_buffer_line = False
for line in lines:
if "buffer size" not in line:
continue
if any(marker in line for marker in _GPU_BUFFER_MARKERS):
if "_Host" not in line and any(
marker in line for marker in _GPU_BUFFER_MARKERS
):
return True
if "model buffer size" in line:
saw_buffer_line = True
# Accept if any "offloaded N/M" has N>0 (a draft model can log 0/k before
# the main model's 33/33); CPU-only only when every offloaded line is zero.
saw_offloaded = False
for line in lines:
match = _OFFLOADED_LAYERS_RE.search(line)
if match:
return int(match.group(1)) > 0
saw_offloaded = True
if int(match.group(1)) > 0:
return True
continue
low = line.lower()
if "offloading" in low and "to gpu" in low:
return True
if saw_offloaded:
return False
after_device_info = False
saw_device_row = False

19
studio/backend/tests/test_llama_cpp_context_fit.py
View file

@ -621,3 +621,22 @@ class TestClassifyGpuOffload:
["system_info: n_threads = 8 | CUDA : ARCHS = 1200 | CPU : AVX2 = 1"]
)
assert inst._classify_gpu_offload(True, [(0, 22805)]) is None
def test_cuda_host_buffer_is_not_gpu(self):
# CUDA_Host is host-pinned CPU RAM; weights on CPU_Mapped means CPU only.
inst = self._backend(
[
"load_tensors: CUDA_Host model buffer size = 21000.0 MiB",
"load_tensors: CPU_Mapped model buffer size = 0.6 MiB",
]
)
assert inst._classify_gpu_offload(True, [(0, 22805)]) is False
def test_draft_zero_before_main_offload_is_gpu(self):
inst = self._backend(
[
"load_tensors: offloaded 0/2 layers to GPU",
"load_tensors: offloaded 33/33 layers to GPU",
]
)
assert inst._classify_gpu_offload(True, [(0, 22805)]) is True

25
studio/install_llama_prebuilt.py
View file

@ -5665,26 +5665,39 @@ def server_log_shows_gpu_offload(log_text: str) -> bool | None:
# Signal 1: per-backend buffer-size lines. A GPU marker on ANY "buffer
# size" line (model / KV / compute) means the GPU holds part of the model
# -- accept. Only the model-buffer location decides CPU-only (KV/compute
# CPU buffers exist even on GPU runs), so the False determination keys on
# "model buffer size" alone.
# -- accept. Exclude host-pinned buffers ("CUDA_Host" / "ROCm_Host" ...):
# those are CPU RAM the GPU backend pinned, not device memory, so a binary
# that pins host memory but offloads no weights must not read as GPU. Only
# the model-buffer location decides CPU-only (KV/compute CPU buffers exist
# even on GPU runs), so the False determination keys on "model buffer size".
saw_buffer_line = False
for line in lines:
if "buffer size" not in line:
continue
if any(marker in line for marker in _GPU_MODEL_BUFFER_MARKERS):
if "_Host" not in line and any(
marker in line for marker in _GPU_MODEL_BUFFER_MARKERS
):
return True
if "model buffer size" in line:
saw_buffer_line = True
# Signal 2: explicit offloaded-layers count.
# Signal 2: explicit offloaded-layers count. Scan every "offloaded N/M"
# line and accept if any has N>0 (a draft/speculative model can log
# "offloaded 0/k" before the main model's "offloaded 33/33"); only when all
# offloaded lines are zero is it CPU-only.
saw_offloaded = False
for line in lines:
match = _OFFLOADED_LAYERS_RE.search(line)
if match:
return int(match.group(1)) > 0
saw_offloaded = True
if int(match.group(1)) > 0:
return True
continue
low = line.lower()
if "offloading" in low and "to gpu" in low:
return True
if saw_offloaded:
return False
# Signal 3: device_info enumeration. Only trust device rows once the
# "device_info:" header has appeared, so the compiled-backend system_info

26
tests/studio/install/test_validate_server_gpu_offload.py
View file

@ -232,6 +232,32 @@ def test_device_row_case_insensitive():
assert server_log_shows_gpu_offload(log) is True
def test_cuda_host_buffer_is_not_gpu_offload():
# CUDA_Host is host-pinned CPU RAM, not device memory. A binary that pins
# host memory but loads weights on CPU must not pass as GPU offload.
log = (
"load_tensors: CUDA_Host model buffer size = 21000.0 MiB\n"
"load_tensors: CPU_Mapped model buffer size = 0.6 MiB\n"
)
assert server_log_shows_gpu_offload(log) is False
# A real device buffer alongside a CUDA_Host line still reads as GPU.
log_ok = (
"load_tensors: CUDA_Host model buffer size = 100.0 MiB\n"
"load_tensors: CUDA0 model buffer size = 21000.0 MiB\n"
)
assert server_log_shows_gpu_offload(log_ok) is True
def test_draft_offloaded_zero_before_main_offload_is_gpu():
# Speculative decoding: a draft model logs "offloaded 0/2" before the main
# model's "offloaded 33/33". The N>0 line must win.
log = (
"load_tensors: offloaded 0/2 layers to GPU\n"
"load_tensors: offloaded 33/33 layers to GPU\n"
)
assert server_log_shows_gpu_offload(log) is True
def test_crlf_log_parses_identically():
# Windows logs use CRLF; classification must not change.
assert (