ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Studio: validate real GPU offload and stop silently shipping CPU-only GGUF

Browse source 
A llama-server whose GPU backend fails to initialize still serves HTTP 200 from
CPU, so install validation accepted it and Studio ran 'GPU' inference on CPU
(#5807/#5106/#5830). Add a log classifier (buffer-size, offloaded-layers, and
device_info signals) and reject a GPU-intended binary that loaded on CPU, so the
resolver advances. Crucially, a GPU host no longer gets a CPU prebuilt appended
to its simple-policy attempts, so when no GPU bundle offloads the installer falls
through to a source build for the native arch instead of silently installing CPU.
Add a --smoke-test CLI for setup scripts and align the runtime classifier with
the same signals so its CPU-only warning fires on current llama.cpp.
This commit is contained in:
danielhanchen 2026-06-01 15:44:26 +00:00
commit 2f88ecd8a4
5 changed files with 1006 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

94
studio/backend/core/inference/llama_cpp.py
View file

@ -59,6 +59,75 @@ from core.inference.tool_call_parser import (
logger = get_logger(__name__)
# ── GPU-offload log classifier (shared shape with install_llama_prebuilt) ──
# llama-server can serve HTTP 200 while running entirely on CPU; detect that so
# Studio can warn instead of silently running a "GPU" model on CPU (#5807 /
# #5106 / #5830). Robust across llama.cpp log formats (the "model buffer size"
# lines were dropped in recent builds): buffer-size markers, "offloaded N/M
# layers to GPU", and the "device_info:" enumeration.
_GPU_BUFFER_MARKERS = ("CUDA", "ROCm", "Metal", "Vulkan", "OpenCL", "SYCL")
_DEVICE_ROW_RE = re.compile(
r"-\s*(?P<dev>(?:CUDA|ROCm|ROCM|HIP|Metal|Vulkan|SYCL|OpenCL|MUSA|CANN|CPU)\w*)\s*:",
re.IGNORECASE,
)
_GPU_DEVICE_PREFIXES = (
"cuda",
"rocm",
"hip",
"metal",
"vulkan",
"sycl",
"opencl",
"musa",
"cann",
)
_OFFLOADED_LAYERS_RE = re.compile(
r"offloaded\s+(\d+)\s*/\s*(\d+)\s+layers?\s+to\s+gpu", re.IGNORECASE
)
def classify_gpu_offload_lines(lines: list[str]) -> Optional[bool]:
"""True if the model landed on a GPU, False if it stayed on CPU despite GPU
intent, None when the log has no usable signal. Priority: buffer-size lines,
then offloaded-layers count, then device_info enumeration."""
saw_buffer_line = False
for line in lines:
if "buffer size" not in line:
continue
if any(marker in line for marker in _GPU_BUFFER_MARKERS):
return True
if "model buffer size" in line:
saw_buffer_line = True
for line in lines:
match = _OFFLOADED_LAYERS_RE.search(line)
if match:
return int(match.group(1)) > 0
low = line.lower()
if "offloading" in low and "to gpu" in low:
return True
after_device_info = False
saw_device_row = False
for line in lines:
if "device_info:" in line:
after_device_info = True
continue
if not after_device_info:
continue
match = _DEVICE_ROW_RE.search(line)
if not match:
continue
saw_device_row = True
dev = match.group("dev").lower()
if any(dev.startswith(prefix) for prefix in _GPU_DEVICE_PREFIXES):
return True
if saw_buffer_line or saw_device_row:
return False
return None
# ── Pre-compiled patterns for plan-without-action re-prompt ──
# Forward-looking intent signals that indicate the model is
# describing what it *will* do rather than giving a final answer.
@ -3865,27 +3934,14 @@ class LlamaCppBackend:
expected_gpu: bool,
detected_gpus: list[tuple[int, int]],
) -> Optional[bool]:
"""True if a GPU model buffer was allocated, False if only CPU
buffers landed despite GPU intent, None when there's no signal
(no GPU detected, no buffer-size lines, etc.)."""
"""True if the model landed on a GPU, False if only CPU buffers landed
despite GPU intent, None when there's no signal (no GPU detected, no
usable log lines, etc.). Delegates to the shared classifier so it tracks
current llama.cpp logs (device_info / offloaded-layers), which dropped
the older ``model buffer size`` lines this used to key on."""
if not detected_gpus or not expected_gpu:
return None
# llama-server logs one ``... model buffer size = N MiB`` line
# per backend buffer; CUDA0 / ROCm0 / Metal / Vulkan0 /
# OpenCL0 / SYCL0 are GPU, CPU / CPU_Mapped are not.
gpu_markers = ("CUDA", "ROCm", "Metal", "Vulkan", "OpenCL", "SYCL")
saw_buffer_line = False
saw_gpu_buffer = False
for line in self._stdout_lines:
if "model buffer size" not in line:
continue
saw_buffer_line = True
if any(marker in line for marker in gpu_markers):
saw_gpu_buffer = True
break
if not saw_buffer_line:
return None
return saw_gpu_buffer
return classify_gpu_offload_lines(self._stdout_lines)
def unload_model(self) -> bool:
"""Terminate the llama-server subprocess and cancel any in-flight download."""

42
studio/backend/tests/test_llama_cpp_context_fit.py
View file

@ -577,3 +577,45 @@ class TestClassifyGpuOffload:
]
)
assert inst._classify_gpu_offload(True, [(0, 22805)]) is True
def test_current_device_info_cuda_returns_true(self):
# Recent llama.cpp dropped the buffer lines and prints device_info.
inst = self._backend(
[
"0.00 I device_info:",
"0.01 I - CUDA0 : NVIDIA GeForce RTX 5070 (12282 MiB free)",
"0.01 I - CPU : AMD Ryzen 7 9700X (32000 MiB free)",
]
)
assert inst._classify_gpu_offload(True, [(0, 12282)]) is True
def test_current_device_info_cpu_only_returns_false(self):
# The #5830 symptom: device_info shows only CPU, CUDA0 missing.
inst = self._backend(
[
"0.00 I device_info:",
"0.00 I - CPU : AMD Ryzen 7 9700X (32000 MiB free)",
"0.00 I srv llama_server: model loaded",
]
)
assert inst._classify_gpu_offload(True, [(0, 12282)]) is False
def test_offloaded_layers_count_decides(self):
assert (
self._backend(["load_tensors: offloaded 33/33 layers to GPU"])
._classify_gpu_offload(True, [(0, 22805)])
is True
)
assert (
self._backend(["load_tensors: offloaded 0/33 layers to GPU"])
._classify_gpu_offload(True, [(0, 22805)])
is False
)
def test_system_info_only_returns_none(self):
# A compiled CUDA backend advertised in system_info is not proof of an
# available device; without device_info/buffer/offload signal -> None.
inst = self._backend(
["system_info: n_threads = 8 | CUDA : ARCHS = 1200 | CPU : AVX2 = 1"]
)
assert inst._classify_gpu_offload(True, [(0, 22805)]) is None

341
studio/install_llama_prebuilt.py
View file

@ -452,6 +452,14 @@ class PrebuiltFallback(RuntimeError):
pass
class GpuOffloadFailure(PrebuiltFallback):
"""The binary started and served a completion but loaded the model only on
CPU despite GPU offload being requested. A subclass of PrebuiltFallback so
the prebuilt resolver still advances to the next candidate, while the
--smoke-test CLI can tell this apart from an inconclusive start/serve
failure (definite CPU-only -> EXIT_FALLBACK; inconclusive -> EXIT_ERROR)."""
class BusyInstallConflict(RuntimeError):
pass
@ -1437,7 +1445,14 @@ def direct_linux_release_plan(
)
if lemonade_choice is not None:
attempts.append(lemonade_choice)
else:
elif not host.has_usable_nvidia:
# CPU-only host (or --cpu-fallback, which zeroes the GPU flags): ship the
# CPU bundle. Do NOT append it for a usable-NVIDIA host: if every CUDA
# bundle fails the GPU-offload check, let install raise PrebuiltFallback
# so setup.sh builds llama.cpp from source for the native arch (which
# offloads) instead of silently shipping a CPU-only "GPU" install
# (#5807). The CPU prebuilt stays the last resort after the source build,
# reached via setup.sh's --cpu-fallback path.
cpu_choice = published_asset_choice_for_kind(bundle, "linux-cpu")
if cpu_choice is not None:
attempts.append(cpu_choice)
@ -1530,19 +1545,26 @@ def direct_upstream_release_plan(
install_kind = "windows-hip",
)
)
cpu_asset = f"llama-{release_tag}-bin-win-cpu-x64.zip"
cpu_url = assets.get(cpu_asset)
if cpu_url:
attempts.append(
AssetChoice(
repo = repo,
tag = release_tag,
name = cpu_asset,
url = cpu_url,
source_label = "upstream",
install_kind = "windows-cpu",
# Append the CPU bundle only for a host with no usable GPU (or
# --cpu-fallback, which zeroes the GPU flags). On a GPU host, omitting it
# lets install raise PrebuiltFallback when every GPU bundle fails the
# offload check, so setup.ps1 builds from source / falls back to the CPU
# prebuilt as a labelled last resort instead of silently shipping a
# CPU-only "GPU" install (#5807).
if not host.has_usable_nvidia and not host.has_rocm:
cpu_asset = f"llama-{release_tag}-bin-win-cpu-x64.zip"
cpu_url = assets.get(cpu_asset)
if cpu_url:
attempts.append(
AssetChoice(
repo = repo,
tag = release_tag,
name = cpu_asset,
url = cpu_url,
source_label = "upstream",
install_kind = "windows-cpu",
)
)
)
elif host.is_windows and host.is_arm64:
# Upstream ggml-org/llama.cpp ships llama-bNNNN-bin-win-cpu-arm64.zip
# (visible in the b9334 release manifest). Without this branch the
@ -5577,6 +5599,128 @@ def validate_quantize(
)
# GPU backend markers that appear in llama-server's "... model buffer size ..."
# load lines (older llama.cpp) and in its "device_info:" enumeration (current
# llama.cpp). CUDA0 / ROCm0 / Metal / Vulkan0 / OpenCL0 / SYCL0 / HIP0 / MUSA0
# / CANN0 are GPU; CPU / CPU_Mapped are not. Kept broad so a single helper
# covers every backend Studio ships.
_GPU_MODEL_BUFFER_MARKERS = ("CUDA", "ROCm", "Metal", "Vulkan", "OpenCL", "SYCL")
# A device_info row looks like "<ts> I - CUDA0 : NVIDIA B200 (...)" or
# "<ts> I - CPU : ...". Matched on the "- <backend> :" shape (the leading
# "- " distinguishes it from the system_info line, which prints
# "| CUDA : ARCHS = ..." for a *compiled* backend even when the CUDA runtime
# failed to initialize at load time). Scanned only after a "device_info:" line.
_DEVICE_ROW_RE = re.compile(
r"-\s*(?P<dev>(?:CUDA|ROCm|ROCM|HIP|Metal|Vulkan|SYCL|OpenCL|MUSA|CANN|CPU)\w*)\s*:",
re.IGNORECASE,
)
# Keep in sync with _GPU_MODEL_BUFFER_MARKERS: every GPU backend matched by
# _DEVICE_ROW_RE must have a prefix here, or its device row counts as CPU-only
# and a working GPU build is wrongly rejected (OpenCL was the gap).
_GPU_DEVICE_PREFIXES = (
"cuda",
"rocm",
"hip",
"metal",
"vulkan",
"sycl",
"opencl",
"musa",
"cann",
)
# "load_tensors: offloaded 33/33 layers to GPU" (or "offloading ... to GPU").
_OFFLOADED_LAYERS_RE = re.compile(
r"offloaded\s+(\d+)\s*/\s*(\d+)\s+layers?\s+to\s+gpu", re.IGNORECASE
)
# install_kind values that ship a real GPU backend and must offload to the GPU.
# A binary launched with --n-gpu-layers 1 under one of these is held to the
# GPU-offload check; CPU kinds (windows-cpu, linux-cpu, ...) are exempt.
_GPU_INSTALL_KINDS = frozenset(
{"linux-cuda", "linux-rocm", "windows-cuda", "windows-hip", "macos-arm64"}
)
def server_log_shows_gpu_offload(log_text: str) -> bool | None:
"""Classify whether llama-server put the model on the GPU.
Robust across llama.cpp log formats (the "model buffer size" lines were
dropped in recent builds), in priority order:
1. ``<backend> model buffer size = N`` lines -- GPU marker => True
(older llama.cpp).
2. ``offloaded N/M layers to GPU`` -- N>0 => True, 0/M => False.
3. ``device_info:`` enumeration -- a GPU device row (``- CUDA0 :`` etc.)
=> True; only a ``- CPU :`` row => False. This is the version-stable
signal and matches how the CPU-only-binary bug is diagnosed in the
field (``device_info`` shows only CPU, CUDA0 missing -- #5830).
Returns True (GPU offload confirmed), False (started but stayed on CPU
despite GPU intent -- the silent fallback in #5807 / #5106), or None when
the log carries no usable signal so callers never reject on no evidence.
"""
lines = log_text.splitlines()
# Signal 1: per-backend buffer-size lines. A GPU marker on ANY "buffer
# size" line (model / KV / compute) means the GPU holds part of the model
# -- accept. Only the model-buffer location decides CPU-only (KV/compute
# CPU buffers exist even on GPU runs), so the False determination keys on
# "model buffer size" alone.
saw_buffer_line = False
for line in lines:
if "buffer size" not in line:
continue
if any(marker in line for marker in _GPU_MODEL_BUFFER_MARKERS):
return True
if "model buffer size" in line:
saw_buffer_line = True
# Signal 2: explicit offloaded-layers count.
for line in lines:
match = _OFFLOADED_LAYERS_RE.search(line)
if match:
return int(match.group(1)) > 0
low = line.lower()
if "offloading" in low and "to gpu" in low:
return True
# Signal 3: device_info enumeration. Only trust device rows once the
# "device_info:" header has appeared, so the compiled-backend system_info
# line ("| CUDA : ARCHS = ...") is never mistaken for an available device.
after_device_info = False
saw_device_row = False
for line in lines:
if "device_info:" in line:
after_device_info = True
continue
if not after_device_info:
continue
match = _DEVICE_ROW_RE.search(line)
if not match:
continue
saw_device_row = True
dev = match.group("dev").lower()
if any(dev.startswith(prefix) for prefix in _GPU_DEVICE_PREFIXES):
return True
if saw_buffer_line or saw_device_row:
# We had a concrete signal, but every buffer/device was CPU.
return False
return None
def read_full_log(log_path: Path, *, max_chars: int = 1_000_000) -> str:
"""Read a server log from the start (model-load buffer lines appear early,
so unlike read_log_excerpt this keeps the head rather than the tail)."""
try:
content = log_path.read_text(encoding = "utf-8", errors = "replace")
except FileNotFoundError:
return ""
return content[:max_chars]
def validate_server(
server_path: Path,
probe_path: Path,
@ -5615,13 +5759,7 @@ def validate_server(
# validation. Use the resolved install_kind as the source of
# truth and fall back to host detection when the caller did not
# pass one (keeps backwards compatibility with older call sites).
_gpu_kinds = {
"linux-cuda",
"linux-rocm",
"windows-cuda",
"windows-hip",
"macos-arm64",
}
_gpu_kinds = _GPU_INSTALL_KINDS
if install_kind is not None:
_enable_gpu_layers = install_kind in _gpu_kinds
else:
@ -5657,6 +5795,7 @@ def validate_server(
startup_started = time.time()
response_body = ""
last_error: Exception | None = None
completion_succeeded = False
while time.time() < deadline:
if process.poll() is not None:
process.wait(timeout = 5)
@ -5697,7 +5836,11 @@ def validate_server(
status_code = response.status
response_body = response.read().decode("utf-8", "replace")
if status_code == 200:
return
# Served a completion. Confirm real GPU offload
# below (outside this try, so GpuOffloadFailure is
# not swallowed by the except clauses).
completion_succeeded = True
break
last_error = RuntimeError(
f"unexpected HTTP status {status_code}"
)
@ -5717,6 +5860,34 @@ def validate_server(
+ output
+ ("\n" + response_body if response_body else "")
)
if completion_succeeded:
# The server served a completion. When GPU offload was
# requested, confirm the model actually landed on the GPU --
# a binary whose GPU backend failed to load (CPU-only build,
# unresolved cudart64_*/cublas64_* DLLs on Windows, or a
# PTX-only build on an older driver) still serves HTTP 200
# from CPU, and accepting it ships a silently CPU-only
# install (#5807, #5106). Reject so the resolver advances to
# the next bundle / source build instead of stopping here.
if _enable_gpu_layers:
log_handle.flush()
if (
server_log_shows_gpu_offload(read_full_log(log_path))
is False
):
raise GpuOffloadFailure(
"llama-server served a completion but loaded the "
"model entirely on CPU despite GPU offload being "
f"requested (install_kind={install_kind}). The "
"binary's GPU backend did not initialize -- on "
"Windows this is usually unresolved "
"cudart64_*.dll / cublas64_*.dll, or a build that "
"does not match the driver (PTX-only). Rejecting "
"so a GPU-capable bundle is selected "
"(unslothai/unsloth#5807, #5106):\n"
+ read_log_excerpt(log_path)
)
return
finally:
if process is not None and process.poll() is None:
process.terminate()
@ -6616,6 +6787,74 @@ def install_prebuilt(
raise SystemExit(EXIT_FALLBACK) from exc
def resolve_smoke_test_install_kind(host: HostInfo) -> str:
"""Best-effort install_kind for a post-build GPU smoke test, derived from
the host. setup.sh / setup.ps1 can override with --install-kind. GPU kinds
(in _GPU_INSTALL_KINDS) make the smoke test require real GPU offload; CPU
kinds skip that check."""
if host.is_macos and host.is_arm64:
return "macos-arm64"
if host.has_rocm:
return "windows-hip" if host.is_windows else "linux-rocm"
if host.has_usable_nvidia:
return "windows-cuda" if host.is_windows else "linux-cuda"
if host.is_windows:
return "windows-cpu"
if host.is_macos:
return "macos-cpu"
return "linux-cpu"
def smoke_test_server_binary(
server_binary: str,
host: HostInfo,
*,
install_dir: str | None = None,
probe: str | None = None,
install_kind: str | None = None,
) -> str:
"""Run validate_server against an already-built llama-server binary.
Raises PrebuiltFallback when the binary fails to start, fails the completion
probe, or (on a GPU host) loads the model only on CPU. Returns the resolved
install_kind on success.
"""
server_path = Path(server_binary).expanduser().resolve()
if not server_path.exists():
raise PrebuiltFallback(
f"smoke-test: llama-server binary not found at {server_path}"
)
resolved_install_dir = (
Path(install_dir).expanduser().resolve() if install_dir else server_path.parent
)
resolved_kind = install_kind or resolve_smoke_test_install_kind(host)
if probe:
probe_path = Path(probe).expanduser().resolve()
if not probe_path.exists():
raise PrebuiltFallback(f"smoke-test: probe model not found at {probe_path}")
validate_server(
server_path,
probe_path,
host,
resolved_install_dir,
install_kind = resolved_kind,
)
else:
with tempfile.TemporaryDirectory(prefix = "unsloth-llama-smoke-") as tmp:
probe_path = Path(tmp) / "stories260K.gguf"
download_validation_model(
probe_path, validation_model_cache_path(resolved_install_dir)
)
validate_server(
server_path,
probe_path,
host,
resolved_install_dir,
install_kind = resolved_kind,
)
return resolved_kind
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description = "Install and validate a prebuilt llama.cpp bundle for Unsloth Studio."
@ -6647,6 +6886,33 @@ def parse_args() -> argparse.Namespace:
action = "store_true",
help = "Use the simplified platform-specific prebuilt selection policy.",
)
parser.add_argument(
"--smoke-test",
metavar = "LLAMA_SERVER",
help = (
"Validate an already-built llama-server binary instead of "
"installing. Launches it against a tiny probe model and, on a GPU "
"host, verifies the model actually offloaded to the GPU. Exits 0 on "
"success, 2 (EXIT_FALLBACK) when the binary only ran on CPU, 1 "
"(EXIT_ERROR) when inconclusive -- used by setup.sh / setup.ps1 to "
"validate source builds."
),
)
parser.add_argument(
"--probe",
help = (
"Optional probe GGUF for --smoke-test. Defaults to the bundled "
"validation model (downloaded if absent)."
),
)
parser.add_argument(
"--install-kind",
help = (
"Override the install_kind used by --smoke-test (e.g. linux-cuda, "
"windows-cuda, linux-rocm, windows-hip, macos-arm64). Auto-derived "
"from the host when omitted."
),
)
parser.add_argument(
"--has-rocm",
action = "store_true",
@ -6788,6 +7054,39 @@ def main() -> int:
)
return EXIT_SUCCESS
if args.smoke_test is not None:
host = detect_host()
try:
resolved_kind = smoke_test_server_binary(
args.smoke_test,
host,
install_dir = args.install_dir,
probe = args.probe,
install_kind = args.install_kind,
)
except GpuOffloadFailure as exc:
# Definitive: GPU was requested but the model loaded on CPU. Exit
# EXIT_FALLBACK so the caller (setup.sh / setup.ps1) rebuilds CPU.
log(f"smoke-test failed (CPU-only): {exc}")
return EXIT_FALLBACK
except PrebuiltFallback as exc:
# Inconclusive: the binary could not be exercised (failed to start,
# missing probe, ...). Exit EXIT_ERROR so the caller keeps the GPU
# build rather than downgrading it on uncertain evidence.
log(f"smoke-test inconclusive: {exc}")
return EXIT_ERROR
if resolved_kind in _GPU_INSTALL_KINDS:
log(
f"smoke-test passed: {args.smoke_test} "
f"(install_kind={resolved_kind}) offloaded to the GPU"
)
else:
log(
f"smoke-test passed: {args.smoke_test} "
f"(install_kind={resolved_kind})"
)
return EXIT_SUCCESS
if not args.install_dir:
raise SystemExit(
"install_llama_prebuilt.py: --install-dir is required unless --resolve-llama-tag, --resolve-install-tag, or --resolve-source-build is used"

120
tests/studio/install/test_selection_logic.py
View file

@ -78,6 +78,7 @@ windows_cuda_upstream_asset_names = (
)
env_int = INSTALL_LLAMA_PREBUILT.env_int
direct_upstream_release_plan = INSTALL_LLAMA_PREBUILT.direct_upstream_release_plan
direct_linux_release_plan = INSTALL_LLAMA_PREBUILT.direct_linux_release_plan
_pinned_windows_cuda_fallback = INSTALL_LLAMA_PREBUILT._pinned_windows_cuda_fallback
CudaRuntimePreference = INSTALL_LLAMA_PREBUILT.CudaRuntimePreference
published_windows_cuda_attempts = INSTALL_LLAMA_PREBUILT.published_windows_cuda_attempts
@ -2297,11 +2298,14 @@ class TestDirectUpstreamBlackwellPin:
self._release(), host, UPSTREAM_REPO, "latest"
)
order = [(a.tag, a.runtime_line or a.install_kind) for a in plan.attempts]
# No windows-cpu tail: a GPU host must fall through to the source build
# when every CUDA bundle fails the offload check, not silently install
# the CPU bundle (#5807). The pin + cuda12 are the only attempts.
assert order == [
("b9360", "cuda13"),
(self.TAG, "cuda12"),
(self.TAG, "windows-cpu"),
]
assert all(a.install_kind != "windows-cpu" for a in plan.attempts)
assert plan.attempts[0].name == "llama-b9360-bin-win-cuda-13.1-x64.zip"
# Direct/upstream path stays unverified-by-manifest (no approved hashes).
assert plan.approved_checksums.artifacts == {}
@ -2324,6 +2328,120 @@ class TestDirectUpstreamBlackwellPin:
assert plan.attempts[0].name == f"llama-{self.TAG}-bin-win-cuda-13.3-x64.zip"
# ===========================================================================
# N.1c2. simple-policy plans: a GPU host never gets a CPU bundle appended
# (#5807). All GPU bundles failing must raise PrebuiltFallback so setup builds
# from source for the native arch, not silently install a CPU-only "GPU" build.
# ===========================================================================
class TestGpuHostNoSilentCpuFallback:
LTAG = "bTEST"
def _linux_release(self, *targets):
names = [f"app-{self.LTAG}-linux-x64-{t}.tar.gz" for t in targets]
return {
"tag_name": self.LTAG,
"assets": [
{"name": n, "browser_download_url": "https://x/" + n} for n in names
],
}
def _win_release(self):
names = [
"llama-b9999-bin-win-cuda-13.3-x64.zip",
"cudart-llama-bin-win-cuda-13.3-x64.zip",
"llama-b9999-bin-win-cpu-x64.zip",
]
return {
"tag_name": "b9999",
"assets": [
{"name": n, "browser_download_url": "https://x/" + n} for n in names
],
}
def _no_torch(self, monkeypatch):
monkeypatch.setattr(
INSTALL_LLAMA_PREBUILT,
"detect_torch_cuda_runtime_preference",
lambda host: CudaRuntimePreference(runtime_line = None, selection_log = []),
)
# -- Linux --------------------------------------------------------------
def test_linux_nvidia_host_has_no_cpu_attempt(self, monkeypatch):
mock_linux_runtime(monkeypatch, ["cuda13", "cuda12"])
self._no_torch(monkeypatch)
host = make_host(compute_caps = ["120"], driver_cuda_version = (13, 0))
plan = direct_linux_release_plan(
self._linux_release("cuda13-newer", "cpu"), host, "unslothai/llama.cpp", "latest"
)
kinds = [a.install_kind for a in plan.attempts]
assert "linux-cuda" in kinds
assert "linux-cpu" not in kinds
def test_linux_nvidia_unmatched_arch_raises_no_cpu(self, monkeypatch):
# GPU present but no CUDA bundle covers the arch: must raise (-> source
# build), not fall to the CPU bundle.
mock_linux_runtime(monkeypatch, ["cuda13", "cuda12"])
self._no_torch(monkeypatch)
host = make_host(compute_caps = ["50"], driver_cuda_version = (13, 0))
with pytest.raises(PrebuiltFallback):
direct_linux_release_plan(
self._linux_release("cuda13-newer", "cpu"),
host,
"unslothai/llama.cpp",
"latest",
)
def test_linux_cpu_host_keeps_cpu_attempt(self, monkeypatch):
mock_linux_runtime(monkeypatch, [])
host = make_host(
has_usable_nvidia = False,
has_physical_nvidia = False,
nvidia_smi = None,
compute_caps = [],
)
plan = direct_linux_release_plan(
self._linux_release("cuda13-newer", "cpu"), host, "unslothai/llama.cpp", "latest"
)
assert [a.install_kind for a in plan.attempts] == ["linux-cpu"]
# -- Windows ------------------------------------------------------------
def test_windows_nvidia_host_has_no_cpu_attempt(self, monkeypatch):
mock_windows_runtime(monkeypatch, ["cuda13"])
self._no_torch(monkeypatch)
host = make_host(
system = "Windows",
machine = "AMD64",
driver_cuda_version = (13, 3),
compute_caps = ["120"],
)
plan = direct_upstream_release_plan(
self._win_release(), host, UPSTREAM_REPO, "latest"
)
kinds = [a.install_kind for a in plan.attempts]
assert "windows-cuda" in kinds
assert "windows-cpu" not in kinds
def test_windows_cpu_host_keeps_cpu_attempt(self, monkeypatch):
mock_windows_runtime(monkeypatch, [])
host = make_host(
system = "Windows",
machine = "AMD64",
has_usable_nvidia = False,
has_physical_nvidia = False,
nvidia_smi = None,
driver_cuda_version = None,
compute_caps = [],
)
plan = direct_upstream_release_plan(
self._win_release(), host, UPSTREAM_REPO, "latest"
)
assert [a.install_kind for a in plan.attempts] == ["windows-cpu"]
# ===========================================================================
# N.1d. published_windows_cuda_attempts -- version-dynamic ordering seed
# ===========================================================================

450
tests/studio/install/test_validate_server_gpu_offload.py Normal file
View file

@ -0,0 +1,450 @@
"""Tests for the GPU-offload validation added to install_llama_prebuilt.py.
Covers issue unslothai/unsloth#5807 (duplicates #5830 / #5106 / #5827): a
llama-server whose GPU backend fails to initialize still serves HTTP 200 from
CPU, so the old validate_server accepted it and Studio shipped a silently
CPU-only install. The hardened validate_server rejects such a binary when GPU
offload was requested, so the resolver / source build falls through to a
GPU-capable bundle instead of a silently CPU-only one.
Stdlib-only; the installer module is spec-loaded by absolute path (no
PYTHONPATH / heavy deps), matching test_install_llama_prebuilt_logic.py.
"""
import importlib.util
import sys
from pathlib import Path
import pytest
PACKAGE_ROOT = Path(__file__).resolve().parents[3]
MODULE_PATH = PACKAGE_ROOT / "studio" / "install_llama_prebuilt.py"
SPEC = importlib.util.spec_from_file_location(
"studio_install_llama_prebuilt", MODULE_PATH
)
assert SPEC is not None and SPEC.loader is not None
M = importlib.util.module_from_spec(SPEC)
sys.modules[SPEC.name] = M
SPEC.loader.exec_module(M)
server_log_shows_gpu_offload = M.server_log_shows_gpu_offload
resolve_smoke_test_install_kind = M.resolve_smoke_test_install_kind
smoke_test_server_binary = M.smoke_test_server_binary
validate_server = M.validate_server
PrebuiltFallback = M.PrebuiltFallback
HostInfo = M.HostInfo
# -- HostInfo factories ------------------------------------------------------
def nvidia_host(**overrides) -> HostInfo:
defaults = dict(
system = "Linux",
machine = "x86_64",
is_windows = False,
is_linux = True,
is_macos = False,
is_x86_64 = True,
is_arm64 = False,
nvidia_smi = "/usr/bin/nvidia-smi",
driver_cuda_version = (13, 0),
compute_caps = ["120"],
visible_cuda_devices = None,
has_physical_nvidia = True,
has_usable_nvidia = True,
has_rocm = False,
)
defaults.update(overrides)
return HostInfo(**defaults)
def windows_nvidia_host(**overrides) -> HostInfo:
return nvidia_host(
system = "Windows",
is_windows = True,
is_linux = False,
nvidia_smi = "nvidia-smi",
**overrides,
)
def rocm_host(**overrides) -> HostInfo:
return nvidia_host(
nvidia_smi = None,
driver_cuda_version = None,
compute_caps = [],
has_physical_nvidia = False,
has_usable_nvidia = False,
has_rocm = True,
**overrides,
)
def macos_arm_host(**overrides) -> HostInfo:
defaults = dict(
system = "Darwin",
machine = "arm64",
is_windows = False,
is_linux = False,
is_macos = True,
is_x86_64 = False,
is_arm64 = True,
nvidia_smi = None,
driver_cuda_version = None,
compute_caps = [],
visible_cuda_devices = None,
has_physical_nvidia = False,
has_usable_nvidia = False,
has_rocm = False,
)
defaults.update(overrides)
return HostInfo(**defaults)
def cpu_host(**overrides) -> HostInfo:
return nvidia_host(
nvidia_smi = None,
driver_cuda_version = None,
compute_caps = [],
has_physical_nvidia = False,
has_usable_nvidia = False,
has_rocm = False,
**overrides,
)
# -- Canned llama-server logs ------------------------------------------------
# Current llama.cpp (>= mid-2026) dropped the "model buffer size" lines and
# enumerates a "device_info:" block instead; older builds still print the
# buffer lines. Both formats must classify correctly.
CUDA_DEVICE_INFO_LOG = (
"0.00.667 I log_info: verbosity = 3\n"
"0.00.667 I device_info:\n"
"0.01.101 I - CUDA0 : NVIDIA GeForce RTX 5070 (12282 MiB, 11000 MiB free)\n"
"0.01.101 I - CPU : AMD Ryzen 7 9700X (32000 MiB free)\n"
"0.01.101 I system_info: n_threads = 8 | CUDA : ARCHS = 1200 | CPU : AVX2 = 1\n"
"0.01.174 I srv llama_server: model loaded\n"
)
CPU_ONLY_DEVICE_INFO_LOG = (
"0.00.003 I log_info: verbosity = 3\n"
"0.00.003 I device_info:\n"
"0.00.003 I - CPU : AMD Ryzen 7 9700X (32000 MiB free)\n"
"0.00.003 I system_info: n_threads = 8 | CPU : AVX2 = 1\n"
"0.00.174 I srv llama_server: model loaded\n"
)
CUDA_BUFFER_LOG = (
"load_tensors: offloaded 33/33 layers to GPU\n"
"load_tensors: CUDA0 model buffer size = 21000.0 MiB\n"
"load_tensors: CPU_Mapped model buffer size = 0.6 MiB\n"
)
CPU_BUFFER_LOG = (
"load_tensors: CPU_Mapped model buffer size = 21000.0 MiB\n"
"load_tensors: CPU model buffer size = 0.6 MiB\n"
)
ROCM_BUFFER_LOG = "load_tensors: ROCm0 model buffer size = 21000.0 MiB\n"
METAL_BUFFER_LOG = "load_tensors: Metal model buffer size = 8000.0 MiB\n"
VULKAN_DEVICE_INFO_LOG = (
"device_info:\n - Vulkan0 : AMD Radeon (16000 MiB free)\n - CPU : x86 (free)\n"
)
OPENCL_DEVICE_INFO_LOG = (
"device_info:\n - OpenCL0 : Adreno (4000 MiB free)\n - CPU : arm (free)\n"
)
OFFLOADED_ZERO_LOG = "load_tensors: offloaded 0/33 layers to GPU\n"
NO_SIGNAL_LOG = "INFO [main] starting server\nload_tensors: file format = GGUF V3\n"
# system_info advertises a *compiled* backend; not an available device.
SYSTEM_INFO_ONLY_LOG = (
"system_info: n_threads = 8 | CUDA : ARCHS = 1200 | CPU : AVX2 = 1\n"
)
# -- 1. Pure classifier ------------------------------------------------------
@pytest.mark.parametrize(
"log_text,expected",
[
(CUDA_DEVICE_INFO_LOG, True),
(CPU_ONLY_DEVICE_INFO_LOG, False),
(CUDA_BUFFER_LOG, True),
(CPU_BUFFER_LOG, False),
(ROCM_BUFFER_LOG, True),
(METAL_BUFFER_LOG, True),
(VULKAN_DEVICE_INFO_LOG, True),
(OPENCL_DEVICE_INFO_LOG, True),
(OFFLOADED_ZERO_LOG, False),
(NO_SIGNAL_LOG, None),
("", None),
(SYSTEM_INFO_ONLY_LOG, None),
],
)
def test_server_log_shows_gpu_offload(log_text, expected):
assert server_log_shows_gpu_offload(log_text) is expected
def test_opencl_device_in_prefixes():
# Regression: OpenCL was matched by the device-row regex but missing from
# the GPU prefix list, so an OpenCL-only device_info wrongly classified as
# CPU-only and a working OpenCL build was rejected.
assert "opencl" in M._GPU_DEVICE_PREFIXES
assert server_log_shows_gpu_offload(OPENCL_DEVICE_INFO_LOG) is True
def test_system_info_not_mistaken_for_device():
# A CUDA-compiled binary whose runtime failed still prints the system_info
# CUDA backend, but enumerates only CPU under device_info -> must be False.
log = (
"device_info:\n - CPU : x86 (free)\n"
"system_info: | CUDA : ARCHS = 1200 | CPU : AVX2 = 1\n"
)
assert server_log_shows_gpu_offload(log) is False
def test_signal2_offloaded_zero_beats_device_info():
# offloaded 0/33 (signal 2) is a definitive "nothing on GPU" and takes
# priority over a device_info CUDA0 row -> False.
log = (
"load_tensors: offloaded 0/33 layers to GPU\n"
"device_info:\n - CUDA0 : NVIDIA (free)\n - CPU : x (free)\n"
)
assert server_log_shows_gpu_offload(log) is False
def test_gpu_buffer_size_without_model_word():
# A GPU-marked buffer line that omits the word "model" (e.g. KV/compute or
# a future format) must still count as GPU offload, even when a CPU
# "model buffer size" line is present (broadened signal 1).
log = (
"load_tensors: CUDA0 KV buffer size = 100.0 MiB\n"
"load_tensors: CPU_Mapped model buffer size = 0.6 MiB\n"
)
assert server_log_shows_gpu_offload(log) is True
def test_device_row_case_insensitive():
log = "device_info:\n - cuda0 : some gpu (free)\n - CPU : x (free)\n"
assert server_log_shows_gpu_offload(log) is True
def test_crlf_log_parses_identically():
# Windows logs use CRLF; classification must not change.
assert server_log_shows_gpu_offload(CUDA_DEVICE_INFO_LOG.replace("\n", "\r\n")) is True
assert (
server_log_shows_gpu_offload(CPU_ONLY_DEVICE_INFO_LOG.replace("\n", "\r\n"))
is False
)
# -- 2. resolve_smoke_test_install_kind --------------------------------------
@pytest.mark.parametrize(
"host,expected",
[
(nvidia_host(), "linux-cuda"),
(windows_nvidia_host(), "windows-cuda"),
(rocm_host(), "linux-rocm"),
(rocm_host(system = "Windows", is_windows = True, is_linux = False), "windows-hip"),
(macos_arm_host(), "macos-arm64"),
(cpu_host(), "linux-cpu"),
(cpu_host(system = "Windows", is_windows = True, is_linux = False), "windows-cpu"),
],
)
def test_resolve_smoke_test_install_kind(host, expected):
assert resolve_smoke_test_install_kind(host) == expected
if expected.endswith("-cpu"):
assert expected not in M._GPU_INSTALL_KINDS
else:
assert expected in M._GPU_INSTALL_KINDS
# -- 3. validate_server integration (mocked subprocess + HTTP) ---------------
class _FakeResponse:
def __init__(self, status: int, body: bytes = b"{}"):
self.status = status
self._body = body
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def read(self):
return self._body
class _FakeProc:
"""Stands in for the llama-server subprocess: writes a canned startup log
to the handle it was given, then reports itself alive until terminated."""
def __init__(self, log_text: str):
self._log_text = log_text
self._alive = True
def __call__(self, command, *args, **kwargs):
handle = kwargs.get("stdout")
if handle is not None:
handle.write(self._log_text)
handle.flush()
return self
def poll(self):
return None if self._alive else 0
def wait(self, timeout = None):
self._alive = False
return 0
def terminate(self):
self._alive = False
def kill(self):
self._alive = False
@pytest.fixture
def patched_server(monkeypatch):
"""Patch validate_server's external touchpoints; return a configure()
helper so each test picks the canned log it wants."""
monkeypatch.setattr(M, "free_local_port", lambda: 18000)
monkeypatch.setattr(M, "binary_env", lambda *a, **k: {})
monkeypatch.setattr(M.time, "sleep", lambda *a, **k: None)
monkeypatch.setattr(M.urllib.request, "urlopen", lambda *a, **k: _FakeResponse(200))
def configure(log_text):
monkeypatch.setattr(M.subprocess, "Popen", _FakeProc(log_text))
return configure
def _run_validate(tmp_path, host, install_kind):
server = tmp_path / "llama-server"
server.write_text("#!/bin/sh\n")
probe = tmp_path / "probe.gguf"
probe.write_bytes(b"GGUF")
validate_server(server, probe, host, tmp_path, install_kind = install_kind)
def test_gpu_intent_cpu_only_rejected(patched_server, tmp_path):
# The #5807 bug: GPU requested, binary ran on CPU -> reject. The exception
# must be GpuOffloadFailure specifically (not just the PrebuiltFallback
# base) so the --smoke-test CLI can map it to EXIT_FALLBACK vs EXIT_ERROR.
patched_server(CPU_ONLY_DEVICE_INFO_LOG)
with pytest.raises(M.GpuOffloadFailure, match = "entirely on CPU"):
_run_validate(tmp_path, windows_nvidia_host(), "windows-cuda")
assert issubclass(M.GpuOffloadFailure, PrebuiltFallback)
def test_gpu_intent_gpu_offload_accepted(patched_server, tmp_path):
patched_server(CUDA_DEVICE_INFO_LOG)
_run_validate(tmp_path, windows_nvidia_host(), "windows-cuda") # no raise
def test_cpu_kind_not_gpu_gated(patched_server, tmp_path):
# A windows-cpu bundle is the intentional fallback; never GPU-gated even
# if it (obviously) loads on CPU.
patched_server(CPU_ONLY_DEVICE_INFO_LOG)
_run_validate(tmp_path, windows_nvidia_host(), "windows-cpu") # no raise
def test_gpu_intent_no_signal_accepted(patched_server, tmp_path):
# No buffer/device signal -> conservative: do not reject on no evidence.
patched_server(NO_SIGNAL_LOG)
_run_validate(tmp_path, nvidia_host(), "linux-cuda") # no raise
def test_rocm_cpu_only_rejected(patched_server, tmp_path):
patched_server(CPU_ONLY_DEVICE_INFO_LOG)
with pytest.raises(PrebuiltFallback):
_run_validate(tmp_path, rocm_host(), "linux-rocm")
# -- 4. smoke_test_server_binary ---------------------------------------------
def test_smoke_test_missing_binary(tmp_path):
with pytest.raises(PrebuiltFallback, match = "not found"):
smoke_test_server_binary(
str(tmp_path / "nope"), nvidia_host(), install_dir = str(tmp_path)
)
def test_smoke_test_delegates_to_validate_server(monkeypatch, tmp_path):
server = tmp_path / "llama-server"
server.write_text("#!/bin/sh\n")
probe = tmp_path / "probe.gguf"
probe.write_bytes(b"GGUF")
seen = {}
def fake_validate(server_path, probe_path, host, install_dir, **kw):
seen["install_kind"] = kw.get("install_kind")
monkeypatch.setattr(M, "validate_server", fake_validate)
kind = smoke_test_server_binary(
str(server), nvidia_host(), install_dir = str(tmp_path), probe = str(probe)
)
assert kind == "linux-cuda"
assert seen["install_kind"] == "linux-cuda"
def test_smoke_test_propagates_failure(monkeypatch, tmp_path):
server = tmp_path / "llama-server"
server.write_text("#!/bin/sh\n")
probe = tmp_path / "probe.gguf"
probe.write_bytes(b"GGUF")
def fake_validate(*a, **k):
raise PrebuiltFallback("loaded the model entirely on CPU")
monkeypatch.setattr(M, "validate_server", fake_validate)
with pytest.raises(PrebuiltFallback):
smoke_test_server_binary(
str(server),
nvidia_host(),
install_dir = str(tmp_path),
probe = str(probe),
)
# -- 5. --smoke-test CLI exit-code contract (the load-bearing fix contract) --
# setup.sh / setup.ps1 branch on: 2 = ran on CPU (rebuild CPU), 1 = inconclusive
# (keep GPU build), 0 = offload confirmed.
def _run_main_smoke(monkeypatch, smoke_impl):
monkeypatch.setattr(M, "detect_host", lambda: nvidia_host())
monkeypatch.setattr(M, "smoke_test_server_binary", smoke_impl)
monkeypatch.setattr(
sys, "argv", ["install_llama_prebuilt.py", "--smoke-test", "/x/llama-server"]
)
return M.main()
def test_main_smoke_exit_success(monkeypatch):
rc = _run_main_smoke(monkeypatch, lambda *a, **k: "linux-cuda")
assert rc == M.EXIT_SUCCESS
def test_main_smoke_exit_fallback_on_cpu_only(monkeypatch):
def impl(*a, **k):
raise M.GpuOffloadFailure("loaded the model entirely on CPU")
rc = _run_main_smoke(monkeypatch, impl)
assert rc == M.EXIT_FALLBACK # 2 -> setup scripts rebuild CPU
def test_main_smoke_exit_error_on_inconclusive(monkeypatch):
def impl(*a, **k):
raise PrebuiltFallback("llama-server exited during startup")
rc = _run_main_smoke(monkeypatch, impl)
assert rc == M.EXIT_ERROR # 1 -> setup scripts keep the GPU build