From 2f88ecd8a431f2432a891fe76fbabd4c40865968 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 1 Jun 2026 15:44:26 +0000 Subject: [PATCH 01/12] Studio: validate real GPU offload and stop silently shipping CPU-only GGUF A llama-server whose GPU backend fails to initialize still serves HTTP 200 from CPU, so install validation accepted it and Studio ran 'GPU' inference on CPU (#5807/#5106/#5830). Add a log classifier (buffer-size, offloaded-layers, and device_info signals) and reject a GPU-intended binary that loaded on CPU, so the resolver advances. Crucially, a GPU host no longer gets a CPU prebuilt appended to its simple-policy attempts, so when no GPU bundle offloads the installer falls through to a source build for the native arch instead of silently installing CPU. Add a --smoke-test CLI for setup scripts and align the runtime classifier with the same signals so its CPU-only warning fires on current llama.cpp. --- studio/backend/core/inference/llama_cpp.py | 94 +++- .../tests/test_llama_cpp_context_fit.py | 42 ++ studio/install_llama_prebuilt.py | 341 ++++++++++++- tests/studio/install/test_selection_logic.py | 120 ++++- .../test_validate_server_gpu_offload.py | 450 ++++++++++++++++++ 5 files changed, 1006 insertions(+), 41 deletions(-) create mode 100644 tests/studio/install/test_validate_server_gpu_offload.py diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 7bcf02dc35..9001396afe 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -59,6 +59,75 @@ from core.inference.tool_call_parser import ( logger = get_logger(__name__) +# ── GPU-offload log classifier (shared shape with install_llama_prebuilt) ── +# llama-server can serve HTTP 200 while running entirely on CPU; detect that so +# Studio can warn instead of silently running a "GPU" model on CPU (#5807 / +# #5106 / #5830). Robust across llama.cpp log formats (the "model buffer size" +# lines were dropped in recent builds): buffer-size markers, "offloaded N/M +# layers to GPU", and the "device_info:" enumeration. +_GPU_BUFFER_MARKERS = ("CUDA", "ROCm", "Metal", "Vulkan", "OpenCL", "SYCL") +_DEVICE_ROW_RE = re.compile( + r"-\s*(?P(?:CUDA|ROCm|ROCM|HIP|Metal|Vulkan|SYCL|OpenCL|MUSA|CANN|CPU)\w*)\s*:", + re.IGNORECASE, +) +_GPU_DEVICE_PREFIXES = ( + "cuda", + "rocm", + "hip", + "metal", + "vulkan", + "sycl", + "opencl", + "musa", + "cann", +) +_OFFLOADED_LAYERS_RE = re.compile( + r"offloaded\s+(\d+)\s*/\s*(\d+)\s+layers?\s+to\s+gpu", re.IGNORECASE +) + + +def classify_gpu_offload_lines(lines: list[str]) -> Optional[bool]: + """True if the model landed on a GPU, False if it stayed on CPU despite GPU + intent, None when the log has no usable signal. Priority: buffer-size lines, + then offloaded-layers count, then device_info enumeration.""" + saw_buffer_line = False + for line in lines: + if "buffer size" not in line: + continue + if any(marker in line for marker in _GPU_BUFFER_MARKERS): + return True + if "model buffer size" in line: + saw_buffer_line = True + + for line in lines: + match = _OFFLOADED_LAYERS_RE.search(line) + if match: + return int(match.group(1)) > 0 + low = line.lower() + if "offloading" in low and "to gpu" in low: + return True + + after_device_info = False + saw_device_row = False + for line in lines: + if "device_info:" in line: + after_device_info = True + continue + if not after_device_info: + continue + match = _DEVICE_ROW_RE.search(line) + if not match: + continue + saw_device_row = True + dev = match.group("dev").lower() + if any(dev.startswith(prefix) for prefix in _GPU_DEVICE_PREFIXES): + return True + + if saw_buffer_line or saw_device_row: + return False + return None + + # ── Pre-compiled patterns for plan-without-action re-prompt ── # Forward-looking intent signals that indicate the model is # describing what it *will* do rather than giving a final answer. @@ -3865,27 +3934,14 @@ class LlamaCppBackend: expected_gpu: bool, detected_gpus: list[tuple[int, int]], ) -> Optional[bool]: - """True if a GPU model buffer was allocated, False if only CPU - buffers landed despite GPU intent, None when there's no signal - (no GPU detected, no buffer-size lines, etc.).""" + """True if the model landed on a GPU, False if only CPU buffers landed + despite GPU intent, None when there's no signal (no GPU detected, no + usable log lines, etc.). Delegates to the shared classifier so it tracks + current llama.cpp logs (device_info / offloaded-layers), which dropped + the older ``model buffer size`` lines this used to key on.""" if not detected_gpus or not expected_gpu: return None - # llama-server logs one ``... model buffer size = N MiB`` line - # per backend buffer; CUDA0 / ROCm0 / Metal / Vulkan0 / - # OpenCL0 / SYCL0 are GPU, CPU / CPU_Mapped are not. - gpu_markers = ("CUDA", "ROCm", "Metal", "Vulkan", "OpenCL", "SYCL") - saw_buffer_line = False - saw_gpu_buffer = False - for line in self._stdout_lines: - if "model buffer size" not in line: - continue - saw_buffer_line = True - if any(marker in line for marker in gpu_markers): - saw_gpu_buffer = True - break - if not saw_buffer_line: - return None - return saw_gpu_buffer + return classify_gpu_offload_lines(self._stdout_lines) def unload_model(self) -> bool: """Terminate the llama-server subprocess and cancel any in-flight download.""" diff --git a/studio/backend/tests/test_llama_cpp_context_fit.py b/studio/backend/tests/test_llama_cpp_context_fit.py index 6fe5372147..065c9726e4 100644 --- a/studio/backend/tests/test_llama_cpp_context_fit.py +++ b/studio/backend/tests/test_llama_cpp_context_fit.py @@ -577,3 +577,45 @@ class TestClassifyGpuOffload: ] ) assert inst._classify_gpu_offload(True, [(0, 22805)]) is True + + def test_current_device_info_cuda_returns_true(self): + # Recent llama.cpp dropped the buffer lines and prints device_info. + inst = self._backend( + [ + "0.00 I device_info:", + "0.01 I - CUDA0 : NVIDIA GeForce RTX 5070 (12282 MiB free)", + "0.01 I - CPU : AMD Ryzen 7 9700X (32000 MiB free)", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 12282)]) is True + + def test_current_device_info_cpu_only_returns_false(self): + # The #5830 symptom: device_info shows only CPU, CUDA0 missing. + inst = self._backend( + [ + "0.00 I device_info:", + "0.00 I - CPU : AMD Ryzen 7 9700X (32000 MiB free)", + "0.00 I srv llama_server: model loaded", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 12282)]) is False + + def test_offloaded_layers_count_decides(self): + assert ( + self._backend(["load_tensors: offloaded 33/33 layers to GPU"]) + ._classify_gpu_offload(True, [(0, 22805)]) + is True + ) + assert ( + self._backend(["load_tensors: offloaded 0/33 layers to GPU"]) + ._classify_gpu_offload(True, [(0, 22805)]) + is False + ) + + def test_system_info_only_returns_none(self): + # A compiled CUDA backend advertised in system_info is not proof of an + # available device; without device_info/buffer/offload signal -> None. + inst = self._backend( + ["system_info: n_threads = 8 | CUDA : ARCHS = 1200 | CPU : AVX2 = 1"] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is None diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index 62b7b2b290..151c459924 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -452,6 +452,14 @@ class PrebuiltFallback(RuntimeError): pass +class GpuOffloadFailure(PrebuiltFallback): + """The binary started and served a completion but loaded the model only on + CPU despite GPU offload being requested. A subclass of PrebuiltFallback so + the prebuilt resolver still advances to the next candidate, while the + --smoke-test CLI can tell this apart from an inconclusive start/serve + failure (definite CPU-only -> EXIT_FALLBACK; inconclusive -> EXIT_ERROR).""" + + class BusyInstallConflict(RuntimeError): pass @@ -1437,7 +1445,14 @@ def direct_linux_release_plan( ) if lemonade_choice is not None: attempts.append(lemonade_choice) - else: + elif not host.has_usable_nvidia: + # CPU-only host (or --cpu-fallback, which zeroes the GPU flags): ship the + # CPU bundle. Do NOT append it for a usable-NVIDIA host: if every CUDA + # bundle fails the GPU-offload check, let install raise PrebuiltFallback + # so setup.sh builds llama.cpp from source for the native arch (which + # offloads) instead of silently shipping a CPU-only "GPU" install + # (#5807). The CPU prebuilt stays the last resort after the source build, + # reached via setup.sh's --cpu-fallback path. cpu_choice = published_asset_choice_for_kind(bundle, "linux-cpu") if cpu_choice is not None: attempts.append(cpu_choice) @@ -1530,19 +1545,26 @@ def direct_upstream_release_plan( install_kind = "windows-hip", ) ) - cpu_asset = f"llama-{release_tag}-bin-win-cpu-x64.zip" - cpu_url = assets.get(cpu_asset) - if cpu_url: - attempts.append( - AssetChoice( - repo = repo, - tag = release_tag, - name = cpu_asset, - url = cpu_url, - source_label = "upstream", - install_kind = "windows-cpu", + # Append the CPU bundle only for a host with no usable GPU (or + # --cpu-fallback, which zeroes the GPU flags). On a GPU host, omitting it + # lets install raise PrebuiltFallback when every GPU bundle fails the + # offload check, so setup.ps1 builds from source / falls back to the CPU + # prebuilt as a labelled last resort instead of silently shipping a + # CPU-only "GPU" install (#5807). + if not host.has_usable_nvidia and not host.has_rocm: + cpu_asset = f"llama-{release_tag}-bin-win-cpu-x64.zip" + cpu_url = assets.get(cpu_asset) + if cpu_url: + attempts.append( + AssetChoice( + repo = repo, + tag = release_tag, + name = cpu_asset, + url = cpu_url, + source_label = "upstream", + install_kind = "windows-cpu", + ) ) - ) elif host.is_windows and host.is_arm64: # Upstream ggml-org/llama.cpp ships llama-bNNNN-bin-win-cpu-arm64.zip # (visible in the b9334 release manifest). Without this branch the @@ -5577,6 +5599,128 @@ def validate_quantize( ) +# GPU backend markers that appear in llama-server's "... model buffer size ..." +# load lines (older llama.cpp) and in its "device_info:" enumeration (current +# llama.cpp). CUDA0 / ROCm0 / Metal / Vulkan0 / OpenCL0 / SYCL0 / HIP0 / MUSA0 +# / CANN0 are GPU; CPU / CPU_Mapped are not. Kept broad so a single helper +# covers every backend Studio ships. +_GPU_MODEL_BUFFER_MARKERS = ("CUDA", "ROCm", "Metal", "Vulkan", "OpenCL", "SYCL") + +# A device_info row looks like " I - CUDA0 : NVIDIA B200 (...)" or +# " I - CPU : ...". Matched on the "- :" shape (the leading +# "- " distinguishes it from the system_info line, which prints +# "| CUDA : ARCHS = ..." for a *compiled* backend even when the CUDA runtime +# failed to initialize at load time). Scanned only after a "device_info:" line. +_DEVICE_ROW_RE = re.compile( + r"-\s*(?P(?:CUDA|ROCm|ROCM|HIP|Metal|Vulkan|SYCL|OpenCL|MUSA|CANN|CPU)\w*)\s*:", + re.IGNORECASE, +) +# Keep in sync with _GPU_MODEL_BUFFER_MARKERS: every GPU backend matched by +# _DEVICE_ROW_RE must have a prefix here, or its device row counts as CPU-only +# and a working GPU build is wrongly rejected (OpenCL was the gap). +_GPU_DEVICE_PREFIXES = ( + "cuda", + "rocm", + "hip", + "metal", + "vulkan", + "sycl", + "opencl", + "musa", + "cann", +) + +# "load_tensors: offloaded 33/33 layers to GPU" (or "offloading ... to GPU"). +_OFFLOADED_LAYERS_RE = re.compile( + r"offloaded\s+(\d+)\s*/\s*(\d+)\s+layers?\s+to\s+gpu", re.IGNORECASE +) + +# install_kind values that ship a real GPU backend and must offload to the GPU. +# A binary launched with --n-gpu-layers 1 under one of these is held to the +# GPU-offload check; CPU kinds (windows-cpu, linux-cpu, ...) are exempt. +_GPU_INSTALL_KINDS = frozenset( + {"linux-cuda", "linux-rocm", "windows-cuda", "windows-hip", "macos-arm64"} +) + + +def server_log_shows_gpu_offload(log_text: str) -> bool | None: + """Classify whether llama-server put the model on the GPU. + + Robust across llama.cpp log formats (the "model buffer size" lines were + dropped in recent builds), in priority order: + + 1. `` model buffer size = N`` lines -- GPU marker => True + (older llama.cpp). + 2. ``offloaded N/M layers to GPU`` -- N>0 => True, 0/M => False. + 3. ``device_info:`` enumeration -- a GPU device row (``- CUDA0 :`` etc.) + => True; only a ``- CPU :`` row => False. This is the version-stable + signal and matches how the CPU-only-binary bug is diagnosed in the + field (``device_info`` shows only CPU, CUDA0 missing -- #5830). + + Returns True (GPU offload confirmed), False (started but stayed on CPU + despite GPU intent -- the silent fallback in #5807 / #5106), or None when + the log carries no usable signal so callers never reject on no evidence. + """ + lines = log_text.splitlines() + + # Signal 1: per-backend buffer-size lines. A GPU marker on ANY "buffer + # size" line (model / KV / compute) means the GPU holds part of the model + # -- accept. Only the model-buffer location decides CPU-only (KV/compute + # CPU buffers exist even on GPU runs), so the False determination keys on + # "model buffer size" alone. + saw_buffer_line = False + for line in lines: + if "buffer size" not in line: + continue + if any(marker in line for marker in _GPU_MODEL_BUFFER_MARKERS): + return True + if "model buffer size" in line: + saw_buffer_line = True + + # Signal 2: explicit offloaded-layers count. + for line in lines: + match = _OFFLOADED_LAYERS_RE.search(line) + if match: + return int(match.group(1)) > 0 + low = line.lower() + if "offloading" in low and "to gpu" in low: + return True + + # Signal 3: device_info enumeration. Only trust device rows once the + # "device_info:" header has appeared, so the compiled-backend system_info + # line ("| CUDA : ARCHS = ...") is never mistaken for an available device. + after_device_info = False + saw_device_row = False + for line in lines: + if "device_info:" in line: + after_device_info = True + continue + if not after_device_info: + continue + match = _DEVICE_ROW_RE.search(line) + if not match: + continue + saw_device_row = True + dev = match.group("dev").lower() + if any(dev.startswith(prefix) for prefix in _GPU_DEVICE_PREFIXES): + return True + + if saw_buffer_line or saw_device_row: + # We had a concrete signal, but every buffer/device was CPU. + return False + return None + + +def read_full_log(log_path: Path, *, max_chars: int = 1_000_000) -> str: + """Read a server log from the start (model-load buffer lines appear early, + so unlike read_log_excerpt this keeps the head rather than the tail).""" + try: + content = log_path.read_text(encoding = "utf-8", errors = "replace") + except FileNotFoundError: + return "" + return content[:max_chars] + + def validate_server( server_path: Path, probe_path: Path, @@ -5615,13 +5759,7 @@ def validate_server( # validation. Use the resolved install_kind as the source of # truth and fall back to host detection when the caller did not # pass one (keeps backwards compatibility with older call sites). - _gpu_kinds = { - "linux-cuda", - "linux-rocm", - "windows-cuda", - "windows-hip", - "macos-arm64", - } + _gpu_kinds = _GPU_INSTALL_KINDS if install_kind is not None: _enable_gpu_layers = install_kind in _gpu_kinds else: @@ -5657,6 +5795,7 @@ def validate_server( startup_started = time.time() response_body = "" last_error: Exception | None = None + completion_succeeded = False while time.time() < deadline: if process.poll() is not None: process.wait(timeout = 5) @@ -5697,7 +5836,11 @@ def validate_server( status_code = response.status response_body = response.read().decode("utf-8", "replace") if status_code == 200: - return + # Served a completion. Confirm real GPU offload + # below (outside this try, so GpuOffloadFailure is + # not swallowed by the except clauses). + completion_succeeded = True + break last_error = RuntimeError( f"unexpected HTTP status {status_code}" ) @@ -5717,6 +5860,34 @@ def validate_server( + output + ("\n" + response_body if response_body else "") ) + if completion_succeeded: + # The server served a completion. When GPU offload was + # requested, confirm the model actually landed on the GPU -- + # a binary whose GPU backend failed to load (CPU-only build, + # unresolved cudart64_*/cublas64_* DLLs on Windows, or a + # PTX-only build on an older driver) still serves HTTP 200 + # from CPU, and accepting it ships a silently CPU-only + # install (#5807, #5106). Reject so the resolver advances to + # the next bundle / source build instead of stopping here. + if _enable_gpu_layers: + log_handle.flush() + if ( + server_log_shows_gpu_offload(read_full_log(log_path)) + is False + ): + raise GpuOffloadFailure( + "llama-server served a completion but loaded the " + "model entirely on CPU despite GPU offload being " + f"requested (install_kind={install_kind}). The " + "binary's GPU backend did not initialize -- on " + "Windows this is usually unresolved " + "cudart64_*.dll / cublas64_*.dll, or a build that " + "does not match the driver (PTX-only). Rejecting " + "so a GPU-capable bundle is selected " + "(unslothai/unsloth#5807, #5106):\n" + + read_log_excerpt(log_path) + ) + return finally: if process is not None and process.poll() is None: process.terminate() @@ -6616,6 +6787,74 @@ def install_prebuilt( raise SystemExit(EXIT_FALLBACK) from exc +def resolve_smoke_test_install_kind(host: HostInfo) -> str: + """Best-effort install_kind for a post-build GPU smoke test, derived from + the host. setup.sh / setup.ps1 can override with --install-kind. GPU kinds + (in _GPU_INSTALL_KINDS) make the smoke test require real GPU offload; CPU + kinds skip that check.""" + if host.is_macos and host.is_arm64: + return "macos-arm64" + if host.has_rocm: + return "windows-hip" if host.is_windows else "linux-rocm" + if host.has_usable_nvidia: + return "windows-cuda" if host.is_windows else "linux-cuda" + if host.is_windows: + return "windows-cpu" + if host.is_macos: + return "macos-cpu" + return "linux-cpu" + + +def smoke_test_server_binary( + server_binary: str, + host: HostInfo, + *, + install_dir: str | None = None, + probe: str | None = None, + install_kind: str | None = None, +) -> str: + """Run validate_server against an already-built llama-server binary. + + Raises PrebuiltFallback when the binary fails to start, fails the completion + probe, or (on a GPU host) loads the model only on CPU. Returns the resolved + install_kind on success. + """ + server_path = Path(server_binary).expanduser().resolve() + if not server_path.exists(): + raise PrebuiltFallback( + f"smoke-test: llama-server binary not found at {server_path}" + ) + resolved_install_dir = ( + Path(install_dir).expanduser().resolve() if install_dir else server_path.parent + ) + resolved_kind = install_kind or resolve_smoke_test_install_kind(host) + if probe: + probe_path = Path(probe).expanduser().resolve() + if not probe_path.exists(): + raise PrebuiltFallback(f"smoke-test: probe model not found at {probe_path}") + validate_server( + server_path, + probe_path, + host, + resolved_install_dir, + install_kind = resolved_kind, + ) + else: + with tempfile.TemporaryDirectory(prefix = "unsloth-llama-smoke-") as tmp: + probe_path = Path(tmp) / "stories260K.gguf" + download_validation_model( + probe_path, validation_model_cache_path(resolved_install_dir) + ) + validate_server( + server_path, + probe_path, + host, + resolved_install_dir, + install_kind = resolved_kind, + ) + return resolved_kind + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description = "Install and validate a prebuilt llama.cpp bundle for Unsloth Studio." @@ -6647,6 +6886,33 @@ def parse_args() -> argparse.Namespace: action = "store_true", help = "Use the simplified platform-specific prebuilt selection policy.", ) + parser.add_argument( + "--smoke-test", + metavar = "LLAMA_SERVER", + help = ( + "Validate an already-built llama-server binary instead of " + "installing. Launches it against a tiny probe model and, on a GPU " + "host, verifies the model actually offloaded to the GPU. Exits 0 on " + "success, 2 (EXIT_FALLBACK) when the binary only ran on CPU, 1 " + "(EXIT_ERROR) when inconclusive -- used by setup.sh / setup.ps1 to " + "validate source builds." + ), + ) + parser.add_argument( + "--probe", + help = ( + "Optional probe GGUF for --smoke-test. Defaults to the bundled " + "validation model (downloaded if absent)." + ), + ) + parser.add_argument( + "--install-kind", + help = ( + "Override the install_kind used by --smoke-test (e.g. linux-cuda, " + "windows-cuda, linux-rocm, windows-hip, macos-arm64). Auto-derived " + "from the host when omitted." + ), + ) parser.add_argument( "--has-rocm", action = "store_true", @@ -6788,6 +7054,39 @@ def main() -> int: ) return EXIT_SUCCESS + if args.smoke_test is not None: + host = detect_host() + try: + resolved_kind = smoke_test_server_binary( + args.smoke_test, + host, + install_dir = args.install_dir, + probe = args.probe, + install_kind = args.install_kind, + ) + except GpuOffloadFailure as exc: + # Definitive: GPU was requested but the model loaded on CPU. Exit + # EXIT_FALLBACK so the caller (setup.sh / setup.ps1) rebuilds CPU. + log(f"smoke-test failed (CPU-only): {exc}") + return EXIT_FALLBACK + except PrebuiltFallback as exc: + # Inconclusive: the binary could not be exercised (failed to start, + # missing probe, ...). Exit EXIT_ERROR so the caller keeps the GPU + # build rather than downgrading it on uncertain evidence. + log(f"smoke-test inconclusive: {exc}") + return EXIT_ERROR + if resolved_kind in _GPU_INSTALL_KINDS: + log( + f"smoke-test passed: {args.smoke_test} " + f"(install_kind={resolved_kind}) offloaded to the GPU" + ) + else: + log( + f"smoke-test passed: {args.smoke_test} " + f"(install_kind={resolved_kind})" + ) + return EXIT_SUCCESS + if not args.install_dir: raise SystemExit( "install_llama_prebuilt.py: --install-dir is required unless --resolve-llama-tag, --resolve-install-tag, or --resolve-source-build is used" diff --git a/tests/studio/install/test_selection_logic.py b/tests/studio/install/test_selection_logic.py index 2a94039018..d02e94e58e 100644 --- a/tests/studio/install/test_selection_logic.py +++ b/tests/studio/install/test_selection_logic.py @@ -78,6 +78,7 @@ windows_cuda_upstream_asset_names = ( ) env_int = INSTALL_LLAMA_PREBUILT.env_int direct_upstream_release_plan = INSTALL_LLAMA_PREBUILT.direct_upstream_release_plan +direct_linux_release_plan = INSTALL_LLAMA_PREBUILT.direct_linux_release_plan _pinned_windows_cuda_fallback = INSTALL_LLAMA_PREBUILT._pinned_windows_cuda_fallback CudaRuntimePreference = INSTALL_LLAMA_PREBUILT.CudaRuntimePreference published_windows_cuda_attempts = INSTALL_LLAMA_PREBUILT.published_windows_cuda_attempts @@ -2297,11 +2298,14 @@ class TestDirectUpstreamBlackwellPin: self._release(), host, UPSTREAM_REPO, "latest" ) order = [(a.tag, a.runtime_line or a.install_kind) for a in plan.attempts] + # No windows-cpu tail: a GPU host must fall through to the source build + # when every CUDA bundle fails the offload check, not silently install + # the CPU bundle (#5807). The pin + cuda12 are the only attempts. assert order == [ ("b9360", "cuda13"), (self.TAG, "cuda12"), - (self.TAG, "windows-cpu"), ] + assert all(a.install_kind != "windows-cpu" for a in plan.attempts) assert plan.attempts[0].name == "llama-b9360-bin-win-cuda-13.1-x64.zip" # Direct/upstream path stays unverified-by-manifest (no approved hashes). assert plan.approved_checksums.artifacts == {} @@ -2324,6 +2328,120 @@ class TestDirectUpstreamBlackwellPin: assert plan.attempts[0].name == f"llama-{self.TAG}-bin-win-cuda-13.3-x64.zip" +# =========================================================================== +# N.1c2. simple-policy plans: a GPU host never gets a CPU bundle appended +# (#5807). All GPU bundles failing must raise PrebuiltFallback so setup builds +# from source for the native arch, not silently install a CPU-only "GPU" build. +# =========================================================================== + + +class TestGpuHostNoSilentCpuFallback: + LTAG = "bTEST" + + def _linux_release(self, *targets): + names = [f"app-{self.LTAG}-linux-x64-{t}.tar.gz" for t in targets] + return { + "tag_name": self.LTAG, + "assets": [ + {"name": n, "browser_download_url": "https://x/" + n} for n in names + ], + } + + def _win_release(self): + names = [ + "llama-b9999-bin-win-cuda-13.3-x64.zip", + "cudart-llama-bin-win-cuda-13.3-x64.zip", + "llama-b9999-bin-win-cpu-x64.zip", + ] + return { + "tag_name": "b9999", + "assets": [ + {"name": n, "browser_download_url": "https://x/" + n} for n in names + ], + } + + def _no_torch(self, monkeypatch): + monkeypatch.setattr( + INSTALL_LLAMA_PREBUILT, + "detect_torch_cuda_runtime_preference", + lambda host: CudaRuntimePreference(runtime_line = None, selection_log = []), + ) + + # -- Linux -------------------------------------------------------------- + + def test_linux_nvidia_host_has_no_cpu_attempt(self, monkeypatch): + mock_linux_runtime(monkeypatch, ["cuda13", "cuda12"]) + self._no_torch(monkeypatch) + host = make_host(compute_caps = ["120"], driver_cuda_version = (13, 0)) + plan = direct_linux_release_plan( + self._linux_release("cuda13-newer", "cpu"), host, "unslothai/llama.cpp", "latest" + ) + kinds = [a.install_kind for a in plan.attempts] + assert "linux-cuda" in kinds + assert "linux-cpu" not in kinds + + def test_linux_nvidia_unmatched_arch_raises_no_cpu(self, monkeypatch): + # GPU present but no CUDA bundle covers the arch: must raise (-> source + # build), not fall to the CPU bundle. + mock_linux_runtime(monkeypatch, ["cuda13", "cuda12"]) + self._no_torch(monkeypatch) + host = make_host(compute_caps = ["50"], driver_cuda_version = (13, 0)) + with pytest.raises(PrebuiltFallback): + direct_linux_release_plan( + self._linux_release("cuda13-newer", "cpu"), + host, + "unslothai/llama.cpp", + "latest", + ) + + def test_linux_cpu_host_keeps_cpu_attempt(self, monkeypatch): + mock_linux_runtime(monkeypatch, []) + host = make_host( + has_usable_nvidia = False, + has_physical_nvidia = False, + nvidia_smi = None, + compute_caps = [], + ) + plan = direct_linux_release_plan( + self._linux_release("cuda13-newer", "cpu"), host, "unslothai/llama.cpp", "latest" + ) + assert [a.install_kind for a in plan.attempts] == ["linux-cpu"] + + # -- Windows ------------------------------------------------------------ + + def test_windows_nvidia_host_has_no_cpu_attempt(self, monkeypatch): + mock_windows_runtime(monkeypatch, ["cuda13"]) + self._no_torch(monkeypatch) + host = make_host( + system = "Windows", + machine = "AMD64", + driver_cuda_version = (13, 3), + compute_caps = ["120"], + ) + plan = direct_upstream_release_plan( + self._win_release(), host, UPSTREAM_REPO, "latest" + ) + kinds = [a.install_kind for a in plan.attempts] + assert "windows-cuda" in kinds + assert "windows-cpu" not in kinds + + def test_windows_cpu_host_keeps_cpu_attempt(self, monkeypatch): + mock_windows_runtime(monkeypatch, []) + host = make_host( + system = "Windows", + machine = "AMD64", + has_usable_nvidia = False, + has_physical_nvidia = False, + nvidia_smi = None, + driver_cuda_version = None, + compute_caps = [], + ) + plan = direct_upstream_release_plan( + self._win_release(), host, UPSTREAM_REPO, "latest" + ) + assert [a.install_kind for a in plan.attempts] == ["windows-cpu"] + + # =========================================================================== # N.1d. published_windows_cuda_attempts -- version-dynamic ordering seed # =========================================================================== diff --git a/tests/studio/install/test_validate_server_gpu_offload.py b/tests/studio/install/test_validate_server_gpu_offload.py new file mode 100644 index 0000000000..1182c7117b --- /dev/null +++ b/tests/studio/install/test_validate_server_gpu_offload.py @@ -0,0 +1,450 @@ +"""Tests for the GPU-offload validation added to install_llama_prebuilt.py. + +Covers issue unslothai/unsloth#5807 (duplicates #5830 / #5106 / #5827): a +llama-server whose GPU backend fails to initialize still serves HTTP 200 from +CPU, so the old validate_server accepted it and Studio shipped a silently +CPU-only install. The hardened validate_server rejects such a binary when GPU +offload was requested, so the resolver / source build falls through to a +GPU-capable bundle instead of a silently CPU-only one. + +Stdlib-only; the installer module is spec-loaded by absolute path (no +PYTHONPATH / heavy deps), matching test_install_llama_prebuilt_logic.py. +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + + +PACKAGE_ROOT = Path(__file__).resolve().parents[3] +MODULE_PATH = PACKAGE_ROOT / "studio" / "install_llama_prebuilt.py" +SPEC = importlib.util.spec_from_file_location( + "studio_install_llama_prebuilt", MODULE_PATH +) +assert SPEC is not None and SPEC.loader is not None +M = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = M +SPEC.loader.exec_module(M) + +server_log_shows_gpu_offload = M.server_log_shows_gpu_offload +resolve_smoke_test_install_kind = M.resolve_smoke_test_install_kind +smoke_test_server_binary = M.smoke_test_server_binary +validate_server = M.validate_server +PrebuiltFallback = M.PrebuiltFallback +HostInfo = M.HostInfo + + +# -- HostInfo factories ------------------------------------------------------ + + +def nvidia_host(**overrides) -> HostInfo: + defaults = dict( + system = "Linux", + machine = "x86_64", + is_windows = False, + is_linux = True, + is_macos = False, + is_x86_64 = True, + is_arm64 = False, + nvidia_smi = "/usr/bin/nvidia-smi", + driver_cuda_version = (13, 0), + compute_caps = ["120"], + visible_cuda_devices = None, + has_physical_nvidia = True, + has_usable_nvidia = True, + has_rocm = False, + ) + defaults.update(overrides) + return HostInfo(**defaults) + + +def windows_nvidia_host(**overrides) -> HostInfo: + return nvidia_host( + system = "Windows", + is_windows = True, + is_linux = False, + nvidia_smi = "nvidia-smi", + **overrides, + ) + + +def rocm_host(**overrides) -> HostInfo: + return nvidia_host( + nvidia_smi = None, + driver_cuda_version = None, + compute_caps = [], + has_physical_nvidia = False, + has_usable_nvidia = False, + has_rocm = True, + **overrides, + ) + + +def macos_arm_host(**overrides) -> HostInfo: + defaults = dict( + system = "Darwin", + machine = "arm64", + is_windows = False, + is_linux = False, + is_macos = True, + is_x86_64 = False, + is_arm64 = True, + nvidia_smi = None, + driver_cuda_version = None, + compute_caps = [], + visible_cuda_devices = None, + has_physical_nvidia = False, + has_usable_nvidia = False, + has_rocm = False, + ) + defaults.update(overrides) + return HostInfo(**defaults) + + +def cpu_host(**overrides) -> HostInfo: + return nvidia_host( + nvidia_smi = None, + driver_cuda_version = None, + compute_caps = [], + has_physical_nvidia = False, + has_usable_nvidia = False, + has_rocm = False, + **overrides, + ) + + +# -- Canned llama-server logs ------------------------------------------------ +# Current llama.cpp (>= mid-2026) dropped the "model buffer size" lines and +# enumerates a "device_info:" block instead; older builds still print the +# buffer lines. Both formats must classify correctly. + +CUDA_DEVICE_INFO_LOG = ( + "0.00.667 I log_info: verbosity = 3\n" + "0.00.667 I device_info:\n" + "0.01.101 I - CUDA0 : NVIDIA GeForce RTX 5070 (12282 MiB, 11000 MiB free)\n" + "0.01.101 I - CPU : AMD Ryzen 7 9700X (32000 MiB free)\n" + "0.01.101 I system_info: n_threads = 8 | CUDA : ARCHS = 1200 | CPU : AVX2 = 1\n" + "0.01.174 I srv llama_server: model loaded\n" +) + +CPU_ONLY_DEVICE_INFO_LOG = ( + "0.00.003 I log_info: verbosity = 3\n" + "0.00.003 I device_info:\n" + "0.00.003 I - CPU : AMD Ryzen 7 9700X (32000 MiB free)\n" + "0.00.003 I system_info: n_threads = 8 | CPU : AVX2 = 1\n" + "0.00.174 I srv llama_server: model loaded\n" +) + +CUDA_BUFFER_LOG = ( + "load_tensors: offloaded 33/33 layers to GPU\n" + "load_tensors: CUDA0 model buffer size = 21000.0 MiB\n" + "load_tensors: CPU_Mapped model buffer size = 0.6 MiB\n" +) + +CPU_BUFFER_LOG = ( + "load_tensors: CPU_Mapped model buffer size = 21000.0 MiB\n" + "load_tensors: CPU model buffer size = 0.6 MiB\n" +) + +ROCM_BUFFER_LOG = "load_tensors: ROCm0 model buffer size = 21000.0 MiB\n" +METAL_BUFFER_LOG = "load_tensors: Metal model buffer size = 8000.0 MiB\n" +VULKAN_DEVICE_INFO_LOG = ( + "device_info:\n - Vulkan0 : AMD Radeon (16000 MiB free)\n - CPU : x86 (free)\n" +) +OPENCL_DEVICE_INFO_LOG = ( + "device_info:\n - OpenCL0 : Adreno (4000 MiB free)\n - CPU : arm (free)\n" +) +OFFLOADED_ZERO_LOG = "load_tensors: offloaded 0/33 layers to GPU\n" +NO_SIGNAL_LOG = "INFO [main] starting server\nload_tensors: file format = GGUF V3\n" +# system_info advertises a *compiled* backend; not an available device. +SYSTEM_INFO_ONLY_LOG = ( + "system_info: n_threads = 8 | CUDA : ARCHS = 1200 | CPU : AVX2 = 1\n" +) + + +# -- 1. Pure classifier ------------------------------------------------------ + + +@pytest.mark.parametrize( + "log_text,expected", + [ + (CUDA_DEVICE_INFO_LOG, True), + (CPU_ONLY_DEVICE_INFO_LOG, False), + (CUDA_BUFFER_LOG, True), + (CPU_BUFFER_LOG, False), + (ROCM_BUFFER_LOG, True), + (METAL_BUFFER_LOG, True), + (VULKAN_DEVICE_INFO_LOG, True), + (OPENCL_DEVICE_INFO_LOG, True), + (OFFLOADED_ZERO_LOG, False), + (NO_SIGNAL_LOG, None), + ("", None), + (SYSTEM_INFO_ONLY_LOG, None), + ], +) +def test_server_log_shows_gpu_offload(log_text, expected): + assert server_log_shows_gpu_offload(log_text) is expected + + +def test_opencl_device_in_prefixes(): + # Regression: OpenCL was matched by the device-row regex but missing from + # the GPU prefix list, so an OpenCL-only device_info wrongly classified as + # CPU-only and a working OpenCL build was rejected. + assert "opencl" in M._GPU_DEVICE_PREFIXES + assert server_log_shows_gpu_offload(OPENCL_DEVICE_INFO_LOG) is True + + +def test_system_info_not_mistaken_for_device(): + # A CUDA-compiled binary whose runtime failed still prints the system_info + # CUDA backend, but enumerates only CPU under device_info -> must be False. + log = ( + "device_info:\n - CPU : x86 (free)\n" + "system_info: | CUDA : ARCHS = 1200 | CPU : AVX2 = 1\n" + ) + assert server_log_shows_gpu_offload(log) is False + + +def test_signal2_offloaded_zero_beats_device_info(): + # offloaded 0/33 (signal 2) is a definitive "nothing on GPU" and takes + # priority over a device_info CUDA0 row -> False. + log = ( + "load_tensors: offloaded 0/33 layers to GPU\n" + "device_info:\n - CUDA0 : NVIDIA (free)\n - CPU : x (free)\n" + ) + assert server_log_shows_gpu_offload(log) is False + + +def test_gpu_buffer_size_without_model_word(): + # A GPU-marked buffer line that omits the word "model" (e.g. KV/compute or + # a future format) must still count as GPU offload, even when a CPU + # "model buffer size" line is present (broadened signal 1). + log = ( + "load_tensors: CUDA0 KV buffer size = 100.0 MiB\n" + "load_tensors: CPU_Mapped model buffer size = 0.6 MiB\n" + ) + assert server_log_shows_gpu_offload(log) is True + + +def test_device_row_case_insensitive(): + log = "device_info:\n - cuda0 : some gpu (free)\n - CPU : x (free)\n" + assert server_log_shows_gpu_offload(log) is True + + +def test_crlf_log_parses_identically(): + # Windows logs use CRLF; classification must not change. + assert server_log_shows_gpu_offload(CUDA_DEVICE_INFO_LOG.replace("\n", "\r\n")) is True + assert ( + server_log_shows_gpu_offload(CPU_ONLY_DEVICE_INFO_LOG.replace("\n", "\r\n")) + is False + ) + + +# -- 2. resolve_smoke_test_install_kind -------------------------------------- + + +@pytest.mark.parametrize( + "host,expected", + [ + (nvidia_host(), "linux-cuda"), + (windows_nvidia_host(), "windows-cuda"), + (rocm_host(), "linux-rocm"), + (rocm_host(system = "Windows", is_windows = True, is_linux = False), "windows-hip"), + (macos_arm_host(), "macos-arm64"), + (cpu_host(), "linux-cpu"), + (cpu_host(system = "Windows", is_windows = True, is_linux = False), "windows-cpu"), + ], +) +def test_resolve_smoke_test_install_kind(host, expected): + assert resolve_smoke_test_install_kind(host) == expected + if expected.endswith("-cpu"): + assert expected not in M._GPU_INSTALL_KINDS + else: + assert expected in M._GPU_INSTALL_KINDS + + +# -- 3. validate_server integration (mocked subprocess + HTTP) --------------- + + +class _FakeResponse: + def __init__(self, status: int, body: bytes = b"{}"): + self.status = status + self._body = body + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def read(self): + return self._body + + +class _FakeProc: + """Stands in for the llama-server subprocess: writes a canned startup log + to the handle it was given, then reports itself alive until terminated.""" + + def __init__(self, log_text: str): + self._log_text = log_text + self._alive = True + + def __call__(self, command, *args, **kwargs): + handle = kwargs.get("stdout") + if handle is not None: + handle.write(self._log_text) + handle.flush() + return self + + def poll(self): + return None if self._alive else 0 + + def wait(self, timeout = None): + self._alive = False + return 0 + + def terminate(self): + self._alive = False + + def kill(self): + self._alive = False + + +@pytest.fixture +def patched_server(monkeypatch): + """Patch validate_server's external touchpoints; return a configure() + helper so each test picks the canned log it wants.""" + monkeypatch.setattr(M, "free_local_port", lambda: 18000) + monkeypatch.setattr(M, "binary_env", lambda *a, **k: {}) + monkeypatch.setattr(M.time, "sleep", lambda *a, **k: None) + monkeypatch.setattr(M.urllib.request, "urlopen", lambda *a, **k: _FakeResponse(200)) + + def configure(log_text): + monkeypatch.setattr(M.subprocess, "Popen", _FakeProc(log_text)) + + return configure + + +def _run_validate(tmp_path, host, install_kind): + server = tmp_path / "llama-server" + server.write_text("#!/bin/sh\n") + probe = tmp_path / "probe.gguf" + probe.write_bytes(b"GGUF") + validate_server(server, probe, host, tmp_path, install_kind = install_kind) + + +def test_gpu_intent_cpu_only_rejected(patched_server, tmp_path): + # The #5807 bug: GPU requested, binary ran on CPU -> reject. The exception + # must be GpuOffloadFailure specifically (not just the PrebuiltFallback + # base) so the --smoke-test CLI can map it to EXIT_FALLBACK vs EXIT_ERROR. + patched_server(CPU_ONLY_DEVICE_INFO_LOG) + with pytest.raises(M.GpuOffloadFailure, match = "entirely on CPU"): + _run_validate(tmp_path, windows_nvidia_host(), "windows-cuda") + assert issubclass(M.GpuOffloadFailure, PrebuiltFallback) + + +def test_gpu_intent_gpu_offload_accepted(patched_server, tmp_path): + patched_server(CUDA_DEVICE_INFO_LOG) + _run_validate(tmp_path, windows_nvidia_host(), "windows-cuda") # no raise + + +def test_cpu_kind_not_gpu_gated(patched_server, tmp_path): + # A windows-cpu bundle is the intentional fallback; never GPU-gated even + # if it (obviously) loads on CPU. + patched_server(CPU_ONLY_DEVICE_INFO_LOG) + _run_validate(tmp_path, windows_nvidia_host(), "windows-cpu") # no raise + + +def test_gpu_intent_no_signal_accepted(patched_server, tmp_path): + # No buffer/device signal -> conservative: do not reject on no evidence. + patched_server(NO_SIGNAL_LOG) + _run_validate(tmp_path, nvidia_host(), "linux-cuda") # no raise + + +def test_rocm_cpu_only_rejected(patched_server, tmp_path): + patched_server(CPU_ONLY_DEVICE_INFO_LOG) + with pytest.raises(PrebuiltFallback): + _run_validate(tmp_path, rocm_host(), "linux-rocm") + + +# -- 4. smoke_test_server_binary --------------------------------------------- + + +def test_smoke_test_missing_binary(tmp_path): + with pytest.raises(PrebuiltFallback, match = "not found"): + smoke_test_server_binary( + str(tmp_path / "nope"), nvidia_host(), install_dir = str(tmp_path) + ) + + +def test_smoke_test_delegates_to_validate_server(monkeypatch, tmp_path): + server = tmp_path / "llama-server" + server.write_text("#!/bin/sh\n") + probe = tmp_path / "probe.gguf" + probe.write_bytes(b"GGUF") + seen = {} + + def fake_validate(server_path, probe_path, host, install_dir, **kw): + seen["install_kind"] = kw.get("install_kind") + + monkeypatch.setattr(M, "validate_server", fake_validate) + kind = smoke_test_server_binary( + str(server), nvidia_host(), install_dir = str(tmp_path), probe = str(probe) + ) + assert kind == "linux-cuda" + assert seen["install_kind"] == "linux-cuda" + + +def test_smoke_test_propagates_failure(monkeypatch, tmp_path): + server = tmp_path / "llama-server" + server.write_text("#!/bin/sh\n") + probe = tmp_path / "probe.gguf" + probe.write_bytes(b"GGUF") + + def fake_validate(*a, **k): + raise PrebuiltFallback("loaded the model entirely on CPU") + + monkeypatch.setattr(M, "validate_server", fake_validate) + with pytest.raises(PrebuiltFallback): + smoke_test_server_binary( + str(server), + nvidia_host(), + install_dir = str(tmp_path), + probe = str(probe), + ) + + +# -- 5. --smoke-test CLI exit-code contract (the load-bearing fix contract) -- +# setup.sh / setup.ps1 branch on: 2 = ran on CPU (rebuild CPU), 1 = inconclusive +# (keep GPU build), 0 = offload confirmed. + + +def _run_main_smoke(monkeypatch, smoke_impl): + monkeypatch.setattr(M, "detect_host", lambda: nvidia_host()) + monkeypatch.setattr(M, "smoke_test_server_binary", smoke_impl) + monkeypatch.setattr( + sys, "argv", ["install_llama_prebuilt.py", "--smoke-test", "/x/llama-server"] + ) + return M.main() + + +def test_main_smoke_exit_success(monkeypatch): + rc = _run_main_smoke(monkeypatch, lambda *a, **k: "linux-cuda") + assert rc == M.EXIT_SUCCESS + + +def test_main_smoke_exit_fallback_on_cpu_only(monkeypatch): + def impl(*a, **k): + raise M.GpuOffloadFailure("loaded the model entirely on CPU") + + rc = _run_main_smoke(monkeypatch, impl) + assert rc == M.EXIT_FALLBACK # 2 -> setup scripts rebuild CPU + + +def test_main_smoke_exit_error_on_inconclusive(monkeypatch): + def impl(*a, **k): + raise PrebuiltFallback("llama-server exited during startup") + + rc = _run_main_smoke(monkeypatch, impl) + assert rc == M.EXIT_ERROR # 1 -> setup scripts keep the GPU build From c85c62e456a9d6cc42de248b5281eac4784f2b82 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 1 Jun 2026 15:53:12 +0000 Subject: [PATCH 02/12] Studio: source-build + CPU last-resort recovery when no GPU prebuilt offloads setup.sh/setup.ps1 now run install_llama_prebuilt.py --smoke-test on a freshly source-built GPU binary and retry a CPU build if it loaded on CPU only. When a GPU host's source build produces no binary, both scripts fall back to the CPU prebuilt (--cpu-fallback) as a labelled last resort instead of leaving the host without llama.cpp. setup.ps1 also guards an empty CUDA arch (no PTX-only binary, #5854). The POSIX smoke-test exit code is captured set -e safe. Adds a fake llama-server and an end-to-end spoof test that runs the real validate_server against it with no GPU. --- studio/setup.ps1 | 141 +++++++++++++++--- studio/setup.sh | 81 ++++++++-- tests/run_all.sh | 1 + tests/sh/test_llama_gpu_smoke.sh | 59 ++++++++ tests/studio/install/fake_llama_server.py | 106 +++++++++++++ .../studio/install/test_gpu_offload_spoof.py | 129 ++++++++++++++++ 6 files changed, 484 insertions(+), 33 deletions(-) create mode 100755 tests/sh/test_llama_gpu_smoke.sh create mode 100644 tests/studio/install/fake_llama_server.py create mode 100644 tests/studio/install/test_gpu_offload_spoof.py diff --git a/studio/setup.ps1 b/studio/setup.ps1 index 28ad512690..b52b02a8d2 100644 --- a/studio/setup.ps1 +++ b/studio/setup.ps1 @@ -2952,39 +2952,53 @@ if (-not $NeedLlamaSourceBuild) { $CmakeArgs += '-DLLAMA_CURL=OFF' } $CmakeArgs += '-DCMAKE_EXE_LINKER_FLAGS=/NODEFAULTLIB:LIBCMT' - # CUDA flags -- only if GPU available, otherwise explicitly disable + # CUDA flags -- only if GPU available, otherwise explicitly disable. + # $LlamaCudaBuild gates the post-build GPU smoke test and CUDA->CPU + # retry below. + $LlamaCudaBuild = $false if ($HasNvidiaSmi -and $NvccPath) { - $CmakeArgs += '-DGGML_CUDA=ON' - # Accept a host MSVC newer than nvcc's whitelist; a fresh toolkit - # (e.g. CUDA 13.3) otherwise aborts with "#error -- unsupported - # Microsoft Visual Studio version!". Mirrors the Linux fix. Via env - # (covers the configure probe + build), after Refresh-Environment, idempotent. - $nvccAllowFlag = '-allow-unsupported-compiler' - if ([string]::IsNullOrEmpty($env:NVCC_PREPEND_FLAGS)) { - $env:NVCC_PREPEND_FLAGS = $nvccAllowFlag - } elseif ($env:NVCC_PREPEND_FLAGS -notlike "*$nvccAllowFlag*") { - $env:NVCC_PREPEND_FLAGS = "$($env:NVCC_PREPEND_FLAGS) $nvccAllowFlag" - } - substep "NVCC_PREPEND_FLAGS = $env:NVCC_PREPEND_FLAGS" - $CmakeArgs += "-DCUDAToolkit_ROOT=$CudaToolkitRoot" - $CmakeArgs += "-DCUDA_TOOLKIT_ROOT_DIR=$CudaToolkitRoot" - $CmakeArgs += "-DCMAKE_CUDA_COMPILER=$NvccPath" + # Resolve a concrete CUDA architecture FIRST. A CUDA build with no + # -DCMAKE_CUDA_ARCHITECTURES is PTX-only and can fail at runtime on a + # driver older than the toolkit ("the provided PTX was compiled with + # an unsupported toolchain", #5854). If we cannot resolve a supported + # arch, build CPU-only instead of shipping a silently broken binary. + $cudaArchFlag = $null if ($CudaArch) { - # Validate nvcc actually supports this architecture if (Test-NvccArchSupport -NvccExe $NvccPath -Arch $CudaArch) { - $CmakeArgs += "-DCMAKE_CUDA_ARCHITECTURES=$CudaArch" + $cudaArchFlag = "-DCMAKE_CUDA_ARCHITECTURES=$CudaArch" } else { - # GPU arch too new for this toolkit -- fall back to highest supported. - # PTX forward-compatibility will JIT-compile for the actual GPU at runtime. + # GPU arch too new for this toolkit -- fall back to highest + # supported. PTX forward-compat will JIT for the real GPU. $maxArch = Get-NvccMaxArch -NvccExe $NvccPath if ($maxArch) { - $CmakeArgs += "-DCMAKE_CUDA_ARCHITECTURES=$maxArch" + $cudaArchFlag = "-DCMAKE_CUDA_ARCHITECTURES=$maxArch" substep "GPU is sm_$CudaArch but nvcc only supports up to sm_$maxArch" "Yellow" substep "Building with sm_$maxArch (PTX will JIT for your GPU at runtime)" "Yellow" } - # else: omit flag entirely, let cmake pick defaults } } + if ($cudaArchFlag) { + $CmakeArgs += '-DGGML_CUDA=ON' + # Accept a host MSVC newer than nvcc's whitelist; a fresh toolkit + # (e.g. CUDA 13.3) otherwise aborts with "#error -- unsupported + # Microsoft Visual Studio version!". Via env (covers the configure + # probe + build), after Refresh-Environment, idempotent. + $nvccAllowFlag = '-allow-unsupported-compiler' + if ([string]::IsNullOrEmpty($env:NVCC_PREPEND_FLAGS)) { + $env:NVCC_PREPEND_FLAGS = $nvccAllowFlag + } elseif ($env:NVCC_PREPEND_FLAGS -notlike "*$nvccAllowFlag*") { + $env:NVCC_PREPEND_FLAGS = "$($env:NVCC_PREPEND_FLAGS) $nvccAllowFlag" + } + substep "NVCC_PREPEND_FLAGS = $env:NVCC_PREPEND_FLAGS" + $CmakeArgs += "-DCUDAToolkit_ROOT=$CudaToolkitRoot" + $CmakeArgs += "-DCUDA_TOOLKIT_ROOT_DIR=$CudaToolkitRoot" + $CmakeArgs += "-DCMAKE_CUDA_COMPILER=$NvccPath" + $CmakeArgs += $cudaArchFlag + $LlamaCudaBuild = $true + } else { + substep "Could not resolve a supported CUDA architecture for this GPU/toolkit; building CPU-only to avoid a PTX-only binary that fails at runtime (#5854)" "Yellow" + $CmakeArgs += '-DGGML_CUDA=OFF' + } } else { $CmakeArgs += '-DGGML_CUDA=OFF' } @@ -3025,6 +3039,60 @@ if (-not $NeedLlamaSourceBuild) { } } + # -- Step C.5: GPU smoke test + CUDA->CPU fallback (#5807 / #5854) -- + # A CUDA build whose runtime backend fails to initialize still links and + # serves HTTP 200, but only from CPU; and setup.ps1 previously had no CPU + # fallback when a CUDA build failed at all. Both gaps are closed here. + if ($LlamaCudaBuild -and $BuildOk) { + $builtServer = Join-Path $BuildDir "bin\Release\llama-server.exe" + if (-not (Test-Path -LiteralPath $builtServer)) { + $builtServer = Join-Path $BuildDir "bin\llama-server.exe" + } + if (Test-Path -LiteralPath $builtServer) { + Write-Host "" + Write-Host "--- GPU smoke test ---" -ForegroundColor Cyan + & python "$PSScriptRoot\install_llama_prebuilt.py" --smoke-test "$builtServer" --install-dir "$LlamaCppDir" 2>&1 | Out-String | Write-Host + $smokeExit = $LASTEXITCODE + if ($smokeExit -eq 2) { + substep "GPU build runs on CPU only (GPU backend failed to initialize)" "Yellow" + $BuildOk = $false + $FailedStep = "GPU smoke test (ran on CPU)" + } elseif ($smokeExit -ne 0) { + substep "GPU smoke test inconclusive (exit $smokeExit); keeping GPU build" "Yellow" + } + } + } + + # If a CUDA build was attempted and configure/build/smoke-test left it + # unusable, retry once with CUDA disabled so the user still gets a working + # (if slower) CPU llama-server instead of nothing. + if ($LlamaCudaBuild -and -not $BuildOk) { + substep "CUDA build unusable at: $FailedStep; retrying CPU-only build..." "Yellow" + $CpuCmakeArgs = @($CmakeArgs | Where-Object { + $_ -ne '-DGGML_CUDA=ON' -and + $_ -notlike '-DCMAKE_CUDA_ARCHITECTURES=*' -and + $_ -notlike '-DCUDAToolkit_ROOT=*' -and + $_ -notlike '-DCUDA_TOOLKIT_ROOT_DIR=*' -and + $_ -notlike '-DCMAKE_CUDA_COMPILER=*' + }) + $CpuCmakeArgs += '-DGGML_CUDA=OFF' + if (Test-Path -LiteralPath $BuildDir) { Remove-Item -LiteralPath $BuildDir -Recurse -Force } + $cpuConfigure = cmake @CpuCmakeArgs 2>&1 | Out-String + if ($LASTEXITCODE -eq 0) { + $cpuBuild = cmake --build $BuildDir --config Release --target llama-server -j $NumCpu 2>&1 | Out-String + if ($LASTEXITCODE -eq 0) { + $BuildOk = $true + $LlamaCudaBuild = $false + $FailedStep = $null + substep "CPU-only llama.cpp build succeeded" "Green" + } else { + Write-LlamaFailureLog -Output $cpuBuild + } + } else { + Write-LlamaFailureLog -Output $cpuConfigure + } + } + # -- Step D: Build llama-quantize (optional, best-effort) -- if ($BuildOk) { Write-Host "" @@ -3083,6 +3151,35 @@ if (-not $NeedLlamaSourceBuild) { } } +# ───────────────────────────────────────────── +# Windows GPU: CPU prebuilt as a last resort +# ───────────────────────────────────────────── +# A GPU host reaches the source build only when no GPU prebuilt offloads (the +# CPU prebuilt is deliberately not offered to a GPU host so it does not short +# circuit the source build, #5807). If that build produced no binary (no CUDA +# toolkit, compile failure), install the CPU prebuilt via --cpu-fallback so the +# host still gets a working (if slower) llama-server instead of nothing. +if ($script:LlamaCppDegraded -and ($HasNvidiaSmi -or $HasROCm)) { + substep "GPU build unavailable; trying CPU prebuilt as a last resort..." "Yellow" + $lastResortArgs = @( + "$PSScriptRoot\install_llama_prebuilt.py", + "--install-dir", $OriginalLlamaCppDir, + "--llama-tag", $RequestedLlamaTag, + "--published-repo", $HelperReleaseRepo, + "--simple-policy", + "--cpu-fallback" + ) + $prevEAPLast = $ErrorActionPreference + $ErrorActionPreference = "Continue" + & python @lastResortArgs 2>&1 | Out-String | Write-Host + $lastResortExit = $LASTEXITCODE + $ErrorActionPreference = $prevEAPLast + if ($lastResortExit -eq 0) { + step "llama.cpp" "CPU prebuilt installed (GPU unavailable; inference will run on CPU)" "Yellow" + $script:LlamaCppDegraded = $false + } +} + # ───────────────────────────────────────────── # Footer # ───────────────────────────────────────────── diff --git a/studio/setup.sh b/studio/setup.sh index 9b29def859..b941fd6a82 100755 --- a/studio/setup.sh +++ b/studio/setup.sh @@ -1340,6 +1340,56 @@ else fi fi + # Map an install_llama_prebuilt.py --smoke-test exit code to a decision + # token. 2 (EXIT_FALLBACK) = definitively GPU-intended but CPU-only -> + # rebuild CPU; 0 = offload confirmed; anything else (1/EXIT_ERROR, + # signals) = inconclusive -> keep the GPU build rather than downgrade on + # uncertain evidence. Tiny + side-effect free so + # tests/sh/test_llama_gpu_smoke.sh can exercise it. + _classify_smoke_exit() { + case "$1" in + 0) echo "ok" ;; + 2) echo "cpu_only" ;; + *) echo "inconclusive" ;; + esac + } + + # Post-build GPU smoke test (#5807 / #5854): a GPU build whose runtime + # backend fails to initialize still links and serves HTTP 200, but only + # from CPU. Confirm the fresh binary actually offloads to the GPU; if it + # ran on CPU only, retry a CPU build so the user gets a working (if + # slower) llama-server instead of a silently CPU-only "GPU" build. + # _gpu_fallback_label is empty for a pure CPU build (nothing to verify). + if [ "$BUILD_OK" = true ]; then + _SMOKE_LABEL="$(_gpu_fallback_label)" + if [ -n "$_SMOKE_LABEL" ] && [ -f "$_BUILD_TMP/build/bin/llama-server" ]; then + # if/else keeps set -e from aborting before we read the code. + if python "$SCRIPT_DIR/install_llama_prebuilt.py" \ + --smoke-test "$_BUILD_TMP/build/bin/llama-server" \ + --install-dir "$_BUILD_TMP" > "$_BUILD_TMP/gpu-smoke.log" 2>&1; then + _SMOKE_RC=0 + else + _SMOKE_RC=$? + fi + case "$(_classify_smoke_exit "$_SMOKE_RC")" in + cpu_only) + substep "$_SMOKE_LABEL build runs on CPU only; retrying CPU build..." "$C_WARN" + rm -rf "$_BUILD_TMP/build" + if run_quiet_no_exit "cmake llama.cpp (cpu fallback)" cmake $CMAKE_GENERATOR_ARGS -S "$_BUILD_TMP" -B "$_BUILD_TMP/build" $CPU_FALLBACK_CMAKE_ARGS; then + _BUILD_DESC="building (CPU fallback after $_SMOKE_LABEL smoke test ran on CPU)" + GPU_BACKEND="" + run_quiet_no_exit "build llama-server (cpu fallback)" cmake --build "$_BUILD_TMP/build" --config Release --target llama-server -j"$NCPU" || BUILD_OK=false + else + BUILD_OK=false + fi + ;; + inconclusive) + substep "GPU smoke test inconclusive (exit $_SMOKE_RC); keeping $_SMOKE_LABEL build" "$C_WARN" + ;; + esac + fi + fi + if [ "$BUILD_OK" = true ]; then run_quiet_no_exit "build llama-quantize" cmake --build "$_BUILD_TMP/build" --config Release --target llama-quantize -j"$NCPU" || true fi @@ -1373,27 +1423,36 @@ else } fi # end _SKIP_GGUF_BUILD check -# ── arm64 Linux GPU: CPU prebuilt as a last resort ── -# arm64 Linux with a GPU has no CUDA prebuilt anywhere (the unslothai fork is -# x64 only; ggml-org ships no Linux CUDA build), so it source-builds for the -# GPU above. If that produced no binary, install ggml-org's arm64 CPU prebuilt -# instead of leaving the host without llama.cpp. +# ── Linux GPU: CPU prebuilt as a last resort ── +# A Linux GPU host reaches a source build when no GPU prebuilt offloads (the +# CPU prebuilt is deliberately not offered to a GPU host so it does not short +# circuit the source build -- #5807). If that build produced no binary (no +# toolkit, compile failure), install a CPU prebuilt instead of leaving the host +# without llama.cpp. arm64 has no CUDA prebuilt anywhere, so it always lands +# here on a degraded GPU build; x86_64 only when its source build also failed. +# Repo: ggml-org ships the arm64 CPU tarball; x86_64 uses the same published +# repo as the primary path (the unslothai fork carries linux-x64-cpu). if [ "$_LLAMA_CPP_DEGRADED" = true ] \ && [ "$_HOST_SYSTEM" = "Linux" ] \ - && { [ "$_HOST_MACHINE" = "aarch64" ] || [ "$_HOST_MACHINE" = "arm64" ]; }; then - substep "GPU source build unavailable; trying ggml-org arm64 CPU prebuilt..." - _ARM64_CPU_CMD=( + && [ "$_LINUX_HAS_GPU" = true ]; then + if [ "$_HOST_MACHINE" = "aarch64" ] || [ "$_HOST_MACHINE" = "arm64" ]; then + _LASTRESORT_CPU_REPO="ggml-org/llama.cpp" + else + _LASTRESORT_CPU_REPO="$_HELPER_RELEASE_REPO" + fi + substep "GPU build unavailable; trying $_LASTRESORT_CPU_REPO CPU prebuilt as a last resort..." + _LASTRESORT_CPU_CMD=( python "$SCRIPT_DIR/install_llama_prebuilt.py" --install-dir "$LLAMA_CPP_DIR" --llama-tag "$_REQUESTED_LLAMA_TAG" - --published-repo "ggml-org/llama.cpp" + --published-repo "$_LASTRESORT_CPU_REPO" --simple-policy --cpu-fallback ) # Trust the installer's exit code: it validates the server before exiting 0, # the same signal the primary prebuilt path above relies on. - if run_quiet_no_exit "arm64 CPU prebuilt" "${_ARM64_CPU_CMD[@]}"; then - step "llama.cpp" "arm64 CPU prebuilt installed (GPU build unavailable)" "$C_WARN" + if run_quiet_no_exit "CPU prebuilt (last resort)" "${_LASTRESORT_CPU_CMD[@]}"; then + step "llama.cpp" "CPU prebuilt installed (GPU unavailable; inference will run on CPU)" "$C_WARN" _LLAMA_CPP_DEGRADED=false print_installed_llama_prebuilt_release "$LLAMA_CPP_DIR" fi diff --git a/tests/run_all.sh b/tests/run_all.sh index d84c930392..147bc81898 100755 --- a/tests/run_all.sh +++ b/tests/run_all.sh @@ -9,6 +9,7 @@ sh "$TESTS_DIR/sh/test_get_torch_index_url.sh" sh "$TESTS_DIR/sh/test_mac_intel_compat.sh" sh "$TESTS_DIR/sh/test_torch_constraint.sh" sh "$TESTS_DIR/sh/test_nvcc_meets_llama_minimum.sh" +sh "$TESTS_DIR/sh/test_llama_gpu_smoke.sh" echo "" echo "=== Python tests ===" diff --git a/tests/sh/test_llama_gpu_smoke.sh b/tests/sh/test_llama_gpu_smoke.sh new file mode 100755 index 0000000000..875158d480 --- /dev/null +++ b/tests/sh/test_llama_gpu_smoke.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Unit tests for _classify_smoke_exit() from studio/setup.sh. +# +# After a llama.cpp source build, setup.sh runs install_llama_prebuilt.py +# --smoke-test against the fresh binary and maps its exit code to a decision: +# 2 (EXIT_FALLBACK) -> "cpu_only" : GPU was requested but the model ran +# on CPU -> rebuild CPU (#5807 / #5854). +# 0 -> "ok" : GPU offload confirmed -> keep build. +# 1 / signals / etc -> "inconclusive" : could not validate -> keep GPU build +# (never downgrade on uncertainty). +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +SETUP_SH="$SCRIPT_DIR/../../studio/setup.sh" +PASS=0 +FAIL=0 + +# Extract just the helper (same approach as test_nvcc_meets_llama_minimum.sh). +# The function and its closing brace sit at 8-space indent inside setup.sh. +_FUNC_FILE=$(mktemp) +sed -n '/ _classify_smoke_exit() {/,/^ }/p' "$SETUP_SH" > "$_FUNC_FILE" + +assert_eq() { + _label="$1"; _expected="$2"; _actual="$3" + if [ "$_actual" = "$_expected" ]; then + echo " PASS: $_label" + PASS=$((PASS + 1)) + else + echo " FAIL: $_label (expected '$_expected', got '$_actual')" + FAIL=$((FAIL + 1)) + fi +} + +run_classify() { + bash -c ". '$_FUNC_FILE'; _classify_smoke_exit '$1'" +} + +echo "=== test_llama_gpu_smoke (_classify_smoke_exit) ===" + +assert_eq "exit 0 -> ok" "ok" "$(run_classify 0)" +assert_eq "exit 2 -> cpu_only" "cpu_only" "$(run_classify 2)" +assert_eq "exit 1 -> inconclusive" "inconclusive" "$(run_classify 1)" +assert_eq "exit 3 -> inconclusive" "inconclusive" "$(run_classify 3)" +assert_eq "exit 137 -> inconclusive" "inconclusive" "$(run_classify 137)" + +# Sanity: the extracted function is non-empty and well-formed. +if [ -s "$_FUNC_FILE" ] && grep -q 'cpu_only' "$_FUNC_FILE"; then + echo " PASS: _classify_smoke_exit extracted from setup.sh" + PASS=$((PASS + 1)) +else + echo " FAIL: _classify_smoke_exit could not be extracted from setup.sh" + FAIL=$((FAIL + 1)) +fi + +rm -f "$_FUNC_FILE" + +echo "" +echo "Results: $PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ] || exit 1 diff --git a/tests/studio/install/fake_llama_server.py b/tests/studio/install/fake_llama_server.py new file mode 100644 index 0000000000..f8ae35ee9d --- /dev/null +++ b/tests/studio/install/fake_llama_server.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""A fake llama-server for GPU-offload validation tests (no GPU required). + +It accepts the arguments install_llama_prebuilt.py's validate_server passes +(``-m`` / ``--host`` / ``--port`` / ``--n-gpu-layers`` / ...), prints a canned +llama.cpp startup log chosen by the ``FAKE_LLAMA_MODE`` env var, then serves +HTTP 200 from ``/completion`` until killed. This lets CI exercise the real +validate_server subprocess + HTTP + log-classifier path on GPU-less Windows / +macOS / Linux runners: the same binary "starts and serves 200" while its log +says CPU-only or GPU, which is exactly the #5807 / #5830 situation. + +FAKE_LLAMA_MODE (default "cuda"): + cuda device_info enumerates CUDA0 (GPU offload confirmed) + cuda_buffer older "CUDA0 model buffer size" + offloaded 33/33 lines + cpu device_info enumerates only CPU (the silent CPU fallback) + offloaded_zero "offloaded 0/33 layers to GPU" (definite CPU-only) + no_signal a log with no offload evidence (validator must not reject) +""" + +import os +import sys +from http.server import BaseHTTPRequestHandler, HTTPServer + + +LOGS = { + "cuda": ( + "0.00 I device_info:\n" + "0.01 I - CUDA0 : NVIDIA GeForce RTX 5070 (12282 MiB, 11000 MiB free)\n" + "0.01 I - CPU : Generic CPU (32000 MiB free)\n" + "0.01 I system_info: n_threads = 8 | CUDA : ARCHS = 1200 | CPU : AVX2 = 1\n" + "0.02 I srv llama_server: model loaded\n" + ), + "cuda_buffer": ( + "load_tensors: offloaded 33/33 layers to GPU\n" + "load_tensors: CUDA0 model buffer size = 21000.0 MiB\n" + "load_tensors: CPU_Mapped model buffer size = 0.6 MiB\n" + "srv llama_server: model loaded\n" + ), + "cpu": ( + "0.00 I device_info:\n" + "0.00 I - CPU : Generic CPU (32000 MiB free)\n" + "0.00 I system_info: n_threads = 8 | CPU : AVX2 = 1\n" + "0.01 I srv llama_server: model loaded\n" + ), + "offloaded_zero": ( + "load_tensors: offloaded 0/33 layers to GPU\n" + "load_tensors: CPU_Mapped model buffer size = 21000.0 MiB\n" + "srv llama_server: model loaded\n" + ), + "no_signal": ( + "INFO [main] starting server\n" + "load_tensors: file format = GGUF V3\n" + "srv llama_server: model loaded\n" + ), +} + + +def _arg(name, default=None): + argv = sys.argv + for i, token in enumerate(argv): + if token == name and i + 1 < len(argv): + return argv[i + 1] + if token.startswith(name + "="): + return token.split("=", 1)[1] + return default + + +class _Handler(BaseHTTPRequestHandler): + def _ok(self): + body = b'{"content": "x", "tokens_predicted": 1}' + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_POST(self): + length = int(self.headers.get("Content-Length", 0) or 0) + if length: + self.rfile.read(length) + self._ok() + + def do_GET(self): + self._ok() + + def log_message(self, *args): + pass # keep stdout clean for the classifier + + +def main() -> int: + mode = os.environ.get("FAKE_LLAMA_MODE", "cuda") + sys.stdout.write(LOGS.get(mode, LOGS["cuda"])) + sys.stdout.flush() + + host = _arg("--host", "127.0.0.1") + port = int(_arg("--port", "8080")) + server = HTTPServer((host, port), _Handler) + try: + server.serve_forever() + except KeyboardInterrupt: + pass + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/studio/install/test_gpu_offload_spoof.py b/tests/studio/install/test_gpu_offload_spoof.py new file mode 100644 index 0000000000..f860b45151 --- /dev/null +++ b/tests/studio/install/test_gpu_offload_spoof.py @@ -0,0 +1,129 @@ +"""End-to-end GPU-offload spoof: run the real validate_server (real subprocess ++ real HTTP + real log classifier) against a fake llama-server, no GPU needed. + +Unlike test_validate_server_gpu_offload.py (which mocks subprocess/urlopen), +this launches an actual process that "starts and serves HTTP 200" while its log +reports CPU-only or GPU offload, reproducing #5807 / #5830 end to end. POSIX +only: validate_server execs the binary path directly, which needs a shebang +wrapper; the Windows equivalent runs in the studio-gpu-offload-smoke workflow +via a .bat shim. +""" + +import importlib.util +import os +import stat +import sys +from pathlib import Path + +import pytest + + +if sys.platform == "win32": + pytest.skip("POSIX-only (Windows covered by the spoof CI workflow)", allow_module_level = True) + +PACKAGE_ROOT = Path(__file__).resolve().parents[3] +MODULE_PATH = PACKAGE_ROOT / "studio" / "install_llama_prebuilt.py" +FAKE_SERVER = Path(__file__).resolve().parent / "fake_llama_server.py" +SPEC = importlib.util.spec_from_file_location("studio_install_llama_prebuilt_e2e", MODULE_PATH) +M = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = M +SPEC.loader.exec_module(M) + +HostInfo = M.HostInfo + + +def linux_cuda_host(**overrides): + defaults = dict( + system = "Linux", + machine = "x86_64", + is_windows = False, + is_linux = True, + is_macos = False, + is_x86_64 = True, + is_arm64 = False, + nvidia_smi = "/usr/bin/nvidia-smi", + driver_cuda_version = (13, 0), + compute_caps = ["120"], + visible_cuda_devices = None, + has_physical_nvidia = True, + has_usable_nvidia = True, + has_rocm = False, + ) + defaults.update(overrides) + return HostInfo(**defaults) + + +@pytest.fixture +def fake_server_binary(tmp_path): + """A `llama-server` that execs the fake server, so validate_server runs it + exactly as it would a real prebuilt binary.""" + binary = tmp_path / "llama-server" + binary.write_text( + "#!/bin/sh\n" + f'exec "{sys.executable}" "{FAKE_SERVER}" "$@"\n' + ) + binary.chmod(binary.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + return binary + + +def _validate(binary, tmp_path, host, install_kind, mode, monkeypatch): + monkeypatch.setenv("FAKE_LLAMA_MODE", mode) + probe = tmp_path / "probe.gguf" + probe.write_bytes(b"GGUF\x00fake") + M.validate_server(binary, probe, host, tmp_path, install_kind = install_kind) + + +def test_cpu_only_binary_tagged_cuda_is_rejected(fake_server_binary, tmp_path, monkeypatch): + # The #5807 case: a binary that serves 200 but loaded the model on CPU. + with pytest.raises(M.GpuOffloadFailure): + _validate(fake_server_binary, tmp_path, linux_cuda_host(), "linux-cuda", "cpu", monkeypatch) + + +def test_offloaded_zero_binary_tagged_cuda_is_rejected(fake_server_binary, tmp_path, monkeypatch): + with pytest.raises(M.GpuOffloadFailure): + _validate( + fake_server_binary, tmp_path, linux_cuda_host(), "linux-cuda", "offloaded_zero", monkeypatch + ) + + +def test_gpu_binary_tagged_cuda_passes(fake_server_binary, tmp_path, monkeypatch): + _validate(fake_server_binary, tmp_path, linux_cuda_host(), "linux-cuda", "cuda", monkeypatch) + + +def test_gpu_buffer_format_passes(fake_server_binary, tmp_path, monkeypatch): + _validate(fake_server_binary, tmp_path, linux_cuda_host(), "linux-cuda", "cuda_buffer", monkeypatch) + + +def test_cpu_only_binary_tagged_cpu_is_accepted(fake_server_binary, tmp_path, monkeypatch): + # A linux-cpu bundle is the intentional fallback; never GPU-gated. + _validate(fake_server_binary, tmp_path, linux_cuda_host(), "linux-cpu", "cpu", monkeypatch) + + +def test_no_signal_binary_tagged_cuda_is_accepted(fake_server_binary, tmp_path, monkeypatch): + # No offload evidence -> conservative: do not reject on no signal. + _validate(fake_server_binary, tmp_path, linux_cuda_host(), "linux-cuda", "no_signal", monkeypatch) + + +def test_smoke_test_cli_exit_codes(fake_server_binary, tmp_path, monkeypatch): + # The contract setup.sh / setup.ps1 depend on, exercised end to end through + # the real --smoke-test CLI: CPU-only -> 2, GPU -> 0. + probe = tmp_path / "probe.gguf" + probe.write_bytes(b"GGUF\x00fake") + monkeypatch.setattr(M, "detect_host", lambda: linux_cuda_host()) + + def run(mode): + monkeypatch.setenv("FAKE_LLAMA_MODE", mode) + monkeypatch.setattr( + sys, + "argv", + [ + "install_llama_prebuilt.py", + "--smoke-test", str(fake_server_binary), + "--probe", str(probe), + "--install-kind", "linux-cuda", + ], + ) + return M.main() + + assert run("cpu") == M.EXIT_FALLBACK + assert run("cuda") == M.EXIT_SUCCESS From a742dd49ce57c6cc5b4c38ae337c0fe6a87bb583 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 1 Jun 2026 15:55:04 +0000 Subject: [PATCH 03/12] CI: cross-platform GPU-offload spoof on Windows/macOS/Linux runners Adds a GPU-less smoke that drives install_llama_prebuilt.py --smoke-test against a fake llama-server emitting CPU-only vs GPU device_info logs, asserting a CPU-only binary tagged as a GPU install is rejected (exit 2) and a GPU one is accepted (exit 0), plus the selection and classifier unit tests, on all three OSes. This is the coverage gap that let the silent CPU-only path ship. --- .../workflows/studio-gpu-offload-smoke.yml | 65 +++++++++++ tests/studio/install/run_smoke_spoof.py | 101 ++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 .github/workflows/studio-gpu-offload-smoke.yml create mode 100644 tests/studio/install/run_smoke_spoof.py diff --git a/.github/workflows/studio-gpu-offload-smoke.yml b/.github/workflows/studio-gpu-offload-smoke.yml new file mode 100644 index 0000000000..92136e8c3b --- /dev/null +++ b/.github/workflows/studio-gpu-offload-smoke.yml @@ -0,0 +1,65 @@ +# Cross-platform GPU-offload validation smoke (no GPU required). +# +# Reproduces the silent CPU-only GGUF bug (#5807 / #5106 / #5830) with a fake +# llama-server that "starts and serves HTTP 200" while its log reports CPU-only +# or GPU offload, and asserts install_llama_prebuilt.py --smoke-test rejects the +# CPU-only-tagged-GPU case (exit 2) and accepts the GPU case (exit 0). Runs on +# GPU-less Windows / macOS / Linux runners, which is why this regressed +# untested. Also runs the pure-Python selection + classifier unit tests. + +name: Studio GPU Offload Smoke + +on: + pull_request: + paths: + - 'studio/install_llama_prebuilt.py' + - 'studio/setup.sh' + - 'studio/setup.ps1' + - 'studio/backend/core/inference/llama_cpp.py' + - 'tests/studio/install/**' + - 'tests/sh/test_llama_gpu_smoke.sh' + - '.github/workflows/studio-gpu-offload-smoke.yml' + push: + branches: [main, pip] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + gpu-offload-spoof: + name: GPU-offload spoof (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install pytest + run: python -m pip install --upgrade pip pytest + + - name: GPU-offload spoof (end to end, real subprocess + HTTP) + run: python tests/studio/install/run_smoke_spoof.py + + - name: Selection + classifier unit tests + run: > + python -m pytest + tests/studio/install/test_validate_server_gpu_offload.py + tests/studio/install/test_selection_logic.py + tests/studio/install/test_gpu_offload_spoof.py + -q + + - name: setup.sh smoke-exit classifier (POSIX) + if: runner.os != 'Windows' + run: bash tests/sh/test_llama_gpu_smoke.sh diff --git a/tests/studio/install/run_smoke_spoof.py b/tests/studio/install/run_smoke_spoof.py new file mode 100644 index 0000000000..2538e5ef52 --- /dev/null +++ b/tests/studio/install/run_smoke_spoof.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Cross-platform GPU-offload spoof driver for CI (no GPU required). + +Builds an OS-appropriate `llama-server` wrapper around fake_llama_server.py and +drives install_llama_prebuilt.py --smoke-test against it, asserting the exit +code contract the setup scripts depend on: + + FAKE_LLAMA_MODE=cpu , GPU install_kind -> exit 2 (EXIT_FALLBACK) rejected + FAKE_LLAMA_MODE=cuda, GPU install_kind -> exit 0 (EXIT_SUCCESS) accepted + FAKE_LLAMA_MODE=cpu , CPU install_kind -> exit 0 not gated + +Runs on windows-latest / macos-latest / ubuntu-latest. Exits non-zero on any +mismatch so the CI job fails loudly. +""" + +import os +import stat +import subprocess +import sys +import tempfile +from pathlib import Path + + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[2] +INSTALLER = REPO / "studio" / "install_llama_prebuilt.py" +FAKE = HERE / "fake_llama_server.py" +IS_WIN = sys.platform == "win32" + +GPU_KIND = { + "win32": "windows-cuda", + "darwin": "macos-arm64", +}.get(sys.platform, "linux-cuda") +CPU_KIND = { + "win32": "windows-cpu", + "darwin": "macos-cpu", +}.get(sys.platform, "linux-cpu") + + +def make_wrapper(workdir: Path) -> Path: + if IS_WIN: + wrapper = workdir / "llama-server.bat" + wrapper.write_text(f'@"{sys.executable}" "{FAKE}" %*\r\n') + return wrapper + wrapper = workdir / "llama-server" + wrapper.write_text(f'#!/bin/sh\nexec "{sys.executable}" "{FAKE}" "$@"\n') + wrapper.chmod(wrapper.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + return wrapper + + +def run_smoke(wrapper: Path, probe: Path, install_kind: str, mode: str) -> int: + env = dict(os.environ, FAKE_LLAMA_MODE=mode) + proc = subprocess.run( + [ + sys.executable, + str(INSTALLER), + "--smoke-test", str(wrapper), + "--probe", str(probe), + "--install-kind", install_kind, + ], + env=env, + capture_output=True, + text=True, + ) + sys.stdout.write(proc.stdout) + sys.stderr.write(proc.stderr) + return proc.returncode + + +def main() -> int: + failures = [] + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + wrapper = make_wrapper(work) + probe = work / "probe.gguf" + probe.write_bytes(b"GGUF\x00fake") + + cases = [ + ("cpu", GPU_KIND, 2, "CPU-only binary tagged GPU is rejected"), + ("offloaded_zero", GPU_KIND, 2, "offloaded 0/N tagged GPU is rejected"), + ("cuda", GPU_KIND, 0, "GPU binary tagged GPU is accepted"), + ("cuda_buffer", GPU_KIND, 0, "GPU buffer-format binary is accepted"), + ("cpu", CPU_KIND, 0, "CPU binary tagged CPU is not gated"), + ("no_signal", GPU_KIND, 0, "no-signal log is not rejected"), + ] + for mode, kind, expected, label in cases: + rc = run_smoke(wrapper, probe, kind, mode) + ok = rc == expected + print(f"[{'PASS' if ok else 'FAIL'}] {label}: mode={mode} kind={kind} exit={rc} (want {expected})") + if not ok: + failures.append(label) + + if failures: + print(f"\n{len(failures)} spoof case(s) failed: {failures}") + return 1 + print("\nAll GPU-offload spoof cases passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 9a8703f43ce448094ff7f0c21d55421f903ea2af Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Jun 2026 15:56:33 +0000 Subject: [PATCH 04/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../tests/test_llama_cpp_context_fit.py | 10 ++- tests/studio/install/fake_llama_server.py | 2 +- tests/studio/install/run_smoke_spoof.py | 21 +++-- .../studio/install/test_gpu_offload_spoof.py | 85 ++++++++++++++----- tests/studio/install/test_selection_logic.py | 10 ++- .../test_validate_server_gpu_offload.py | 4 +- 6 files changed, 97 insertions(+), 35 deletions(-) diff --git a/studio/backend/tests/test_llama_cpp_context_fit.py b/studio/backend/tests/test_llama_cpp_context_fit.py index 065c9726e4..3d295f711c 100644 --- a/studio/backend/tests/test_llama_cpp_context_fit.py +++ b/studio/backend/tests/test_llama_cpp_context_fit.py @@ -602,13 +602,15 @@ class TestClassifyGpuOffload: def test_offloaded_layers_count_decides(self): assert ( - self._backend(["load_tensors: offloaded 33/33 layers to GPU"]) - ._classify_gpu_offload(True, [(0, 22805)]) + self._backend( + ["load_tensors: offloaded 33/33 layers to GPU"] + )._classify_gpu_offload(True, [(0, 22805)]) is True ) assert ( - self._backend(["load_tensors: offloaded 0/33 layers to GPU"]) - ._classify_gpu_offload(True, [(0, 22805)]) + self._backend( + ["load_tensors: offloaded 0/33 layers to GPU"] + )._classify_gpu_offload(True, [(0, 22805)]) is False ) diff --git a/tests/studio/install/fake_llama_server.py b/tests/studio/install/fake_llama_server.py index f8ae35ee9d..c23a291d9a 100644 --- a/tests/studio/install/fake_llama_server.py +++ b/tests/studio/install/fake_llama_server.py @@ -55,7 +55,7 @@ LOGS = { } -def _arg(name, default=None): +def _arg(name, default = None): argv = sys.argv for i, token in enumerate(argv): if token == name and i + 1 < len(argv): diff --git a/tests/studio/install/run_smoke_spoof.py b/tests/studio/install/run_smoke_spoof.py index 2538e5ef52..aa94492c27 100644 --- a/tests/studio/install/run_smoke_spoof.py +++ b/tests/studio/install/run_smoke_spoof.py @@ -49,18 +49,21 @@ def make_wrapper(workdir: Path) -> Path: def run_smoke(wrapper: Path, probe: Path, install_kind: str, mode: str) -> int: - env = dict(os.environ, FAKE_LLAMA_MODE=mode) + env = dict(os.environ, FAKE_LLAMA_MODE = mode) proc = subprocess.run( [ sys.executable, str(INSTALLER), - "--smoke-test", str(wrapper), - "--probe", str(probe), - "--install-kind", install_kind, + "--smoke-test", + str(wrapper), + "--probe", + str(probe), + "--install-kind", + install_kind, ], - env=env, - capture_output=True, - text=True, + env = env, + capture_output = True, + text = True, ) sys.stdout.write(proc.stdout) sys.stderr.write(proc.stderr) @@ -86,7 +89,9 @@ def main() -> int: for mode, kind, expected, label in cases: rc = run_smoke(wrapper, probe, kind, mode) ok = rc == expected - print(f"[{'PASS' if ok else 'FAIL'}] {label}: mode={mode} kind={kind} exit={rc} (want {expected})") + print( + f"[{'PASS' if ok else 'FAIL'}] {label}: mode={mode} kind={kind} exit={rc} (want {expected})" + ) if not ok: failures.append(label) diff --git a/tests/studio/install/test_gpu_offload_spoof.py b/tests/studio/install/test_gpu_offload_spoof.py index f860b45151..389c96e5af 100644 --- a/tests/studio/install/test_gpu_offload_spoof.py +++ b/tests/studio/install/test_gpu_offload_spoof.py @@ -19,12 +19,16 @@ import pytest if sys.platform == "win32": - pytest.skip("POSIX-only (Windows covered by the spoof CI workflow)", allow_module_level = True) + pytest.skip( + "POSIX-only (Windows covered by the spoof CI workflow)", allow_module_level = True + ) PACKAGE_ROOT = Path(__file__).resolve().parents[3] MODULE_PATH = PACKAGE_ROOT / "studio" / "install_llama_prebuilt.py" FAKE_SERVER = Path(__file__).resolve().parent / "fake_llama_server.py" -SPEC = importlib.util.spec_from_file_location("studio_install_llama_prebuilt_e2e", MODULE_PATH) +SPEC = importlib.util.spec_from_file_location( + "studio_install_llama_prebuilt_e2e", MODULE_PATH +) M = importlib.util.module_from_spec(SPEC) sys.modules[SPEC.name] = M SPEC.loader.exec_module(M) @@ -58,10 +62,7 @@ def fake_server_binary(tmp_path): """A `llama-server` that execs the fake server, so validate_server runs it exactly as it would a real prebuilt binary.""" binary = tmp_path / "llama-server" - binary.write_text( - "#!/bin/sh\n" - f'exec "{sys.executable}" "{FAKE_SERVER}" "$@"\n' - ) + binary.write_text("#!/bin/sh\n" f'exec "{sys.executable}" "{FAKE_SERVER}" "$@"\n') binary.chmod(binary.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) return binary @@ -73,35 +74,78 @@ def _validate(binary, tmp_path, host, install_kind, mode, monkeypatch): M.validate_server(binary, probe, host, tmp_path, install_kind = install_kind) -def test_cpu_only_binary_tagged_cuda_is_rejected(fake_server_binary, tmp_path, monkeypatch): +def test_cpu_only_binary_tagged_cuda_is_rejected( + fake_server_binary, tmp_path, monkeypatch +): # The #5807 case: a binary that serves 200 but loaded the model on CPU. with pytest.raises(M.GpuOffloadFailure): - _validate(fake_server_binary, tmp_path, linux_cuda_host(), "linux-cuda", "cpu", monkeypatch) + _validate( + fake_server_binary, + tmp_path, + linux_cuda_host(), + "linux-cuda", + "cpu", + monkeypatch, + ) -def test_offloaded_zero_binary_tagged_cuda_is_rejected(fake_server_binary, tmp_path, monkeypatch): +def test_offloaded_zero_binary_tagged_cuda_is_rejected( + fake_server_binary, tmp_path, monkeypatch +): with pytest.raises(M.GpuOffloadFailure): _validate( - fake_server_binary, tmp_path, linux_cuda_host(), "linux-cuda", "offloaded_zero", monkeypatch + fake_server_binary, + tmp_path, + linux_cuda_host(), + "linux-cuda", + "offloaded_zero", + monkeypatch, ) def test_gpu_binary_tagged_cuda_passes(fake_server_binary, tmp_path, monkeypatch): - _validate(fake_server_binary, tmp_path, linux_cuda_host(), "linux-cuda", "cuda", monkeypatch) + _validate( + fake_server_binary, + tmp_path, + linux_cuda_host(), + "linux-cuda", + "cuda", + monkeypatch, + ) def test_gpu_buffer_format_passes(fake_server_binary, tmp_path, monkeypatch): - _validate(fake_server_binary, tmp_path, linux_cuda_host(), "linux-cuda", "cuda_buffer", monkeypatch) + _validate( + fake_server_binary, + tmp_path, + linux_cuda_host(), + "linux-cuda", + "cuda_buffer", + monkeypatch, + ) -def test_cpu_only_binary_tagged_cpu_is_accepted(fake_server_binary, tmp_path, monkeypatch): +def test_cpu_only_binary_tagged_cpu_is_accepted( + fake_server_binary, tmp_path, monkeypatch +): # A linux-cpu bundle is the intentional fallback; never GPU-gated. - _validate(fake_server_binary, tmp_path, linux_cuda_host(), "linux-cpu", "cpu", monkeypatch) + _validate( + fake_server_binary, tmp_path, linux_cuda_host(), "linux-cpu", "cpu", monkeypatch + ) -def test_no_signal_binary_tagged_cuda_is_accepted(fake_server_binary, tmp_path, monkeypatch): +def test_no_signal_binary_tagged_cuda_is_accepted( + fake_server_binary, tmp_path, monkeypatch +): # No offload evidence -> conservative: do not reject on no signal. - _validate(fake_server_binary, tmp_path, linux_cuda_host(), "linux-cuda", "no_signal", monkeypatch) + _validate( + fake_server_binary, + tmp_path, + linux_cuda_host(), + "linux-cuda", + "no_signal", + monkeypatch, + ) def test_smoke_test_cli_exit_codes(fake_server_binary, tmp_path, monkeypatch): @@ -118,9 +162,12 @@ def test_smoke_test_cli_exit_codes(fake_server_binary, tmp_path, monkeypatch): "argv", [ "install_llama_prebuilt.py", - "--smoke-test", str(fake_server_binary), - "--probe", str(probe), - "--install-kind", "linux-cuda", + "--smoke-test", + str(fake_server_binary), + "--probe", + str(probe), + "--install-kind", + "linux-cuda", ], ) return M.main() diff --git a/tests/studio/install/test_selection_logic.py b/tests/studio/install/test_selection_logic.py index d02e94e58e..1838bb889f 100644 --- a/tests/studio/install/test_selection_logic.py +++ b/tests/studio/install/test_selection_logic.py @@ -2374,7 +2374,10 @@ class TestGpuHostNoSilentCpuFallback: self._no_torch(monkeypatch) host = make_host(compute_caps = ["120"], driver_cuda_version = (13, 0)) plan = direct_linux_release_plan( - self._linux_release("cuda13-newer", "cpu"), host, "unslothai/llama.cpp", "latest" + self._linux_release("cuda13-newer", "cpu"), + host, + "unslothai/llama.cpp", + "latest", ) kinds = [a.install_kind for a in plan.attempts] assert "linux-cuda" in kinds @@ -2403,7 +2406,10 @@ class TestGpuHostNoSilentCpuFallback: compute_caps = [], ) plan = direct_linux_release_plan( - self._linux_release("cuda13-newer", "cpu"), host, "unslothai/llama.cpp", "latest" + self._linux_release("cuda13-newer", "cpu"), + host, + "unslothai/llama.cpp", + "latest", ) assert [a.install_kind for a in plan.attempts] == ["linux-cpu"] diff --git a/tests/studio/install/test_validate_server_gpu_offload.py b/tests/studio/install/test_validate_server_gpu_offload.py index 1182c7117b..561ea94ab8 100644 --- a/tests/studio/install/test_validate_server_gpu_offload.py +++ b/tests/studio/install/test_validate_server_gpu_offload.py @@ -234,7 +234,9 @@ def test_device_row_case_insensitive(): def test_crlf_log_parses_identically(): # Windows logs use CRLF; classification must not change. - assert server_log_shows_gpu_offload(CUDA_DEVICE_INFO_LOG.replace("\n", "\r\n")) is True + assert ( + server_log_shows_gpu_offload(CUDA_DEVICE_INFO_LOG.replace("\n", "\r\n")) is True + ) assert ( server_log_shows_gpu_offload(CPU_ONLY_DEVICE_INFO_LOG.replace("\n", "\r\n")) is False From 17070c08863f9d2fe122c01a6a39124442e0bf47 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 1 Jun 2026 16:03:21 +0000 Subject: [PATCH 05/12] Studio: harden GPU-offload classifier against CUDA_Host and split offload lines Exclude host-pinned buffers (CUDA_Host etc.) from the GPU buffer-size signal so a binary that pins host memory but loads weights on CPU is not misread as GPU offload. Scan every 'offloaded N/M layers to GPU' line and accept if any N>0 so a speculative draft model logging 0/k before the main model's 33/33 is not flagged CPU-only. Same fix in the installer and runtime classifiers. --- studio/backend/core/inference/llama_cpp.py | 16 ++++++++++-- .../tests/test_llama_cpp_context_fit.py | 19 ++++++++++++++ studio/install_llama_prebuilt.py | 25 +++++++++++++----- .../test_validate_server_gpu_offload.py | 26 +++++++++++++++++++ 4 files changed, 78 insertions(+), 8 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 9001396afe..67758b7073 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -90,22 +90,34 @@ def classify_gpu_offload_lines(lines: list[str]) -> Optional[bool]: """True if the model landed on a GPU, False if it stayed on CPU despite GPU intent, None when the log has no usable signal. Priority: buffer-size lines, then offloaded-layers count, then device_info enumeration.""" + # Exclude host-pinned buffers ("CUDA_Host" ...): CPU RAM the GPU backend + # pinned, not device memory, so they must not read as GPU offload. saw_buffer_line = False for line in lines: if "buffer size" not in line: continue - if any(marker in line for marker in _GPU_BUFFER_MARKERS): + if "_Host" not in line and any( + marker in line for marker in _GPU_BUFFER_MARKERS + ): return True if "model buffer size" in line: saw_buffer_line = True + # Accept if any "offloaded N/M" has N>0 (a draft model can log 0/k before + # the main model's 33/33); CPU-only only when every offloaded line is zero. + saw_offloaded = False for line in lines: match = _OFFLOADED_LAYERS_RE.search(line) if match: - return int(match.group(1)) > 0 + saw_offloaded = True + if int(match.group(1)) > 0: + return True + continue low = line.lower() if "offloading" in low and "to gpu" in low: return True + if saw_offloaded: + return False after_device_info = False saw_device_row = False diff --git a/studio/backend/tests/test_llama_cpp_context_fit.py b/studio/backend/tests/test_llama_cpp_context_fit.py index 3d295f711c..0e069684c7 100644 --- a/studio/backend/tests/test_llama_cpp_context_fit.py +++ b/studio/backend/tests/test_llama_cpp_context_fit.py @@ -621,3 +621,22 @@ class TestClassifyGpuOffload: ["system_info: n_threads = 8 | CUDA : ARCHS = 1200 | CPU : AVX2 = 1"] ) assert inst._classify_gpu_offload(True, [(0, 22805)]) is None + + def test_cuda_host_buffer_is_not_gpu(self): + # CUDA_Host is host-pinned CPU RAM; weights on CPU_Mapped means CPU only. + inst = self._backend( + [ + "load_tensors: CUDA_Host model buffer size = 21000.0 MiB", + "load_tensors: CPU_Mapped model buffer size = 0.6 MiB", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is False + + def test_draft_zero_before_main_offload_is_gpu(self): + inst = self._backend( + [ + "load_tensors: offloaded 0/2 layers to GPU", + "load_tensors: offloaded 33/33 layers to GPU", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is True diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index 151c459924..c1c520f5f9 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -5665,26 +5665,39 @@ def server_log_shows_gpu_offload(log_text: str) -> bool | None: # Signal 1: per-backend buffer-size lines. A GPU marker on ANY "buffer # size" line (model / KV / compute) means the GPU holds part of the model - # -- accept. Only the model-buffer location decides CPU-only (KV/compute - # CPU buffers exist even on GPU runs), so the False determination keys on - # "model buffer size" alone. + # -- accept. Exclude host-pinned buffers ("CUDA_Host" / "ROCm_Host" ...): + # those are CPU RAM the GPU backend pinned, not device memory, so a binary + # that pins host memory but offloads no weights must not read as GPU. Only + # the model-buffer location decides CPU-only (KV/compute CPU buffers exist + # even on GPU runs), so the False determination keys on "model buffer size". saw_buffer_line = False for line in lines: if "buffer size" not in line: continue - if any(marker in line for marker in _GPU_MODEL_BUFFER_MARKERS): + if "_Host" not in line and any( + marker in line for marker in _GPU_MODEL_BUFFER_MARKERS + ): return True if "model buffer size" in line: saw_buffer_line = True - # Signal 2: explicit offloaded-layers count. + # Signal 2: explicit offloaded-layers count. Scan every "offloaded N/M" + # line and accept if any has N>0 (a draft/speculative model can log + # "offloaded 0/k" before the main model's "offloaded 33/33"); only when all + # offloaded lines are zero is it CPU-only. + saw_offloaded = False for line in lines: match = _OFFLOADED_LAYERS_RE.search(line) if match: - return int(match.group(1)) > 0 + saw_offloaded = True + if int(match.group(1)) > 0: + return True + continue low = line.lower() if "offloading" in low and "to gpu" in low: return True + if saw_offloaded: + return False # Signal 3: device_info enumeration. Only trust device rows once the # "device_info:" header has appeared, so the compiled-backend system_info diff --git a/tests/studio/install/test_validate_server_gpu_offload.py b/tests/studio/install/test_validate_server_gpu_offload.py index 561ea94ab8..49c6cb8ad9 100644 --- a/tests/studio/install/test_validate_server_gpu_offload.py +++ b/tests/studio/install/test_validate_server_gpu_offload.py @@ -232,6 +232,32 @@ def test_device_row_case_insensitive(): assert server_log_shows_gpu_offload(log) is True +def test_cuda_host_buffer_is_not_gpu_offload(): + # CUDA_Host is host-pinned CPU RAM, not device memory. A binary that pins + # host memory but loads weights on CPU must not pass as GPU offload. + log = ( + "load_tensors: CUDA_Host model buffer size = 21000.0 MiB\n" + "load_tensors: CPU_Mapped model buffer size = 0.6 MiB\n" + ) + assert server_log_shows_gpu_offload(log) is False + # A real device buffer alongside a CUDA_Host line still reads as GPU. + log_ok = ( + "load_tensors: CUDA_Host model buffer size = 100.0 MiB\n" + "load_tensors: CUDA0 model buffer size = 21000.0 MiB\n" + ) + assert server_log_shows_gpu_offload(log_ok) is True + + +def test_draft_offloaded_zero_before_main_offload_is_gpu(): + # Speculative decoding: a draft model logs "offloaded 0/2" before the main + # model's "offloaded 33/33". The N>0 line must win. + log = ( + "load_tensors: offloaded 0/2 layers to GPU\n" + "load_tensors: offloaded 33/33 layers to GPU\n" + ) + assert server_log_shows_gpu_offload(log) is True + + def test_crlf_log_parses_identically(): # Windows logs use CRLF; classification must not change. assert ( From 1546f0328e87a9e96f2a5e30d0b6671570ff85f3 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 1 Jun 2026 16:14:46 +0000 Subject: [PATCH 06/12] Studio: make explicit offloaded-layer count authoritative in GPU classifier Reviewers found a real CPU-only log shape that passed: an 'offloading 0 repeating layers to GPU' planning line, or a GPU KV/compute buffer, could read as GPU before the definitive 'offloaded 0/33' was seen. Check the explicit counted offload first (any N>0 wins, all zero is CPU-only), restrict the buffer-size signal to GPU model buffers (KV/compute on GPU with weights on CPU is still CPU inference), and add HIP/MUSA/CANN to the model-buffer markers so an older log naming those backends is not misread as CPU. Same in both classifiers. --- studio/backend/core/inference/llama_cpp.py | 72 +++++++++----- .../tests/test_llama_cpp_context_fit.py | 16 ++++ studio/install_llama_prebuilt.py | 94 ++++++++++++------- .../test_validate_server_gpu_offload.py | 31 +++++- 4 files changed, 148 insertions(+), 65 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 67758b7073..9d3dd8e5bb 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -65,7 +65,18 @@ logger = get_logger(__name__) # #5106 / #5830). Robust across llama.cpp log formats (the "model buffer size" # lines were dropped in recent builds): buffer-size markers, "offloaded N/M # layers to GPU", and the "device_info:" enumeration. -_GPU_BUFFER_MARKERS = ("CUDA", "ROCm", "Metal", "Vulkan", "OpenCL", "SYCL") +_GPU_BUFFER_MARKERS = ( + "CUDA", + "ROCm", + "ROCM", + "HIP", + "Metal", + "Vulkan", + "OpenCL", + "SYCL", + "MUSA", + "CANN", +) _DEVICE_ROW_RE = re.compile( r"-\s*(?P(?:CUDA|ROCm|ROCM|HIP|Metal|Vulkan|SYCL|OpenCL|MUSA|CANN|CPU)\w*)\s*:", re.IGNORECASE, @@ -84,40 +95,49 @@ _GPU_DEVICE_PREFIXES = ( _OFFLOADED_LAYERS_RE = re.compile( r"offloaded\s+(\d+)\s*/\s*(\d+)\s+layers?\s+to\s+gpu", re.IGNORECASE ) +_OFFLOADING_COUNT_RE = re.compile( + r"offloading\s+(\d+)\s+(?:repeating\s+|non-repeating\s+)?layers?\s+to\s+gpu", + re.IGNORECASE, +) def classify_gpu_offload_lines(lines: list[str]) -> Optional[bool]: """True if the model landed on a GPU, False if it stayed on CPU despite GPU - intent, None when the log has no usable signal. Priority: buffer-size lines, - then offloaded-layers count, then device_info enumeration.""" - # Exclude host-pinned buffers ("CUDA_Host" ...): CPU RAM the GPU backend - # pinned, not device memory, so they must not read as GPU offload. - saw_buffer_line = False + intent, None when the log has no usable signal. Priority: explicit + offloaded-layer counts (authoritative), then GPU model-buffer lines, then + device_info enumeration.""" + # Signal 1: explicit offloaded counts win over everything (a KV/compute + # buffer can be on the GPU while 0 model layers are offloaded). Any counted + # line N>0 => True; all 0 => False (scan all; a draft model can log 0/k + # before the main model's 33/33). Uncounted "offloading output layer to GPU" + # is only a weak positive when no counted line exists. + saw_zero_count = False + saw_uncounted_offloading = False for line in lines: - if "buffer size" not in line: + match = _OFFLOADED_LAYERS_RE.search(line) or _OFFLOADING_COUNT_RE.search(line) + if match: + if int(match.group(1)) > 0: + return True + saw_zero_count = True continue + low = line.lower() + if "offloading" in low and "to gpu" in low: + saw_uncounted_offloading = True + if saw_zero_count: + return False + if saw_uncounted_offloading: + return True + + # Signal 2: GPU marker on a *model* buffer (exclude host-pinned _Host). + saw_model_buffer = False + for line in lines: + if "model buffer size" not in line: + continue + saw_model_buffer = True if "_Host" not in line and any( marker in line for marker in _GPU_BUFFER_MARKERS ): return True - if "model buffer size" in line: - saw_buffer_line = True - - # Accept if any "offloaded N/M" has N>0 (a draft model can log 0/k before - # the main model's 33/33); CPU-only only when every offloaded line is zero. - saw_offloaded = False - for line in lines: - match = _OFFLOADED_LAYERS_RE.search(line) - if match: - saw_offloaded = True - if int(match.group(1)) > 0: - return True - continue - low = line.lower() - if "offloading" in low and "to gpu" in low: - return True - if saw_offloaded: - return False after_device_info = False saw_device_row = False @@ -135,7 +155,7 @@ def classify_gpu_offload_lines(lines: list[str]) -> Optional[bool]: if any(dev.startswith(prefix) for prefix in _GPU_DEVICE_PREFIXES): return True - if saw_buffer_line or saw_device_row: + if saw_model_buffer or saw_device_row: return False return None diff --git a/studio/backend/tests/test_llama_cpp_context_fit.py b/studio/backend/tests/test_llama_cpp_context_fit.py index 0e069684c7..ed82d4fa96 100644 --- a/studio/backend/tests/test_llama_cpp_context_fit.py +++ b/studio/backend/tests/test_llama_cpp_context_fit.py @@ -640,3 +640,19 @@ class TestClassifyGpuOffload: ] ) assert inst._classify_gpu_offload(True, [(0, 22805)]) is True + + def test_offloading_zero_repeating_does_not_mask_zero_total(self): + inst = self._backend( + [ + "llm_load_tensors: offloading 0 repeating layers to GPU", + "llm_load_tensors: offloaded 0/33 layers to GPU", + "llm_load_tensors: CPU model buffer size = 7338.64 MiB", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is False + + def test_hip_model_buffer_is_gpu(self): + inst = self._backend( + ["load_tensors: HIP0 model buffer size = 21000.0 MiB"] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is True diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index c1c520f5f9..97962311fc 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -5604,7 +5604,18 @@ def validate_quantize( # llama.cpp). CUDA0 / ROCm0 / Metal / Vulkan0 / OpenCL0 / SYCL0 / HIP0 / MUSA0 # / CANN0 are GPU; CPU / CPU_Mapped are not. Kept broad so a single helper # covers every backend Studio ships. -_GPU_MODEL_BUFFER_MARKERS = ("CUDA", "ROCm", "Metal", "Vulkan", "OpenCL", "SYCL") +_GPU_MODEL_BUFFER_MARKERS = ( + "CUDA", + "ROCm", + "ROCM", + "HIP", + "Metal", + "Vulkan", + "OpenCL", + "SYCL", + "MUSA", + "CANN", +) # A device_info row looks like " I - CUDA0 : NVIDIA B200 (...)" or # " I - CPU : ...". Matched on the "- :" shape (the leading @@ -5630,10 +5641,18 @@ _GPU_DEVICE_PREFIXES = ( "cann", ) -# "load_tensors: offloaded 33/33 layers to GPU" (or "offloading ... to GPU"). +# "load_tensors: offloaded 33/33 layers to GPU". _OFFLOADED_LAYERS_RE = re.compile( r"offloaded\s+(\d+)\s*/\s*(\d+)\s+layers?\s+to\s+gpu", re.IGNORECASE ) +# "offloading 0 repeating layers to GPU" / "offloading 1 non-repeating ...". +# A counted form, so a zero here is a definite CPU-only signal (it usually +# precedes "offloaded 0/N"); kept separate from the uncounted +# "offloading output layer to GPU" phrasing, which carries no number. +_OFFLOADING_COUNT_RE = re.compile( + r"offloading\s+(\d+)\s+(?:repeating\s+|non-repeating\s+)?layers?\s+to\s+gpu", + re.IGNORECASE, +) # install_kind values that ship a real GPU backend and must offload to the GPU. # A binary launched with --n-gpu-layers 1 under one of these is held to the @@ -5649,9 +5668,12 @@ def server_log_shows_gpu_offload(log_text: str) -> bool | None: Robust across llama.cpp log formats (the "model buffer size" lines were dropped in recent builds), in priority order: - 1. `` model buffer size = N`` lines -- GPU marker => True - (older llama.cpp). - 2. ``offloaded N/M layers to GPU`` -- N>0 => True, 0/M => False. + 1. ``offloaded N/M layers to GPU`` / ``offloading N ... layers to GPU`` -- + the explicit layer count is authoritative: N>0 => True, all 0 => False. + It must win over auxiliary GPU buffers (a KV/compute buffer can sit on + the GPU while 0 model layers are offloaded -- still CPU inference). + 2. `` model buffer size = N`` lines -- a GPU marker on a *model* + buffer (not host-pinned ``CUDA_Host``) => True (older llama.cpp). 3. ``device_info:`` enumeration -- a GPU device row (``- CUDA0 :`` etc.) => True; only a ``- CPU :`` row => False. This is the version-stable signal and matches how the CPU-only-binary bug is diagnosed in the @@ -5663,41 +5685,43 @@ def server_log_shows_gpu_offload(log_text: str) -> bool | None: """ lines = log_text.splitlines() - # Signal 1: per-backend buffer-size lines. A GPU marker on ANY "buffer - # size" line (model / KV / compute) means the GPU holds part of the model - # -- accept. Exclude host-pinned buffers ("CUDA_Host" / "ROCm_Host" ...): - # those are CPU RAM the GPU backend pinned, not device memory, so a binary - # that pins host memory but offloads no weights must not read as GPU. Only - # the model-buffer location decides CPU-only (KV/compute CPU buffers exist - # even on GPU runs), so the False determination keys on "model buffer size". - saw_buffer_line = False + # Signal 1: explicit offloaded-layers counts (authoritative). Any counted + # line with N>0 => True; if every counted line is 0 => False (a draft model + # can log "offloaded 0/k" before the main model's "offloaded 33/33", so scan + # all before deciding). The uncounted "offloading output layer to GPU" + # phrasing carries no number, so it is only a weak positive used when no + # counted line exists at all. + saw_zero_count = False + saw_uncounted_offloading = False for line in lines: - if "buffer size" not in line: + match = _OFFLOADED_LAYERS_RE.search(line) or _OFFLOADING_COUNT_RE.search(line) + if match: + if int(match.group(1)) > 0: + return True + saw_zero_count = True continue + low = line.lower() + if "offloading" in low and "to gpu" in low: + saw_uncounted_offloading = True + if saw_zero_count: + return False + if saw_uncounted_offloading: + return True + + # Signal 2: per-backend model-buffer lines. A GPU marker on a *model* buffer + # means model weights live on the GPU. Exclude host-pinned buffers + # ("CUDA_Host" / "ROCm_Host" ...): those are CPU RAM the GPU backend pinned, + # not device memory. KV/compute buffers are ignored here -- they can be on + # the GPU even when all weights are on CPU. + saw_model_buffer = False + for line in lines: + if "model buffer size" not in line: + continue + saw_model_buffer = True if "_Host" not in line and any( marker in line for marker in _GPU_MODEL_BUFFER_MARKERS ): return True - if "model buffer size" in line: - saw_buffer_line = True - - # Signal 2: explicit offloaded-layers count. Scan every "offloaded N/M" - # line and accept if any has N>0 (a draft/speculative model can log - # "offloaded 0/k" before the main model's "offloaded 33/33"); only when all - # offloaded lines are zero is it CPU-only. - saw_offloaded = False - for line in lines: - match = _OFFLOADED_LAYERS_RE.search(line) - if match: - saw_offloaded = True - if int(match.group(1)) > 0: - return True - continue - low = line.lower() - if "offloading" in low and "to gpu" in low: - return True - if saw_offloaded: - return False # Signal 3: device_info enumeration. Only trust device rows once the # "device_info:" header has appeared, so the compiled-backend system_info @@ -5718,7 +5742,7 @@ def server_log_shows_gpu_offload(log_text: str) -> bool | None: if any(dev.startswith(prefix) for prefix in _GPU_DEVICE_PREFIXES): return True - if saw_buffer_line or saw_device_row: + if saw_model_buffer or saw_device_row: # We had a concrete signal, but every buffer/device was CPU. return False return None diff --git a/tests/studio/install/test_validate_server_gpu_offload.py b/tests/studio/install/test_validate_server_gpu_offload.py index 49c6cb8ad9..6b0afc9de0 100644 --- a/tests/studio/install/test_validate_server_gpu_offload.py +++ b/tests/studio/install/test_validate_server_gpu_offload.py @@ -216,17 +216,40 @@ def test_signal2_offloaded_zero_beats_device_info(): assert server_log_shows_gpu_offload(log) is False -def test_gpu_buffer_size_without_model_word(): - # A GPU-marked buffer line that omits the word "model" (e.g. KV/compute or - # a future format) must still count as GPU offload, even when a CPU - # "model buffer size" line is present (broadened signal 1). +def test_kv_buffer_on_gpu_with_cpu_model_is_not_offload(): + # A GPU KV/compute buffer while the model weights sit on CPU is still CPU + # inference; only a GPU *model* buffer counts as offload. log = ( "load_tensors: CUDA0 KV buffer size = 100.0 MiB\n" "load_tensors: CPU_Mapped model buffer size = 0.6 MiB\n" ) + assert server_log_shows_gpu_offload(log) is False + + +def test_offloading_zero_repeating_does_not_mask_zero_total(): + # Real CPU-only shape: a planning line says "offloading 0 repeating layers" + # before the definitive "offloaded 0/33". The count must decide -> False. + log = ( + "llm_load_tensors: offloading 0 repeating layers to GPU\n" + "llm_load_tensors: offloaded 0/33 layers to GPU\n" + "llm_load_tensors: CPU model buffer size = 7338.64 MiB\n" + ) + assert server_log_shows_gpu_offload(log) is False + + +def test_uncounted_offloading_output_layer_is_gpu(): + # The uncounted "offloading output layer to GPU" phrasing (no number, no + # later zero count) is a weak positive. + log = "llm_load_tensors: offloading output layer to GPU\n" assert server_log_shows_gpu_offload(log) is True +def test_hip_musa_cann_model_buffer_is_gpu(): + for marker in ("HIP0", "MUSA0", "CANN0"): + log = f"load_tensors: {marker} model buffer size = 21000.0 MiB\n" + assert server_log_shows_gpu_offload(log) is True, marker + + def test_device_row_case_insensitive(): log = "device_info:\n - cuda0 : some gpu (free)\n - CPU : x (free)\n" assert server_log_shows_gpu_offload(log) is True From ecf90078dd9107f25ee3085c1641fa9658a20643 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 1 Jun 2026 16:16:22 +0000 Subject: [PATCH 07/12] Studio: forward the resolved GPU kind to the source-build smoke test The post-build smoke test relied on the installer re-detecting the GPU, but on amd-smi-only or name-inferred ROCm hosts that probe can miss the GPU and resolve a CPU kind, skipping the offload gate. setup.sh/setup.ps1 now pass --install-kind from the backend they just built, and the --smoke-test CLI applies --has-rocm / --rocm-gfx host overrides like the install path does. --- studio/install_llama_prebuilt.py | 11 ++++++++++- studio/setup.ps1 | 5 ++++- studio/setup.sh | 16 ++++++++++++++-- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index 97962311fc..705ba708d2 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -7092,7 +7092,16 @@ def main() -> int: return EXIT_SUCCESS if args.smoke_test is not None: - host = detect_host() + # Honor the same host overrides as the install path so a caller that + # forwards --has-rocm / --rocm-gfx (setup.sh/setup.ps1 do, because the + # installer's own probe can miss amd-smi-only hosts) is held to the GPU + # offload check instead of silently resolving a CPU kind. + host = _apply_host_overrides( + detect_host(), + override_has_rocm = args.has_rocm, + override_rocm_gfx = args.rocm_gfx, + force_cpu = args.cpu_fallback, + ) try: resolved_kind = smoke_test_server_binary( args.smoke_test, diff --git a/studio/setup.ps1 b/studio/setup.ps1 index b52b02a8d2..3a4e9d4ca3 100644 --- a/studio/setup.ps1 +++ b/studio/setup.ps1 @@ -3051,7 +3051,10 @@ if (-not $NeedLlamaSourceBuild) { if (Test-Path -LiteralPath $builtServer) { Write-Host "" Write-Host "--- GPU smoke test ---" -ForegroundColor Cyan - & python "$PSScriptRoot\install_llama_prebuilt.py" --smoke-test "$builtServer" --install-dir "$LlamaCppDir" 2>&1 | Out-String | Write-Host + # $LlamaCudaBuild gates this block, so the build is CUDA; pass the + # explicit kind so the installer's own probe cannot resolve a CPU + # kind and skip the offload gate. + & python "$PSScriptRoot\install_llama_prebuilt.py" --smoke-test "$builtServer" --install-dir "$LlamaCppDir" --install-kind "windows-cuda" 2>&1 | Out-String | Write-Host $smokeExit = $LASTEXITCODE if ($smokeExit -eq 2) { substep "GPU build runs on CPU only (GPU backend failed to initialize)" "Yellow" diff --git a/studio/setup.sh b/studio/setup.sh index b941fd6a82..10aa831b72 100755 --- a/studio/setup.sh +++ b/studio/setup.sh @@ -1362,11 +1362,23 @@ else # _gpu_fallback_label is empty for a pure CPU build (nothing to verify). if [ "$BUILD_OK" = true ]; then _SMOKE_LABEL="$(_gpu_fallback_label)" - if [ -n "$_SMOKE_LABEL" ] && [ -f "$_BUILD_TMP/build/bin/llama-server" ]; then + # Pass the GPU kind setup resolved, not the installer's own probe: + # on amd-smi-only / name-inferred ROCm hosts the child probe can miss + # the GPU and resolve a CPU kind, which would skip the offload gate. + _SMOKE_KIND="" + if [ "$_TRY_METAL_CPU_FALLBACK" = true ]; then + _SMOKE_KIND="macos-arm64" + elif [ "$GPU_BACKEND" = "cuda" ]; then + _SMOKE_KIND="linux-cuda" + elif [ "$GPU_BACKEND" = "rocm" ]; then + _SMOKE_KIND="linux-rocm" + fi + if [ -n "$_SMOKE_LABEL" ] && [ -n "$_SMOKE_KIND" ] && [ -f "$_BUILD_TMP/build/bin/llama-server" ]; then # if/else keeps set -e from aborting before we read the code. if python "$SCRIPT_DIR/install_llama_prebuilt.py" \ --smoke-test "$_BUILD_TMP/build/bin/llama-server" \ - --install-dir "$_BUILD_TMP" > "$_BUILD_TMP/gpu-smoke.log" 2>&1; then + --install-dir "$_BUILD_TMP" \ + --install-kind "$_SMOKE_KIND" > "$_BUILD_TMP/gpu-smoke.log" 2>&1; then _SMOKE_RC=0 else _SMOKE_RC=$? From 100e27ffe76425b053c5f4d7fd9ddbc5605c6939 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 1 Jun 2026 16:23:35 +0000 Subject: [PATCH 08/12] Studio: re-validate matching existing installs and reused source builds (#5807) A metadata match used to short-circuit straight to reuse, so a previously installed CPU-only 'GPU' binary survived every rerun and restart -- the exact 'picks CPU forever' report. install_prebuilt now smoke-tests a matching GPU install and reinstalls if it loads on CPU; setup.sh/setup.ps1 smoke-test a reused source build on a GPU host and rebuild if it ran on CPU. Non-GPU installs keep the fast path. --- studio/install_llama_prebuilt.py | 70 ++++++++++++++-- studio/setup.ps1 | 10 +++ studio/setup.sh | 39 +++++++-- .../test_validate_server_gpu_offload.py | 81 +++++++++++++++++++ 4 files changed, 190 insertions(+), 10 deletions(-) diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index 705ba708d2..f7d4b1fa21 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -6704,6 +6704,44 @@ def validate_prebuilt_attempts( raise PrebuiltFallback("no prebuilt bundle passed validation") +def existing_gpu_install_offloads( + install_dir: Path, + host: HostInfo, + plan: InstallReleasePlan, + probe_path: Path, +) -> bool: + """For a GPU plan whose metadata matches the existing install, smoke-test + the already-installed llama-server's offload before reusing it. Returns True + to keep the existing install (non-GPU kind, passed, or inconclusive) and + False on a definite CPU-only load so the caller reinstalls. This closes the + "reinstall/restart keeps the silently CPU-only binary" path (#5807): without + it, a metadata match short-circuits the new offload validation entirely.""" + choice = plan.attempts[0] + if choice.install_kind not in _GPU_INSTALL_KINDS: + return True + server_name = "llama-server.exe" if host.is_windows else "llama-server" + try: + server_path = discover_installed_executable(install_dir, server_name) + except PrebuiltFallback: + return True + try: + validate_server( + server_path, + probe_path, + host, + install_dir, + runtime_line = choice.runtime_line, + install_kind = choice.install_kind, + ) + except GpuOffloadFailure: + return False + except PrebuiltFallback: + # Inconclusive (failed to start, etc.): keep the existing install rather + # than reinstall on uncertain evidence. + return True + return True + + def install_prebuilt( install_dir: Path, llama_tag: str, @@ -6747,8 +6785,12 @@ def install_prebuilt( published_repo, published_release_tag, ) - if release_plans and existing_install_matches_plan( - install_dir, host, release_plans[0] + # Non-GPU match: keep the fast path (no probe download). A GPU match + # must still be smoke-tested below, so it does not short-circuit here. + if ( + release_plans + and existing_install_matches_plan(install_dir, host, release_plans[0]) + and release_plans[0].attempts[0].install_kind not in _GPU_INSTALL_KINDS ): current = release_plans[0] log( @@ -6765,9 +6807,27 @@ def install_prebuilt( release_count = len(release_plans) for release_index, plan in enumerate(release_plans): choice = plan.attempts[0] - if existing_install_matches_plan(install_dir, host, plan): + # A metadata match used to skip straight to reuse; now a + # matching GPU install is smoke-tested first so a previously + # installed CPU-only "GPU" binary is rebuilt instead of kept + # forever across reruns/restarts (#5807). + matched = existing_install_matches_plan(install_dir, host, plan) + existing_install_dir: Path | None = install_dir + if matched and not existing_gpu_install_offloads( + install_dir, host, plan, probe_path + ): log( - "existing llama.cpp install already matches fallback release " + "existing GPU llama.cpp install served a completion but " + "loaded the model on CPU; reinstalling to recover GPU " + "offload (unslothai/unsloth#5807)" + ) + matched = False + # Don't let validate_prebuilt_attempts short-circuit on + # the same bad install we just rejected. + existing_install_dir = None + if matched: + log( + "existing llama.cpp install already matches release " f"{plan.release_tag} upstream_tag={plan.llama_tag}; skipping reinstall" ) return @@ -6788,7 +6848,7 @@ def install_prebuilt( release_tag = plan.release_tag, approved_checksums = plan.approved_checksums, initial_fallback_used = release_index > 0, - existing_install_dir = install_dir, + existing_install_dir = existing_install_dir, ) except ExistingInstallSatisfied: return diff --git a/studio/setup.ps1 b/studio/setup.ps1 index 3a4e9d4ca3..da4729f135 100644 --- a/studio/setup.ps1 +++ b/studio/setup.ps1 @@ -2633,6 +2633,16 @@ if (Test-Path -LiteralPath $LlamaServerBin) { Write-Host " Existing llama-server was built with CUDA but no GPU detected -- rebuilding" -ForegroundColor Yellow $NeedRebuild = $true } + # The cache check catches a CPU build on a GPU host; this catches a CUDA + # build that still loads on CPU at runtime (PTX / runtime-init failure, + # #5807). Smoke-test before reusing it on a GPU host. + if (-not $NeedRebuild -and $HasNvidiaSmi -and $cachedCuda) { + & python "$PSScriptRoot\install_llama_prebuilt.py" --smoke-test "$LlamaServerBin" --install-dir "$LlamaCppDir" --install-kind "windows-cuda" 2>&1 | Out-String | Write-Host + if ($LASTEXITCODE -eq 2) { + Write-Host " Existing CUDA llama-server runs on CPU only -- rebuilding" -ForegroundColor Yellow + $NeedRebuild = $true + } + } } } diff --git a/studio/setup.sh b/studio/setup.sh index 10aa831b72..a16488c0b8 100755 --- a/studio/setup.sh +++ b/studio/setup.sh @@ -919,12 +919,41 @@ if [ "$_NEED_LLAMA_SOURCE_BUILD" = true ] && \ [ -z "$_LLAMA_PR" ] && \ [ -x "$LLAMA_CPP_DIR/build/bin/llama-server" ] && \ [ -x "$LLAMA_CPP_DIR/build/bin/llama-quantize" ]; then - step "llama.cpp" "existing source build found; skipping rebuild" - ln -sf build/bin/llama-quantize "$LLAMA_CPP_DIR/llama-quantize" - if [ "$_STUDIO_HOME_IS_CUSTOM" = true ]; then - : > "$LLAMA_CPP_DIR/$_STUDIO_OWNED_MARKER" 2>/dev/null || true + _REUSE_SOURCE=true + # On a GPU host, smoke-test the existing source binary first so a stale + # CPU-only build (e.g. an earlier no-toolkit fallback) is rebuilt instead of + # silently reused on a GPU host (#5807). Exit 2 = ran on CPU -> rebuild; + # anything else keeps the build (never downgrade on uncertain evidence). + _REUSE_KIND="" + if [ "$_HOST_SYSTEM" = "Darwin" ] && { [ "$_HOST_MACHINE" = "arm64" ] || [ "$_HOST_MACHINE" = "aarch64" ]; }; then + _REUSE_KIND="macos-arm64" + elif command -v nvidia-smi >/dev/null 2>&1; then + _REUSE_KIND="linux-cuda" + elif [ "$_LINUX_HAS_GPU" = true ]; then + _REUSE_KIND="linux-rocm" + fi + if [ -n "$_REUSE_KIND" ]; then + if python "$SCRIPT_DIR/install_llama_prebuilt.py" \ + --smoke-test "$LLAMA_CPP_DIR/build/bin/llama-server" \ + --install-dir "$LLAMA_CPP_DIR" \ + --install-kind "$_REUSE_KIND" > "$LLAMA_CPP_DIR/gpu-smoke-existing.log" 2>&1; then + _REUSE_RC=0 + else + _REUSE_RC=$? + fi + if [ "$_REUSE_RC" -eq 2 ]; then + substep "existing source build runs on CPU only; rebuilding for GPU..." "$C_WARN" + _REUSE_SOURCE=false + fi + fi + if [ "$_REUSE_SOURCE" = true ]; then + step "llama.cpp" "existing source build found; skipping rebuild" + ln -sf build/bin/llama-quantize "$LLAMA_CPP_DIR/llama-quantize" + if [ "$_STUDIO_HOME_IS_CUSTOM" = true ]; then + : > "$LLAMA_CPP_DIR/$_STUDIO_OWNED_MARKER" 2>/dev/null || true + fi + _NEED_LLAMA_SOURCE_BUILD=false fi - _NEED_LLAMA_SOURCE_BUILD=false fi # ── 8. WSL: pre-install GGUF build dependencies for fallback source builds ── diff --git a/tests/studio/install/test_validate_server_gpu_offload.py b/tests/studio/install/test_validate_server_gpu_offload.py index 6b0afc9de0..cc64d2560d 100644 --- a/tests/studio/install/test_validate_server_gpu_offload.py +++ b/tests/studio/install/test_validate_server_gpu_offload.py @@ -499,3 +499,84 @@ def test_main_smoke_exit_error_on_inconclusive(monkeypatch): rc = _run_main_smoke(monkeypatch, impl) assert rc == M.EXIT_ERROR # 1 -> setup scripts keep the GPU build + + +# -- 6. existing_gpu_install_offloads: re-validate a matching install (#5807) -- + + +def _gpu_plan(install_kind = "linux-cuda"): + choice = M.AssetChoice( + repo = "unslothai/llama.cpp", + tag = "b9001", + name = "app-b9001-linux-x64-cuda13-newer.tar.gz", + url = "https://example.com/x", + source_label = "published", + install_kind = install_kind, + ) + return M.InstallReleasePlan( + requested_tag = "latest", + llama_tag = "b9001", + release_tag = "rel", + attempts = [choice], + approved_checksums = M.ApprovedReleaseChecksums( + repo = "unslothai/llama.cpp", release_tag = "rel", upstream_tag = "b9001", + artifacts = {}, + ), + ) + + +def _with_server(tmp_path): + server = tmp_path / "llama-server" + server.write_text("#!/bin/sh\n") + probe = tmp_path / "probe.gguf" + probe.write_bytes(b"GGUF") + return probe + + +def test_existing_cpu_kind_install_is_kept(tmp_path): + # A non-GPU existing install is never offload-gated. + probe = _with_server(tmp_path) + assert M.existing_gpu_install_offloads( + tmp_path, nvidia_host(), _gpu_plan("linux-cpu"), probe + ) is True + + +def test_existing_gpu_install_cpu_only_triggers_reinstall(monkeypatch, tmp_path): + probe = _with_server(tmp_path) + + def fake_validate(*a, **k): + raise M.GpuOffloadFailure("loaded the model entirely on CPU") + + monkeypatch.setattr(M, "validate_server", fake_validate) + assert M.existing_gpu_install_offloads( + tmp_path, nvidia_host(), _gpu_plan("linux-cuda"), probe + ) is False + + +def test_existing_gpu_install_offloading_is_kept(monkeypatch, tmp_path): + probe = _with_server(tmp_path) + monkeypatch.setattr(M, "validate_server", lambda *a, **k: None) + assert M.existing_gpu_install_offloads( + tmp_path, nvidia_host(), _gpu_plan("linux-cuda"), probe + ) is True + + +def test_existing_gpu_install_inconclusive_is_kept(monkeypatch, tmp_path): + probe = _with_server(tmp_path) + + def fake_validate(*a, **k): + raise PrebuiltFallback("llama-server exited during startup") + + monkeypatch.setattr(M, "validate_server", fake_validate) + assert M.existing_gpu_install_offloads( + tmp_path, nvidia_host(), _gpu_plan("linux-cuda"), probe + ) is True + + +def test_existing_gpu_install_no_binary_is_kept(tmp_path): + # No llama-server present -> let the normal flow reinstall, don't crash. + probe = tmp_path / "probe.gguf" + probe.write_bytes(b"GGUF") + assert M.existing_gpu_install_offloads( + tmp_path, nvidia_host(), _gpu_plan("linux-cuda"), probe + ) is True From 889e889d9cdaafa8ed0e14f5f318e4b155b43b95 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 1 Jun 2026 16:30:50 +0000 Subject: [PATCH 09/12] Studio: gate ROCm hosts out of CPU prebuilt and require offload signal in smoke-test - direct_upstream_release_plan: Linux x86_64 and arm64 CPU branches now also require not host.has_rocm, so a ROCm host whose HIP prebuilt is missing or rejected never silently takes a CPU upstream tarball as a 'success'. - setup.ps1: a Windows ROCm host with no usable HIP prebuilt and no HIP source build path is now marked LlamaCppDegraded so the CPU last-resort installs a clearly labelled CPU build instead of a 'built' install that runs on CPU. - validate_server gains require_gpu_signal; smoke_test_server_binary sets it for GPU install kinds so the '0 = offload confirmed' CLI contract treats a no-GPU-signal log as inconclusive (EXIT_ERROR) rather than a silent pass. - run_smoke_spoof + unit test cover the new no-signal-is-inconclusive contract. --- studio/install_llama_prebuilt.py | 39 ++++++++++++++++--- studio/setup.ps1 | 8 ++++ tests/studio/install/run_smoke_spoof.py | 2 +- .../test_validate_server_gpu_offload.py | 17 +++++++- 4 files changed, 58 insertions(+), 8 deletions(-) diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index f7d4b1fa21..6d2756b83e 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -1611,7 +1611,12 @@ def direct_upstream_release_plan( install_kind = "macos-x64", ) ) - elif host.is_linux and host.is_x86_64 and not host.has_usable_nvidia: + elif ( + host.is_linux + and host.is_x86_64 + and not host.has_usable_nvidia + and not host.has_rocm + ): asset_name = f"llama-{release_tag}-bin-ubuntu-x64.tar.gz" asset_url = assets.get(asset_name) if asset_url: @@ -1625,7 +1630,12 @@ def direct_upstream_release_plan( install_kind = "linux-cpu", ) ) - elif host.is_linux and host.is_arm64 and not host.has_usable_nvidia: + elif ( + host.is_linux + and host.is_arm64 + and not host.has_usable_nvidia + and not host.has_rocm + ): # Upstream ggml-org/llama.cpp ships llama-bNNNN-bin-ubuntu-arm64.tar.gz # (visible in the b9334 release manifest). Without this branch the # selector returned 0 attempts and the installer fell back to a @@ -5766,6 +5776,7 @@ def validate_server( *, runtime_line: str | None = None, install_kind: str | None = None, + require_gpu_signal: bool = False, ) -> None: last_failure: PrebuiltFallback | None = None for port_attempt in range(1, SERVER_PORT_BIND_ATTEMPTS + 1): @@ -5908,10 +5919,10 @@ def validate_server( # the next bundle / source build instead of stopping here. if _enable_gpu_layers: log_handle.flush() - if ( - server_log_shows_gpu_offload(read_full_log(log_path)) - is False - ): + offload = server_log_shows_gpu_offload( + read_full_log(log_path) + ) + if offload is False: raise GpuOffloadFailure( "llama-server served a completion but loaded the " "model entirely on CPU despite GPU offload being " @@ -5924,6 +5935,17 @@ def validate_server( "(unslothai/unsloth#5807, #5106):\n" + read_log_excerpt(log_path) ) + # No GPU signal: install validation stays conservative + # (never reject on no evidence), but --smoke-test sets + # require_gpu_signal so its "0 = offload confirmed" + # contract does not pass an unproven log. + if offload is None and require_gpu_signal: + raise PrebuiltFallback( + "llama-server served a completion but its startup " + "log carried no GPU-offload signal; smoke-test " + "result is inconclusive:\n" + + read_log_excerpt(log_path) + ) return finally: if process is not None and process.poll() is None: @@ -6925,6 +6947,9 @@ def smoke_test_server_binary( Path(install_dir).expanduser().resolve() if install_dir else server_path.parent ) resolved_kind = install_kind or resolve_smoke_test_install_kind(host) + # For a GPU kind, the CLI contract is "0 = offload confirmed", so an + # inconclusive (no-signal) log must not pass -- require a positive signal. + require_signal = resolved_kind in _GPU_INSTALL_KINDS if probe: probe_path = Path(probe).expanduser().resolve() if not probe_path.exists(): @@ -6935,6 +6960,7 @@ def smoke_test_server_binary( host, resolved_install_dir, install_kind = resolved_kind, + require_gpu_signal = require_signal, ) else: with tempfile.TemporaryDirectory(prefix = "unsloth-llama-smoke-") as tmp: @@ -6948,6 +6974,7 @@ def smoke_test_server_binary( host, resolved_install_dir, install_kind = resolved_kind, + require_gpu_signal = require_signal, ) return resolved_kind diff --git a/studio/setup.ps1 b/studio/setup.ps1 index da4729f135..bd13b1d561 100644 --- a/studio/setup.ps1 +++ b/studio/setup.ps1 @@ -2667,6 +2667,14 @@ if (-not $NeedLlamaSourceBuild) { substep "GGUF inference and export will not be available." "Yellow" substep "Install CMake from https://cmake.org/download/ and re-run setup." "Yellow" $script:LlamaCppDegraded = $true +} elseif ($HasROCm -and -not $HasNvidiaSmi) { + # Windows has no HIP source-build path, so a ROCm host whose HIP prebuilt was + # missing or rejected must not silently CPU-source-build. Mark degraded so + # the CPU-prebuilt last resort below installs a clearly labelled CPU build + # instead of a "built" install that runs on CPU (#5807). + Write-Host "" + step "llama.cpp" "no usable HIP prebuilt and no Windows HIP source build; using CPU prebuilt" "Yellow" + $script:LlamaCppDegraded = $true } else { # A source build is committed here. The CUDA toolkit is only needed now, so # resolve (and winget-install if needed) it lazily, failing fast if no diff --git a/tests/studio/install/run_smoke_spoof.py b/tests/studio/install/run_smoke_spoof.py index aa94492c27..b4aa5391f6 100644 --- a/tests/studio/install/run_smoke_spoof.py +++ b/tests/studio/install/run_smoke_spoof.py @@ -84,7 +84,7 @@ def main() -> int: ("cuda", GPU_KIND, 0, "GPU binary tagged GPU is accepted"), ("cuda_buffer", GPU_KIND, 0, "GPU buffer-format binary is accepted"), ("cpu", CPU_KIND, 0, "CPU binary tagged CPU is not gated"), - ("no_signal", GPU_KIND, 0, "no-signal log is not rejected"), + ("no_signal", GPU_KIND, 1, "no-signal GPU log is inconclusive (exit 1)"), ] for mode, kind, expected, label in cases: rc = run_smoke(wrapper, probe, kind, mode) diff --git a/tests/studio/install/test_validate_server_gpu_offload.py b/tests/studio/install/test_validate_server_gpu_offload.py index cc64d2560d..82c7746d5e 100644 --- a/tests/studio/install/test_validate_server_gpu_offload.py +++ b/tests/studio/install/test_validate_server_gpu_offload.py @@ -408,11 +408,26 @@ def test_cpu_kind_not_gpu_gated(patched_server, tmp_path): def test_gpu_intent_no_signal_accepted(patched_server, tmp_path): - # No buffer/device signal -> conservative: do not reject on no evidence. + # No buffer/device signal -> conservative: do not reject on no evidence + # (plain install validation; require_gpu_signal defaults False). patched_server(NO_SIGNAL_LOG) _run_validate(tmp_path, nvidia_host(), "linux-cuda") # no raise +def test_smoke_test_no_signal_gpu_is_inconclusive(patched_server, tmp_path): + # The smoke-test CLI sets require_gpu_signal, so a no-signal GPU log is + # inconclusive (PrebuiltFallback -> EXIT_ERROR), not a silent pass. + patched_server(NO_SIGNAL_LOG) + server = tmp_path / "llama-server" + server.write_text("#!/bin/sh\n") + probe = tmp_path / "probe.gguf" + probe.write_bytes(b"GGUF") + with pytest.raises(PrebuiltFallback): + smoke_test_server_binary( + str(server), nvidia_host(), install_dir = str(tmp_path), probe = str(probe) + ) + + def test_rocm_cpu_only_rejected(patched_server, tmp_path): patched_server(CPU_ONLY_DEVICE_INFO_LOG) with pytest.raises(PrebuiltFallback): From e9f52c21149eda7328cc49535053ebda502d0612 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Jun 2026 16:32:28 +0000 Subject: [PATCH 10/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../tests/test_llama_cpp_context_fit.py | 4 +- studio/install_llama_prebuilt.py | 7 +-- .../test_validate_server_gpu_offload.py | 49 +++++++++++++------ 3 files changed, 36 insertions(+), 24 deletions(-) diff --git a/studio/backend/tests/test_llama_cpp_context_fit.py b/studio/backend/tests/test_llama_cpp_context_fit.py index ed82d4fa96..e260e84f75 100644 --- a/studio/backend/tests/test_llama_cpp_context_fit.py +++ b/studio/backend/tests/test_llama_cpp_context_fit.py @@ -652,7 +652,5 @@ class TestClassifyGpuOffload: assert inst._classify_gpu_offload(True, [(0, 22805)]) is False def test_hip_model_buffer_is_gpu(self): - inst = self._backend( - ["load_tensors: HIP0 model buffer size = 21000.0 MiB"] - ) + inst = self._backend(["load_tensors: HIP0 model buffer size = 21000.0 MiB"]) assert inst._classify_gpu_offload(True, [(0, 22805)]) is True diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index 6d2756b83e..f32e814713 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -5919,9 +5919,7 @@ def validate_server( # the next bundle / source build instead of stopping here. if _enable_gpu_layers: log_handle.flush() - offload = server_log_shows_gpu_offload( - read_full_log(log_path) - ) + offload = server_log_shows_gpu_offload(read_full_log(log_path)) if offload is False: raise GpuOffloadFailure( "llama-server served a completion but loaded the " @@ -5943,8 +5941,7 @@ def validate_server( raise PrebuiltFallback( "llama-server served a completion but its startup " "log carried no GPU-offload signal; smoke-test " - "result is inconclusive:\n" - + read_log_excerpt(log_path) + "result is inconclusive:\n" + read_log_excerpt(log_path) ) return finally: diff --git a/tests/studio/install/test_validate_server_gpu_offload.py b/tests/studio/install/test_validate_server_gpu_offload.py index 82c7746d5e..6028b98558 100644 --- a/tests/studio/install/test_validate_server_gpu_offload.py +++ b/tests/studio/install/test_validate_server_gpu_offload.py @@ -534,7 +534,9 @@ def _gpu_plan(install_kind = "linux-cuda"): release_tag = "rel", attempts = [choice], approved_checksums = M.ApprovedReleaseChecksums( - repo = "unslothai/llama.cpp", release_tag = "rel", upstream_tag = "b9001", + repo = "unslothai/llama.cpp", + release_tag = "rel", + upstream_tag = "b9001", artifacts = {}, ), ) @@ -551,9 +553,12 @@ def _with_server(tmp_path): def test_existing_cpu_kind_install_is_kept(tmp_path): # A non-GPU existing install is never offload-gated. probe = _with_server(tmp_path) - assert M.existing_gpu_install_offloads( - tmp_path, nvidia_host(), _gpu_plan("linux-cpu"), probe - ) is True + assert ( + M.existing_gpu_install_offloads( + tmp_path, nvidia_host(), _gpu_plan("linux-cpu"), probe + ) + is True + ) def test_existing_gpu_install_cpu_only_triggers_reinstall(monkeypatch, tmp_path): @@ -563,17 +568,23 @@ def test_existing_gpu_install_cpu_only_triggers_reinstall(monkeypatch, tmp_path) raise M.GpuOffloadFailure("loaded the model entirely on CPU") monkeypatch.setattr(M, "validate_server", fake_validate) - assert M.existing_gpu_install_offloads( - tmp_path, nvidia_host(), _gpu_plan("linux-cuda"), probe - ) is False + assert ( + M.existing_gpu_install_offloads( + tmp_path, nvidia_host(), _gpu_plan("linux-cuda"), probe + ) + is False + ) def test_existing_gpu_install_offloading_is_kept(monkeypatch, tmp_path): probe = _with_server(tmp_path) monkeypatch.setattr(M, "validate_server", lambda *a, **k: None) - assert M.existing_gpu_install_offloads( - tmp_path, nvidia_host(), _gpu_plan("linux-cuda"), probe - ) is True + assert ( + M.existing_gpu_install_offloads( + tmp_path, nvidia_host(), _gpu_plan("linux-cuda"), probe + ) + is True + ) def test_existing_gpu_install_inconclusive_is_kept(monkeypatch, tmp_path): @@ -583,15 +594,21 @@ def test_existing_gpu_install_inconclusive_is_kept(monkeypatch, tmp_path): raise PrebuiltFallback("llama-server exited during startup") monkeypatch.setattr(M, "validate_server", fake_validate) - assert M.existing_gpu_install_offloads( - tmp_path, nvidia_host(), _gpu_plan("linux-cuda"), probe - ) is True + assert ( + M.existing_gpu_install_offloads( + tmp_path, nvidia_host(), _gpu_plan("linux-cuda"), probe + ) + is True + ) def test_existing_gpu_install_no_binary_is_kept(tmp_path): # No llama-server present -> let the normal flow reinstall, don't crash. probe = tmp_path / "probe.gguf" probe.write_bytes(b"GGUF") - assert M.existing_gpu_install_offloads( - tmp_path, nvidia_host(), _gpu_plan("linux-cuda"), probe - ) is True + assert ( + M.existing_gpu_install_offloads( + tmp_path, nvidia_host(), _gpu_plan("linux-cuda"), probe + ) + is True + ) From b990701a3ef0a500f30bfb6f1793e0d028d1a291 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 1 Jun 2026 17:22:45 +0000 Subject: [PATCH 11/12] Studio: exempt macOS Metal from the GPU-offload rejection GitHub macOS runners (and any headless or virtualized Mac) have no usable Metal, so the macos-arm64 prebuilt loads the validation model on CPU. The new offload check rejected it and forced a source build that also runs on CPU and then failed to launch, breaking 'Install + load (macos-*)' and 'Studio API & Auth Tests'. A CPU-only Metal load is an unfixable environment limitation, not a fixable binary fault like a missing cudart/cublas DLL or a PTX-only CUDA build, so a rebuild gives the same result. Split the GPU kinds into _GPU_INSTALL_KINDS (launched with --n-gpu-layers, so a broken libggml-metal.dylib still surfaces) and _GPU_OFFLOAD_REQUIRED_KINDS (CUDA/ROCm/HIP only, where a CPU-only load is rejected). validate_server, the existing-install re-validation, the fast-path reuse gate, the smoke-test require-signal, and the success message all key on the offload-required set, so macOS accepts its prebuilt while CUDA/ROCm rejection is unchanged. run_smoke_spoof drives the rejection contract with a CUDA kind on every runner (install_kind is explicit, so it is OS-independent) and adds a macOS-only case asserting a CPU-only Metal load is accepted. New unit tests cover the split. --- studio/install_llama_prebuilt.py | 76 +++++++++++++------ tests/studio/install/run_smoke_spoof.py | 32 +++++--- .../test_validate_server_gpu_offload.py | 34 +++++++++ 3 files changed, 110 insertions(+), 32 deletions(-) diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index f32e814713..ee438a362e 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -5664,13 +5664,27 @@ _OFFLOADING_COUNT_RE = re.compile( re.IGNORECASE, ) -# install_kind values that ship a real GPU backend and must offload to the GPU. -# A binary launched with --n-gpu-layers 1 under one of these is held to the -# GPU-offload check; CPU kinds (windows-cpu, linux-cpu, ...) are exempt. +# install_kind values that ship a real GPU backend and are launched with +# --n-gpu-layers 1 during validation, so a real Metal / CUDA / HIP backend is +# loaded and a broken GPU dylib/DLL surfaces. CPU kinds (windows-cpu, +# linux-cpu, ...) are exempt. _GPU_INSTALL_KINDS = frozenset( {"linux-cuda", "linux-rocm", "windows-cuda", "windows-hip", "macos-arm64"} ) +# Subset whose CPU-only load is a *fixable* fault -- a missing cudart64_* / +# cublas64_* DLL, a PTX-only build on an older driver, or a HIP backend that +# did not initialize -- that a different bundle or a source build can repair, +# so validation rejects it to advance the resolver (#5807, #5106). macOS Metal +# is excluded on purpose: on Apple Silicon the macos-arm64 prebuilt is already +# the correct artifact, and a CPU-only load means Metal is unavailable in this +# environment (a headless or virtualized host such as a CI runner), which no +# rebuild can fix. Rejecting it would force a source build that also runs on +# CPU and needlessly breaks the install. macOS is still launched with +# --n-gpu-layers (it is in _GPU_INSTALL_KINDS) so a broken libggml-metal.dylib +# still surfaces; it is only exempt from the CPU-only *rejection*. +_GPU_OFFLOAD_REQUIRED_KINDS = _GPU_INSTALL_KINDS - frozenset({"macos-arm64"}) + def server_log_shows_gpu_offload(log_text: str) -> bool | None: """Classify whether llama-server put the model on the GPU. @@ -5810,6 +5824,7 @@ def validate_server( _gpu_kinds = _GPU_INSTALL_KINDS if install_kind is not None: _enable_gpu_layers = install_kind in _gpu_kinds + _require_gpu_offload = install_kind in _GPU_OFFLOAD_REQUIRED_KINDS else: # Older call sites that don't pass install_kind: keep ROCm # hosts in the GPU-validation path so an AMD-only Linux host @@ -5820,6 +5835,10 @@ def validate_server( or host.has_rocm or (host.is_macos and host.is_arm64) ) + # macOS Metal is exercised but never *required* (a CPU-only Metal + # load is an environment limitation, not a fixable binary fault; + # see _GPU_OFFLOAD_REQUIRED_KINDS). + _require_gpu_offload = host.has_usable_nvidia or host.has_rocm if _enable_gpu_layers: command.extend(["--n-gpu-layers", "1"]) @@ -5909,15 +5928,18 @@ def validate_server( + ("\n" + response_body if response_body else "") ) if completion_succeeded: - # The server served a completion. When GPU offload was - # requested, confirm the model actually landed on the GPU -- - # a binary whose GPU backend failed to load (CPU-only build, - # unresolved cudart64_*/cublas64_* DLLs on Windows, or a - # PTX-only build on an older driver) still serves HTTP 200 - # from CPU, and accepting it ships a silently CPU-only - # install (#5807, #5106). Reject so the resolver advances to - # the next bundle / source build instead of stopping here. - if _enable_gpu_layers: + # The server served a completion. When GPU offload is + # *required* (CUDA / ROCm / HIP -- see + # _GPU_OFFLOAD_REQUIRED_KINDS), confirm the model actually + # landed on the GPU: a binary whose GPU backend failed to + # load (CPU-only build, unresolved cudart64_*/cublas64_* + # DLLs on Windows, or a PTX-only build on an older driver) + # still serves HTTP 200 from CPU, and accepting it ships a + # silently CPU-only install (#5807, #5106). Reject so the + # resolver advances to the next bundle / source build. macOS + # Metal is intentionally excluded: a CPU-only Metal load is + # an unfixable environment limitation, not a bad binary. + if _require_gpu_offload: log_handle.flush() offload = server_log_shows_gpu_offload(read_full_log(log_path)) if offload is False: @@ -6736,7 +6758,10 @@ def existing_gpu_install_offloads( "reinstall/restart keeps the silently CPU-only binary" path (#5807): without it, a metadata match short-circuits the new offload validation entirely.""" choice = plan.attempts[0] - if choice.install_kind not in _GPU_INSTALL_KINDS: + # Only re-validate kinds whose CPU-only load is a fixable fault. macOS Metal + # is exempt (a CPU-only load is unfixable, see _GPU_OFFLOAD_REQUIRED_KINDS), + # so it keeps the existing install without a per-restart smoke test. + if choice.install_kind not in _GPU_OFFLOAD_REQUIRED_KINDS: return True server_name = "llama-server.exe" if host.is_windows else "llama-server" try: @@ -6804,12 +6829,15 @@ def install_prebuilt( published_repo, published_release_tag, ) - # Non-GPU match: keep the fast path (no probe download). A GPU match - # must still be smoke-tested below, so it does not short-circuit here. + # Non-(offload-required) match: keep the fast path (no probe + # download). A CUDA/ROCm/HIP match must still be smoke-tested below, + # so it does not short-circuit here; macOS Metal (not offload + # required) keeps the fast path. if ( release_plans and existing_install_matches_plan(install_dir, host, release_plans[0]) - and release_plans[0].attempts[0].install_kind not in _GPU_INSTALL_KINDS + and release_plans[0].attempts[0].install_kind + not in _GPU_OFFLOAD_REQUIRED_KINDS ): current = release_plans[0] log( @@ -6905,9 +6933,9 @@ def install_prebuilt( def resolve_smoke_test_install_kind(host: HostInfo) -> str: """Best-effort install_kind for a post-build GPU smoke test, derived from - the host. setup.sh / setup.ps1 can override with --install-kind. GPU kinds - (in _GPU_INSTALL_KINDS) make the smoke test require real GPU offload; CPU - kinds skip that check.""" + the host. setup.sh / setup.ps1 can override with --install-kind. Offload- + required kinds (in _GPU_OFFLOAD_REQUIRED_KINDS) make the smoke test require + real GPU offload; CPU kinds and macOS Metal skip that check.""" if host.is_macos and host.is_arm64: return "macos-arm64" if host.has_rocm: @@ -6944,9 +6972,11 @@ def smoke_test_server_binary( Path(install_dir).expanduser().resolve() if install_dir else server_path.parent ) resolved_kind = install_kind or resolve_smoke_test_install_kind(host) - # For a GPU kind, the CLI contract is "0 = offload confirmed", so an - # inconclusive (no-signal) log must not pass -- require a positive signal. - require_signal = resolved_kind in _GPU_INSTALL_KINDS + # For an offload-required kind (CUDA/ROCm/HIP), the CLI contract is + # "0 = offload confirmed", so an inconclusive (no-signal) log must not pass + # -- require a positive signal. macOS Metal is exempt (see + # _GPU_OFFLOAD_REQUIRED_KINDS): it only needs to load and serve. + require_signal = resolved_kind in _GPU_OFFLOAD_REQUIRED_KINDS if probe: probe_path = Path(probe).expanduser().resolve() if not probe_path.exists(): @@ -7205,7 +7235,7 @@ def main() -> int: # build rather than downgrading it on uncertain evidence. log(f"smoke-test inconclusive: {exc}") return EXIT_ERROR - if resolved_kind in _GPU_INSTALL_KINDS: + if resolved_kind in _GPU_OFFLOAD_REQUIRED_KINDS: log( f"smoke-test passed: {args.smoke_test} " f"(install_kind={resolved_kind}) offloaded to the GPU" diff --git a/tests/studio/install/run_smoke_spoof.py b/tests/studio/install/run_smoke_spoof.py index b4aa5391f6..da056aaa90 100644 --- a/tests/studio/install/run_smoke_spoof.py +++ b/tests/studio/install/run_smoke_spoof.py @@ -27,10 +27,12 @@ INSTALLER = REPO / "studio" / "install_llama_prebuilt.py" FAKE = HERE / "fake_llama_server.py" IS_WIN = sys.platform == "win32" -GPU_KIND = { - "win32": "windows-cuda", - "darwin": "macos-arm64", -}.get(sys.platform, "linux-cuda") +# Offload-required GPU kind for the rejection contract. install_kind is passed +# explicitly, so the classifier / validate_server logic is exercised +# independent of the (GPU-less) runner OS. macOS Metal is intentionally NOT +# offload-required, so use a CUDA kind even on macOS to drive the CUDA/ROCm +# rejection path; Metal's accept-CPU behavior is covered by the macOS-only case. +GPU_REQ_KIND = "windows-cuda" if IS_WIN else "linux-cuda" CPU_KIND = { "win32": "windows-cpu", "darwin": "macos-cpu", @@ -79,13 +81,25 @@ def main() -> int: probe.write_bytes(b"GGUF\x00fake") cases = [ - ("cpu", GPU_KIND, 2, "CPU-only binary tagged GPU is rejected"), - ("offloaded_zero", GPU_KIND, 2, "offloaded 0/N tagged GPU is rejected"), - ("cuda", GPU_KIND, 0, "GPU binary tagged GPU is accepted"), - ("cuda_buffer", GPU_KIND, 0, "GPU buffer-format binary is accepted"), + ("cpu", GPU_REQ_KIND, 2, "CPU-only binary tagged GPU is rejected"), + ("offloaded_zero", GPU_REQ_KIND, 2, "offloaded 0/N tagged GPU is rejected"), + ("cuda", GPU_REQ_KIND, 0, "GPU binary tagged GPU is accepted"), + ("cuda_buffer", GPU_REQ_KIND, 0, "GPU buffer-format binary is accepted"), ("cpu", CPU_KIND, 0, "CPU binary tagged CPU is not gated"), - ("no_signal", GPU_KIND, 1, "no-signal GPU log is inconclusive (exit 1)"), + ("no_signal", GPU_REQ_KIND, 1, "no-signal GPU log is inconclusive (exit 1)"), ] + if sys.platform == "darwin": + # macOS Metal is not offload-required: a CPU-only Metal load is an + # unfixable environment limitation (headless / virtualized host), so + # the macos-arm64 prebuilt is accepted rather than rejected. + cases.append( + ( + "cpu", + "macos-arm64", + 0, + "macOS Metal CPU-only load is accepted (no rebuild remedy)", + ) + ) for mode, kind, expected, label in cases: rc = run_smoke(wrapper, probe, kind, mode) ok = rc == expected diff --git a/tests/studio/install/test_validate_server_gpu_offload.py b/tests/studio/install/test_validate_server_gpu_offload.py index 6028b98558..cac1b643bb 100644 --- a/tests/studio/install/test_validate_server_gpu_offload.py +++ b/tests/studio/install/test_validate_server_gpu_offload.py @@ -434,6 +434,40 @@ def test_rocm_cpu_only_rejected(patched_server, tmp_path): _run_validate(tmp_path, rocm_host(), "linux-rocm") +def test_macos_metal_not_in_offload_required_kinds(): + # macos-arm64 ships a GPU backend (Metal) and is launched with + # --n-gpu-layers, but a CPU-only Metal load is an unfixable environment + # limitation, so it must NOT be in the offload-required (reject) set. + assert "macos-arm64" in M._GPU_INSTALL_KINDS + assert "macos-arm64" not in M._GPU_OFFLOAD_REQUIRED_KINDS + + +def test_macos_metal_cpu_only_not_rejected(patched_server, tmp_path): + # The macOS regression in #5858 CI: GitHub macOS runners have no usable + # Metal, so the macos-arm64 prebuilt loads on CPU. It must be accepted, not + # rejected into a source build that also runs on CPU and breaks the install. + patched_server(CPU_ONLY_DEVICE_INFO_LOG) + _run_validate(tmp_path, macos_arm_host(), "macos-arm64") # no raise + + +def test_smoke_test_macos_metal_cpu_only_passes(patched_server, tmp_path): + # The smoke-test CLI must also accept a CPU-only macOS Metal load (exit 0), + # so setup.sh does not pointlessly retry a CPU source build on a Mac. + patched_server(CPU_ONLY_DEVICE_INFO_LOG) + server = tmp_path / "llama-server" + server.write_text("#!/bin/sh\n") + probe = tmp_path / "probe.gguf" + probe.write_bytes(b"GGUF") + # No raise -> the CLI maps this to EXIT_SUCCESS. + smoke_test_server_binary( + str(server), + macos_arm_host(), + install_dir = str(tmp_path), + probe = str(probe), + install_kind = "macos-arm64", + ) + + # -- 4. smoke_test_server_binary --------------------------------------------- From 2bbd1a679aefd4b7e39ba4301b85434633e4ab82 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:23:02 +0000 Subject: [PATCH 12/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/studio/install/run_smoke_spoof.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/studio/install/run_smoke_spoof.py b/tests/studio/install/run_smoke_spoof.py index da056aaa90..0a68bce0da 100644 --- a/tests/studio/install/run_smoke_spoof.py +++ b/tests/studio/install/run_smoke_spoof.py @@ -86,7 +86,12 @@ def main() -> int: ("cuda", GPU_REQ_KIND, 0, "GPU binary tagged GPU is accepted"), ("cuda_buffer", GPU_REQ_KIND, 0, "GPU buffer-format binary is accepted"), ("cpu", CPU_KIND, 0, "CPU binary tagged CPU is not gated"), - ("no_signal", GPU_REQ_KIND, 1, "no-signal GPU log is inconclusive (exit 1)"), + ( + "no_signal", + GPU_REQ_KIND, + 1, + "no-signal GPU log is inconclusive (exit 1)", + ), ] if sys.platform == "darwin": # macOS Metal is not offload-required: a CPU-only Metal load is an