diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 7bcf02dc35..9001396afe 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -59,6 +59,75 @@ from core.inference.tool_call_parser import ( logger = get_logger(__name__) +# ── GPU-offload log classifier (shared shape with install_llama_prebuilt) ── +# llama-server can serve HTTP 200 while running entirely on CPU; detect that so +# Studio can warn instead of silently running a "GPU" model on CPU (#5807 / +# #5106 / #5830). Robust across llama.cpp log formats (the "model buffer size" +# lines were dropped in recent builds): buffer-size markers, "offloaded N/M +# layers to GPU", and the "device_info:" enumeration. +_GPU_BUFFER_MARKERS = ("CUDA", "ROCm", "Metal", "Vulkan", "OpenCL", "SYCL") +_DEVICE_ROW_RE = re.compile( + r"-\s*(?P(?:CUDA|ROCm|ROCM|HIP|Metal|Vulkan|SYCL|OpenCL|MUSA|CANN|CPU)\w*)\s*:", + re.IGNORECASE, +) +_GPU_DEVICE_PREFIXES = ( + "cuda", + "rocm", + "hip", + "metal", + "vulkan", + "sycl", + "opencl", + "musa", + "cann", +) +_OFFLOADED_LAYERS_RE = re.compile( + r"offloaded\s+(\d+)\s*/\s*(\d+)\s+layers?\s+to\s+gpu", re.IGNORECASE +) + + +def classify_gpu_offload_lines(lines: list[str]) -> Optional[bool]: + """True if the model landed on a GPU, False if it stayed on CPU despite GPU + intent, None when the log has no usable signal. Priority: buffer-size lines, + then offloaded-layers count, then device_info enumeration.""" + saw_buffer_line = False + for line in lines: + if "buffer size" not in line: + continue + if any(marker in line for marker in _GPU_BUFFER_MARKERS): + return True + if "model buffer size" in line: + saw_buffer_line = True + + for line in lines: + match = _OFFLOADED_LAYERS_RE.search(line) + if match: + return int(match.group(1)) > 0 + low = line.lower() + if "offloading" in low and "to gpu" in low: + return True + + after_device_info = False + saw_device_row = False + for line in lines: + if "device_info:" in line: + after_device_info = True + continue + if not after_device_info: + continue + match = _DEVICE_ROW_RE.search(line) + if not match: + continue + saw_device_row = True + dev = match.group("dev").lower() + if any(dev.startswith(prefix) for prefix in _GPU_DEVICE_PREFIXES): + return True + + if saw_buffer_line or saw_device_row: + return False + return None + + # ── Pre-compiled patterns for plan-without-action re-prompt ── # Forward-looking intent signals that indicate the model is # describing what it *will* do rather than giving a final answer. @@ -3865,27 +3934,14 @@ class LlamaCppBackend: expected_gpu: bool, detected_gpus: list[tuple[int, int]], ) -> Optional[bool]: - """True if a GPU model buffer was allocated, False if only CPU - buffers landed despite GPU intent, None when there's no signal - (no GPU detected, no buffer-size lines, etc.).""" + """True if the model landed on a GPU, False if only CPU buffers landed + despite GPU intent, None when there's no signal (no GPU detected, no + usable log lines, etc.). Delegates to the shared classifier so it tracks + current llama.cpp logs (device_info / offloaded-layers), which dropped + the older ``model buffer size`` lines this used to key on.""" if not detected_gpus or not expected_gpu: return None - # llama-server logs one ``... model buffer size = N MiB`` line - # per backend buffer; CUDA0 / ROCm0 / Metal / Vulkan0 / - # OpenCL0 / SYCL0 are GPU, CPU / CPU_Mapped are not. - gpu_markers = ("CUDA", "ROCm", "Metal", "Vulkan", "OpenCL", "SYCL") - saw_buffer_line = False - saw_gpu_buffer = False - for line in self._stdout_lines: - if "model buffer size" not in line: - continue - saw_buffer_line = True - if any(marker in line for marker in gpu_markers): - saw_gpu_buffer = True - break - if not saw_buffer_line: - return None - return saw_gpu_buffer + return classify_gpu_offload_lines(self._stdout_lines) def unload_model(self) -> bool: """Terminate the llama-server subprocess and cancel any in-flight download.""" diff --git a/studio/backend/tests/test_llama_cpp_context_fit.py b/studio/backend/tests/test_llama_cpp_context_fit.py index 6fe5372147..065c9726e4 100644 --- a/studio/backend/tests/test_llama_cpp_context_fit.py +++ b/studio/backend/tests/test_llama_cpp_context_fit.py @@ -577,3 +577,45 @@ class TestClassifyGpuOffload: ] ) assert inst._classify_gpu_offload(True, [(0, 22805)]) is True + + def test_current_device_info_cuda_returns_true(self): + # Recent llama.cpp dropped the buffer lines and prints device_info. + inst = self._backend( + [ + "0.00 I device_info:", + "0.01 I - CUDA0 : NVIDIA GeForce RTX 5070 (12282 MiB free)", + "0.01 I - CPU : AMD Ryzen 7 9700X (32000 MiB free)", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 12282)]) is True + + def test_current_device_info_cpu_only_returns_false(self): + # The #5830 symptom: device_info shows only CPU, CUDA0 missing. + inst = self._backend( + [ + "0.00 I device_info:", + "0.00 I - CPU : AMD Ryzen 7 9700X (32000 MiB free)", + "0.00 I srv llama_server: model loaded", + ] + ) + assert inst._classify_gpu_offload(True, [(0, 12282)]) is False + + def test_offloaded_layers_count_decides(self): + assert ( + self._backend(["load_tensors: offloaded 33/33 layers to GPU"]) + ._classify_gpu_offload(True, [(0, 22805)]) + is True + ) + assert ( + self._backend(["load_tensors: offloaded 0/33 layers to GPU"]) + ._classify_gpu_offload(True, [(0, 22805)]) + is False + ) + + def test_system_info_only_returns_none(self): + # A compiled CUDA backend advertised in system_info is not proof of an + # available device; without device_info/buffer/offload signal -> None. + inst = self._backend( + ["system_info: n_threads = 8 | CUDA : ARCHS = 1200 | CPU : AVX2 = 1"] + ) + assert inst._classify_gpu_offload(True, [(0, 22805)]) is None diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index 62b7b2b290..151c459924 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -452,6 +452,14 @@ class PrebuiltFallback(RuntimeError): pass +class GpuOffloadFailure(PrebuiltFallback): + """The binary started and served a completion but loaded the model only on + CPU despite GPU offload being requested. A subclass of PrebuiltFallback so + the prebuilt resolver still advances to the next candidate, while the + --smoke-test CLI can tell this apart from an inconclusive start/serve + failure (definite CPU-only -> EXIT_FALLBACK; inconclusive -> EXIT_ERROR).""" + + class BusyInstallConflict(RuntimeError): pass @@ -1437,7 +1445,14 @@ def direct_linux_release_plan( ) if lemonade_choice is not None: attempts.append(lemonade_choice) - else: + elif not host.has_usable_nvidia: + # CPU-only host (or --cpu-fallback, which zeroes the GPU flags): ship the + # CPU bundle. Do NOT append it for a usable-NVIDIA host: if every CUDA + # bundle fails the GPU-offload check, let install raise PrebuiltFallback + # so setup.sh builds llama.cpp from source for the native arch (which + # offloads) instead of silently shipping a CPU-only "GPU" install + # (#5807). The CPU prebuilt stays the last resort after the source build, + # reached via setup.sh's --cpu-fallback path. cpu_choice = published_asset_choice_for_kind(bundle, "linux-cpu") if cpu_choice is not None: attempts.append(cpu_choice) @@ -1530,19 +1545,26 @@ def direct_upstream_release_plan( install_kind = "windows-hip", ) ) - cpu_asset = f"llama-{release_tag}-bin-win-cpu-x64.zip" - cpu_url = assets.get(cpu_asset) - if cpu_url: - attempts.append( - AssetChoice( - repo = repo, - tag = release_tag, - name = cpu_asset, - url = cpu_url, - source_label = "upstream", - install_kind = "windows-cpu", + # Append the CPU bundle only for a host with no usable GPU (or + # --cpu-fallback, which zeroes the GPU flags). On a GPU host, omitting it + # lets install raise PrebuiltFallback when every GPU bundle fails the + # offload check, so setup.ps1 builds from source / falls back to the CPU + # prebuilt as a labelled last resort instead of silently shipping a + # CPU-only "GPU" install (#5807). + if not host.has_usable_nvidia and not host.has_rocm: + cpu_asset = f"llama-{release_tag}-bin-win-cpu-x64.zip" + cpu_url = assets.get(cpu_asset) + if cpu_url: + attempts.append( + AssetChoice( + repo = repo, + tag = release_tag, + name = cpu_asset, + url = cpu_url, + source_label = "upstream", + install_kind = "windows-cpu", + ) ) - ) elif host.is_windows and host.is_arm64: # Upstream ggml-org/llama.cpp ships llama-bNNNN-bin-win-cpu-arm64.zip # (visible in the b9334 release manifest). Without this branch the @@ -5577,6 +5599,128 @@ def validate_quantize( ) +# GPU backend markers that appear in llama-server's "... model buffer size ..." +# load lines (older llama.cpp) and in its "device_info:" enumeration (current +# llama.cpp). CUDA0 / ROCm0 / Metal / Vulkan0 / OpenCL0 / SYCL0 / HIP0 / MUSA0 +# / CANN0 are GPU; CPU / CPU_Mapped are not. Kept broad so a single helper +# covers every backend Studio ships. +_GPU_MODEL_BUFFER_MARKERS = ("CUDA", "ROCm", "Metal", "Vulkan", "OpenCL", "SYCL") + +# A device_info row looks like " I - CUDA0 : NVIDIA B200 (...)" or +# " I - CPU : ...". Matched on the "- :" shape (the leading +# "- " distinguishes it from the system_info line, which prints +# "| CUDA : ARCHS = ..." for a *compiled* backend even when the CUDA runtime +# failed to initialize at load time). Scanned only after a "device_info:" line. +_DEVICE_ROW_RE = re.compile( + r"-\s*(?P(?:CUDA|ROCm|ROCM|HIP|Metal|Vulkan|SYCL|OpenCL|MUSA|CANN|CPU)\w*)\s*:", + re.IGNORECASE, +) +# Keep in sync with _GPU_MODEL_BUFFER_MARKERS: every GPU backend matched by +# _DEVICE_ROW_RE must have a prefix here, or its device row counts as CPU-only +# and a working GPU build is wrongly rejected (OpenCL was the gap). +_GPU_DEVICE_PREFIXES = ( + "cuda", + "rocm", + "hip", + "metal", + "vulkan", + "sycl", + "opencl", + "musa", + "cann", +) + +# "load_tensors: offloaded 33/33 layers to GPU" (or "offloading ... to GPU"). +_OFFLOADED_LAYERS_RE = re.compile( + r"offloaded\s+(\d+)\s*/\s*(\d+)\s+layers?\s+to\s+gpu", re.IGNORECASE +) + +# install_kind values that ship a real GPU backend and must offload to the GPU. +# A binary launched with --n-gpu-layers 1 under one of these is held to the +# GPU-offload check; CPU kinds (windows-cpu, linux-cpu, ...) are exempt. +_GPU_INSTALL_KINDS = frozenset( + {"linux-cuda", "linux-rocm", "windows-cuda", "windows-hip", "macos-arm64"} +) + + +def server_log_shows_gpu_offload(log_text: str) -> bool | None: + """Classify whether llama-server put the model on the GPU. + + Robust across llama.cpp log formats (the "model buffer size" lines were + dropped in recent builds), in priority order: + + 1. `` model buffer size = N`` lines -- GPU marker => True + (older llama.cpp). + 2. ``offloaded N/M layers to GPU`` -- N>0 => True, 0/M => False. + 3. ``device_info:`` enumeration -- a GPU device row (``- CUDA0 :`` etc.) + => True; only a ``- CPU :`` row => False. This is the version-stable + signal and matches how the CPU-only-binary bug is diagnosed in the + field (``device_info`` shows only CPU, CUDA0 missing -- #5830). + + Returns True (GPU offload confirmed), False (started but stayed on CPU + despite GPU intent -- the silent fallback in #5807 / #5106), or None when + the log carries no usable signal so callers never reject on no evidence. + """ + lines = log_text.splitlines() + + # Signal 1: per-backend buffer-size lines. A GPU marker on ANY "buffer + # size" line (model / KV / compute) means the GPU holds part of the model + # -- accept. Only the model-buffer location decides CPU-only (KV/compute + # CPU buffers exist even on GPU runs), so the False determination keys on + # "model buffer size" alone. + saw_buffer_line = False + for line in lines: + if "buffer size" not in line: + continue + if any(marker in line for marker in _GPU_MODEL_BUFFER_MARKERS): + return True + if "model buffer size" in line: + saw_buffer_line = True + + # Signal 2: explicit offloaded-layers count. + for line in lines: + match = _OFFLOADED_LAYERS_RE.search(line) + if match: + return int(match.group(1)) > 0 + low = line.lower() + if "offloading" in low and "to gpu" in low: + return True + + # Signal 3: device_info enumeration. Only trust device rows once the + # "device_info:" header has appeared, so the compiled-backend system_info + # line ("| CUDA : ARCHS = ...") is never mistaken for an available device. + after_device_info = False + saw_device_row = False + for line in lines: + if "device_info:" in line: + after_device_info = True + continue + if not after_device_info: + continue + match = _DEVICE_ROW_RE.search(line) + if not match: + continue + saw_device_row = True + dev = match.group("dev").lower() + if any(dev.startswith(prefix) for prefix in _GPU_DEVICE_PREFIXES): + return True + + if saw_buffer_line or saw_device_row: + # We had a concrete signal, but every buffer/device was CPU. + return False + return None + + +def read_full_log(log_path: Path, *, max_chars: int = 1_000_000) -> str: + """Read a server log from the start (model-load buffer lines appear early, + so unlike read_log_excerpt this keeps the head rather than the tail).""" + try: + content = log_path.read_text(encoding = "utf-8", errors = "replace") + except FileNotFoundError: + return "" + return content[:max_chars] + + def validate_server( server_path: Path, probe_path: Path, @@ -5615,13 +5759,7 @@ def validate_server( # validation. Use the resolved install_kind as the source of # truth and fall back to host detection when the caller did not # pass one (keeps backwards compatibility with older call sites). - _gpu_kinds = { - "linux-cuda", - "linux-rocm", - "windows-cuda", - "windows-hip", - "macos-arm64", - } + _gpu_kinds = _GPU_INSTALL_KINDS if install_kind is not None: _enable_gpu_layers = install_kind in _gpu_kinds else: @@ -5657,6 +5795,7 @@ def validate_server( startup_started = time.time() response_body = "" last_error: Exception | None = None + completion_succeeded = False while time.time() < deadline: if process.poll() is not None: process.wait(timeout = 5) @@ -5697,7 +5836,11 @@ def validate_server( status_code = response.status response_body = response.read().decode("utf-8", "replace") if status_code == 200: - return + # Served a completion. Confirm real GPU offload + # below (outside this try, so GpuOffloadFailure is + # not swallowed by the except clauses). + completion_succeeded = True + break last_error = RuntimeError( f"unexpected HTTP status {status_code}" ) @@ -5717,6 +5860,34 @@ def validate_server( + output + ("\n" + response_body if response_body else "") ) + if completion_succeeded: + # The server served a completion. When GPU offload was + # requested, confirm the model actually landed on the GPU -- + # a binary whose GPU backend failed to load (CPU-only build, + # unresolved cudart64_*/cublas64_* DLLs on Windows, or a + # PTX-only build on an older driver) still serves HTTP 200 + # from CPU, and accepting it ships a silently CPU-only + # install (#5807, #5106). Reject so the resolver advances to + # the next bundle / source build instead of stopping here. + if _enable_gpu_layers: + log_handle.flush() + if ( + server_log_shows_gpu_offload(read_full_log(log_path)) + is False + ): + raise GpuOffloadFailure( + "llama-server served a completion but loaded the " + "model entirely on CPU despite GPU offload being " + f"requested (install_kind={install_kind}). The " + "binary's GPU backend did not initialize -- on " + "Windows this is usually unresolved " + "cudart64_*.dll / cublas64_*.dll, or a build that " + "does not match the driver (PTX-only). Rejecting " + "so a GPU-capable bundle is selected " + "(unslothai/unsloth#5807, #5106):\n" + + read_log_excerpt(log_path) + ) + return finally: if process is not None and process.poll() is None: process.terminate() @@ -6616,6 +6787,74 @@ def install_prebuilt( raise SystemExit(EXIT_FALLBACK) from exc +def resolve_smoke_test_install_kind(host: HostInfo) -> str: + """Best-effort install_kind for a post-build GPU smoke test, derived from + the host. setup.sh / setup.ps1 can override with --install-kind. GPU kinds + (in _GPU_INSTALL_KINDS) make the smoke test require real GPU offload; CPU + kinds skip that check.""" + if host.is_macos and host.is_arm64: + return "macos-arm64" + if host.has_rocm: + return "windows-hip" if host.is_windows else "linux-rocm" + if host.has_usable_nvidia: + return "windows-cuda" if host.is_windows else "linux-cuda" + if host.is_windows: + return "windows-cpu" + if host.is_macos: + return "macos-cpu" + return "linux-cpu" + + +def smoke_test_server_binary( + server_binary: str, + host: HostInfo, + *, + install_dir: str | None = None, + probe: str | None = None, + install_kind: str | None = None, +) -> str: + """Run validate_server against an already-built llama-server binary. + + Raises PrebuiltFallback when the binary fails to start, fails the completion + probe, or (on a GPU host) loads the model only on CPU. Returns the resolved + install_kind on success. + """ + server_path = Path(server_binary).expanduser().resolve() + if not server_path.exists(): + raise PrebuiltFallback( + f"smoke-test: llama-server binary not found at {server_path}" + ) + resolved_install_dir = ( + Path(install_dir).expanduser().resolve() if install_dir else server_path.parent + ) + resolved_kind = install_kind or resolve_smoke_test_install_kind(host) + if probe: + probe_path = Path(probe).expanduser().resolve() + if not probe_path.exists(): + raise PrebuiltFallback(f"smoke-test: probe model not found at {probe_path}") + validate_server( + server_path, + probe_path, + host, + resolved_install_dir, + install_kind = resolved_kind, + ) + else: + with tempfile.TemporaryDirectory(prefix = "unsloth-llama-smoke-") as tmp: + probe_path = Path(tmp) / "stories260K.gguf" + download_validation_model( + probe_path, validation_model_cache_path(resolved_install_dir) + ) + validate_server( + server_path, + probe_path, + host, + resolved_install_dir, + install_kind = resolved_kind, + ) + return resolved_kind + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description = "Install and validate a prebuilt llama.cpp bundle for Unsloth Studio." @@ -6647,6 +6886,33 @@ def parse_args() -> argparse.Namespace: action = "store_true", help = "Use the simplified platform-specific prebuilt selection policy.", ) + parser.add_argument( + "--smoke-test", + metavar = "LLAMA_SERVER", + help = ( + "Validate an already-built llama-server binary instead of " + "installing. Launches it against a tiny probe model and, on a GPU " + "host, verifies the model actually offloaded to the GPU. Exits 0 on " + "success, 2 (EXIT_FALLBACK) when the binary only ran on CPU, 1 " + "(EXIT_ERROR) when inconclusive -- used by setup.sh / setup.ps1 to " + "validate source builds." + ), + ) + parser.add_argument( + "--probe", + help = ( + "Optional probe GGUF for --smoke-test. Defaults to the bundled " + "validation model (downloaded if absent)." + ), + ) + parser.add_argument( + "--install-kind", + help = ( + "Override the install_kind used by --smoke-test (e.g. linux-cuda, " + "windows-cuda, linux-rocm, windows-hip, macos-arm64). Auto-derived " + "from the host when omitted." + ), + ) parser.add_argument( "--has-rocm", action = "store_true", @@ -6788,6 +7054,39 @@ def main() -> int: ) return EXIT_SUCCESS + if args.smoke_test is not None: + host = detect_host() + try: + resolved_kind = smoke_test_server_binary( + args.smoke_test, + host, + install_dir = args.install_dir, + probe = args.probe, + install_kind = args.install_kind, + ) + except GpuOffloadFailure as exc: + # Definitive: GPU was requested but the model loaded on CPU. Exit + # EXIT_FALLBACK so the caller (setup.sh / setup.ps1) rebuilds CPU. + log(f"smoke-test failed (CPU-only): {exc}") + return EXIT_FALLBACK + except PrebuiltFallback as exc: + # Inconclusive: the binary could not be exercised (failed to start, + # missing probe, ...). Exit EXIT_ERROR so the caller keeps the GPU + # build rather than downgrading it on uncertain evidence. + log(f"smoke-test inconclusive: {exc}") + return EXIT_ERROR + if resolved_kind in _GPU_INSTALL_KINDS: + log( + f"smoke-test passed: {args.smoke_test} " + f"(install_kind={resolved_kind}) offloaded to the GPU" + ) + else: + log( + f"smoke-test passed: {args.smoke_test} " + f"(install_kind={resolved_kind})" + ) + return EXIT_SUCCESS + if not args.install_dir: raise SystemExit( "install_llama_prebuilt.py: --install-dir is required unless --resolve-llama-tag, --resolve-install-tag, or --resolve-source-build is used" diff --git a/tests/studio/install/test_selection_logic.py b/tests/studio/install/test_selection_logic.py index 2a94039018..d02e94e58e 100644 --- a/tests/studio/install/test_selection_logic.py +++ b/tests/studio/install/test_selection_logic.py @@ -78,6 +78,7 @@ windows_cuda_upstream_asset_names = ( ) env_int = INSTALL_LLAMA_PREBUILT.env_int direct_upstream_release_plan = INSTALL_LLAMA_PREBUILT.direct_upstream_release_plan +direct_linux_release_plan = INSTALL_LLAMA_PREBUILT.direct_linux_release_plan _pinned_windows_cuda_fallback = INSTALL_LLAMA_PREBUILT._pinned_windows_cuda_fallback CudaRuntimePreference = INSTALL_LLAMA_PREBUILT.CudaRuntimePreference published_windows_cuda_attempts = INSTALL_LLAMA_PREBUILT.published_windows_cuda_attempts @@ -2297,11 +2298,14 @@ class TestDirectUpstreamBlackwellPin: self._release(), host, UPSTREAM_REPO, "latest" ) order = [(a.tag, a.runtime_line or a.install_kind) for a in plan.attempts] + # No windows-cpu tail: a GPU host must fall through to the source build + # when every CUDA bundle fails the offload check, not silently install + # the CPU bundle (#5807). The pin + cuda12 are the only attempts. assert order == [ ("b9360", "cuda13"), (self.TAG, "cuda12"), - (self.TAG, "windows-cpu"), ] + assert all(a.install_kind != "windows-cpu" for a in plan.attempts) assert plan.attempts[0].name == "llama-b9360-bin-win-cuda-13.1-x64.zip" # Direct/upstream path stays unverified-by-manifest (no approved hashes). assert plan.approved_checksums.artifacts == {} @@ -2324,6 +2328,120 @@ class TestDirectUpstreamBlackwellPin: assert plan.attempts[0].name == f"llama-{self.TAG}-bin-win-cuda-13.3-x64.zip" +# =========================================================================== +# N.1c2. simple-policy plans: a GPU host never gets a CPU bundle appended +# (#5807). All GPU bundles failing must raise PrebuiltFallback so setup builds +# from source for the native arch, not silently install a CPU-only "GPU" build. +# =========================================================================== + + +class TestGpuHostNoSilentCpuFallback: + LTAG = "bTEST" + + def _linux_release(self, *targets): + names = [f"app-{self.LTAG}-linux-x64-{t}.tar.gz" for t in targets] + return { + "tag_name": self.LTAG, + "assets": [ + {"name": n, "browser_download_url": "https://x/" + n} for n in names + ], + } + + def _win_release(self): + names = [ + "llama-b9999-bin-win-cuda-13.3-x64.zip", + "cudart-llama-bin-win-cuda-13.3-x64.zip", + "llama-b9999-bin-win-cpu-x64.zip", + ] + return { + "tag_name": "b9999", + "assets": [ + {"name": n, "browser_download_url": "https://x/" + n} for n in names + ], + } + + def _no_torch(self, monkeypatch): + monkeypatch.setattr( + INSTALL_LLAMA_PREBUILT, + "detect_torch_cuda_runtime_preference", + lambda host: CudaRuntimePreference(runtime_line = None, selection_log = []), + ) + + # -- Linux -------------------------------------------------------------- + + def test_linux_nvidia_host_has_no_cpu_attempt(self, monkeypatch): + mock_linux_runtime(monkeypatch, ["cuda13", "cuda12"]) + self._no_torch(monkeypatch) + host = make_host(compute_caps = ["120"], driver_cuda_version = (13, 0)) + plan = direct_linux_release_plan( + self._linux_release("cuda13-newer", "cpu"), host, "unslothai/llama.cpp", "latest" + ) + kinds = [a.install_kind for a in plan.attempts] + assert "linux-cuda" in kinds + assert "linux-cpu" not in kinds + + def test_linux_nvidia_unmatched_arch_raises_no_cpu(self, monkeypatch): + # GPU present but no CUDA bundle covers the arch: must raise (-> source + # build), not fall to the CPU bundle. + mock_linux_runtime(monkeypatch, ["cuda13", "cuda12"]) + self._no_torch(monkeypatch) + host = make_host(compute_caps = ["50"], driver_cuda_version = (13, 0)) + with pytest.raises(PrebuiltFallback): + direct_linux_release_plan( + self._linux_release("cuda13-newer", "cpu"), + host, + "unslothai/llama.cpp", + "latest", + ) + + def test_linux_cpu_host_keeps_cpu_attempt(self, monkeypatch): + mock_linux_runtime(monkeypatch, []) + host = make_host( + has_usable_nvidia = False, + has_physical_nvidia = False, + nvidia_smi = None, + compute_caps = [], + ) + plan = direct_linux_release_plan( + self._linux_release("cuda13-newer", "cpu"), host, "unslothai/llama.cpp", "latest" + ) + assert [a.install_kind for a in plan.attempts] == ["linux-cpu"] + + # -- Windows ------------------------------------------------------------ + + def test_windows_nvidia_host_has_no_cpu_attempt(self, monkeypatch): + mock_windows_runtime(monkeypatch, ["cuda13"]) + self._no_torch(monkeypatch) + host = make_host( + system = "Windows", + machine = "AMD64", + driver_cuda_version = (13, 3), + compute_caps = ["120"], + ) + plan = direct_upstream_release_plan( + self._win_release(), host, UPSTREAM_REPO, "latest" + ) + kinds = [a.install_kind for a in plan.attempts] + assert "windows-cuda" in kinds + assert "windows-cpu" not in kinds + + def test_windows_cpu_host_keeps_cpu_attempt(self, monkeypatch): + mock_windows_runtime(monkeypatch, []) + host = make_host( + system = "Windows", + machine = "AMD64", + has_usable_nvidia = False, + has_physical_nvidia = False, + nvidia_smi = None, + driver_cuda_version = None, + compute_caps = [], + ) + plan = direct_upstream_release_plan( + self._win_release(), host, UPSTREAM_REPO, "latest" + ) + assert [a.install_kind for a in plan.attempts] == ["windows-cpu"] + + # =========================================================================== # N.1d. published_windows_cuda_attempts -- version-dynamic ordering seed # =========================================================================== diff --git a/tests/studio/install/test_validate_server_gpu_offload.py b/tests/studio/install/test_validate_server_gpu_offload.py new file mode 100644 index 0000000000..1182c7117b --- /dev/null +++ b/tests/studio/install/test_validate_server_gpu_offload.py @@ -0,0 +1,450 @@ +"""Tests for the GPU-offload validation added to install_llama_prebuilt.py. + +Covers issue unslothai/unsloth#5807 (duplicates #5830 / #5106 / #5827): a +llama-server whose GPU backend fails to initialize still serves HTTP 200 from +CPU, so the old validate_server accepted it and Studio shipped a silently +CPU-only install. The hardened validate_server rejects such a binary when GPU +offload was requested, so the resolver / source build falls through to a +GPU-capable bundle instead of a silently CPU-only one. + +Stdlib-only; the installer module is spec-loaded by absolute path (no +PYTHONPATH / heavy deps), matching test_install_llama_prebuilt_logic.py. +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + + +PACKAGE_ROOT = Path(__file__).resolve().parents[3] +MODULE_PATH = PACKAGE_ROOT / "studio" / "install_llama_prebuilt.py" +SPEC = importlib.util.spec_from_file_location( + "studio_install_llama_prebuilt", MODULE_PATH +) +assert SPEC is not None and SPEC.loader is not None +M = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = M +SPEC.loader.exec_module(M) + +server_log_shows_gpu_offload = M.server_log_shows_gpu_offload +resolve_smoke_test_install_kind = M.resolve_smoke_test_install_kind +smoke_test_server_binary = M.smoke_test_server_binary +validate_server = M.validate_server +PrebuiltFallback = M.PrebuiltFallback +HostInfo = M.HostInfo + + +# -- HostInfo factories ------------------------------------------------------ + + +def nvidia_host(**overrides) -> HostInfo: + defaults = dict( + system = "Linux", + machine = "x86_64", + is_windows = False, + is_linux = True, + is_macos = False, + is_x86_64 = True, + is_arm64 = False, + nvidia_smi = "/usr/bin/nvidia-smi", + driver_cuda_version = (13, 0), + compute_caps = ["120"], + visible_cuda_devices = None, + has_physical_nvidia = True, + has_usable_nvidia = True, + has_rocm = False, + ) + defaults.update(overrides) + return HostInfo(**defaults) + + +def windows_nvidia_host(**overrides) -> HostInfo: + return nvidia_host( + system = "Windows", + is_windows = True, + is_linux = False, + nvidia_smi = "nvidia-smi", + **overrides, + ) + + +def rocm_host(**overrides) -> HostInfo: + return nvidia_host( + nvidia_smi = None, + driver_cuda_version = None, + compute_caps = [], + has_physical_nvidia = False, + has_usable_nvidia = False, + has_rocm = True, + **overrides, + ) + + +def macos_arm_host(**overrides) -> HostInfo: + defaults = dict( + system = "Darwin", + machine = "arm64", + is_windows = False, + is_linux = False, + is_macos = True, + is_x86_64 = False, + is_arm64 = True, + nvidia_smi = None, + driver_cuda_version = None, + compute_caps = [], + visible_cuda_devices = None, + has_physical_nvidia = False, + has_usable_nvidia = False, + has_rocm = False, + ) + defaults.update(overrides) + return HostInfo(**defaults) + + +def cpu_host(**overrides) -> HostInfo: + return nvidia_host( + nvidia_smi = None, + driver_cuda_version = None, + compute_caps = [], + has_physical_nvidia = False, + has_usable_nvidia = False, + has_rocm = False, + **overrides, + ) + + +# -- Canned llama-server logs ------------------------------------------------ +# Current llama.cpp (>= mid-2026) dropped the "model buffer size" lines and +# enumerates a "device_info:" block instead; older builds still print the +# buffer lines. Both formats must classify correctly. + +CUDA_DEVICE_INFO_LOG = ( + "0.00.667 I log_info: verbosity = 3\n" + "0.00.667 I device_info:\n" + "0.01.101 I - CUDA0 : NVIDIA GeForce RTX 5070 (12282 MiB, 11000 MiB free)\n" + "0.01.101 I - CPU : AMD Ryzen 7 9700X (32000 MiB free)\n" + "0.01.101 I system_info: n_threads = 8 | CUDA : ARCHS = 1200 | CPU : AVX2 = 1\n" + "0.01.174 I srv llama_server: model loaded\n" +) + +CPU_ONLY_DEVICE_INFO_LOG = ( + "0.00.003 I log_info: verbosity = 3\n" + "0.00.003 I device_info:\n" + "0.00.003 I - CPU : AMD Ryzen 7 9700X (32000 MiB free)\n" + "0.00.003 I system_info: n_threads = 8 | CPU : AVX2 = 1\n" + "0.00.174 I srv llama_server: model loaded\n" +) + +CUDA_BUFFER_LOG = ( + "load_tensors: offloaded 33/33 layers to GPU\n" + "load_tensors: CUDA0 model buffer size = 21000.0 MiB\n" + "load_tensors: CPU_Mapped model buffer size = 0.6 MiB\n" +) + +CPU_BUFFER_LOG = ( + "load_tensors: CPU_Mapped model buffer size = 21000.0 MiB\n" + "load_tensors: CPU model buffer size = 0.6 MiB\n" +) + +ROCM_BUFFER_LOG = "load_tensors: ROCm0 model buffer size = 21000.0 MiB\n" +METAL_BUFFER_LOG = "load_tensors: Metal model buffer size = 8000.0 MiB\n" +VULKAN_DEVICE_INFO_LOG = ( + "device_info:\n - Vulkan0 : AMD Radeon (16000 MiB free)\n - CPU : x86 (free)\n" +) +OPENCL_DEVICE_INFO_LOG = ( + "device_info:\n - OpenCL0 : Adreno (4000 MiB free)\n - CPU : arm (free)\n" +) +OFFLOADED_ZERO_LOG = "load_tensors: offloaded 0/33 layers to GPU\n" +NO_SIGNAL_LOG = "INFO [main] starting server\nload_tensors: file format = GGUF V3\n" +# system_info advertises a *compiled* backend; not an available device. +SYSTEM_INFO_ONLY_LOG = ( + "system_info: n_threads = 8 | CUDA : ARCHS = 1200 | CPU : AVX2 = 1\n" +) + + +# -- 1. Pure classifier ------------------------------------------------------ + + +@pytest.mark.parametrize( + "log_text,expected", + [ + (CUDA_DEVICE_INFO_LOG, True), + (CPU_ONLY_DEVICE_INFO_LOG, False), + (CUDA_BUFFER_LOG, True), + (CPU_BUFFER_LOG, False), + (ROCM_BUFFER_LOG, True), + (METAL_BUFFER_LOG, True), + (VULKAN_DEVICE_INFO_LOG, True), + (OPENCL_DEVICE_INFO_LOG, True), + (OFFLOADED_ZERO_LOG, False), + (NO_SIGNAL_LOG, None), + ("", None), + (SYSTEM_INFO_ONLY_LOG, None), + ], +) +def test_server_log_shows_gpu_offload(log_text, expected): + assert server_log_shows_gpu_offload(log_text) is expected + + +def test_opencl_device_in_prefixes(): + # Regression: OpenCL was matched by the device-row regex but missing from + # the GPU prefix list, so an OpenCL-only device_info wrongly classified as + # CPU-only and a working OpenCL build was rejected. + assert "opencl" in M._GPU_DEVICE_PREFIXES + assert server_log_shows_gpu_offload(OPENCL_DEVICE_INFO_LOG) is True + + +def test_system_info_not_mistaken_for_device(): + # A CUDA-compiled binary whose runtime failed still prints the system_info + # CUDA backend, but enumerates only CPU under device_info -> must be False. + log = ( + "device_info:\n - CPU : x86 (free)\n" + "system_info: | CUDA : ARCHS = 1200 | CPU : AVX2 = 1\n" + ) + assert server_log_shows_gpu_offload(log) is False + + +def test_signal2_offloaded_zero_beats_device_info(): + # offloaded 0/33 (signal 2) is a definitive "nothing on GPU" and takes + # priority over a device_info CUDA0 row -> False. + log = ( + "load_tensors: offloaded 0/33 layers to GPU\n" + "device_info:\n - CUDA0 : NVIDIA (free)\n - CPU : x (free)\n" + ) + assert server_log_shows_gpu_offload(log) is False + + +def test_gpu_buffer_size_without_model_word(): + # A GPU-marked buffer line that omits the word "model" (e.g. KV/compute or + # a future format) must still count as GPU offload, even when a CPU + # "model buffer size" line is present (broadened signal 1). + log = ( + "load_tensors: CUDA0 KV buffer size = 100.0 MiB\n" + "load_tensors: CPU_Mapped model buffer size = 0.6 MiB\n" + ) + assert server_log_shows_gpu_offload(log) is True + + +def test_device_row_case_insensitive(): + log = "device_info:\n - cuda0 : some gpu (free)\n - CPU : x (free)\n" + assert server_log_shows_gpu_offload(log) is True + + +def test_crlf_log_parses_identically(): + # Windows logs use CRLF; classification must not change. + assert server_log_shows_gpu_offload(CUDA_DEVICE_INFO_LOG.replace("\n", "\r\n")) is True + assert ( + server_log_shows_gpu_offload(CPU_ONLY_DEVICE_INFO_LOG.replace("\n", "\r\n")) + is False + ) + + +# -- 2. resolve_smoke_test_install_kind -------------------------------------- + + +@pytest.mark.parametrize( + "host,expected", + [ + (nvidia_host(), "linux-cuda"), + (windows_nvidia_host(), "windows-cuda"), + (rocm_host(), "linux-rocm"), + (rocm_host(system = "Windows", is_windows = True, is_linux = False), "windows-hip"), + (macos_arm_host(), "macos-arm64"), + (cpu_host(), "linux-cpu"), + (cpu_host(system = "Windows", is_windows = True, is_linux = False), "windows-cpu"), + ], +) +def test_resolve_smoke_test_install_kind(host, expected): + assert resolve_smoke_test_install_kind(host) == expected + if expected.endswith("-cpu"): + assert expected not in M._GPU_INSTALL_KINDS + else: + assert expected in M._GPU_INSTALL_KINDS + + +# -- 3. validate_server integration (mocked subprocess + HTTP) --------------- + + +class _FakeResponse: + def __init__(self, status: int, body: bytes = b"{}"): + self.status = status + self._body = body + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def read(self): + return self._body + + +class _FakeProc: + """Stands in for the llama-server subprocess: writes a canned startup log + to the handle it was given, then reports itself alive until terminated.""" + + def __init__(self, log_text: str): + self._log_text = log_text + self._alive = True + + def __call__(self, command, *args, **kwargs): + handle = kwargs.get("stdout") + if handle is not None: + handle.write(self._log_text) + handle.flush() + return self + + def poll(self): + return None if self._alive else 0 + + def wait(self, timeout = None): + self._alive = False + return 0 + + def terminate(self): + self._alive = False + + def kill(self): + self._alive = False + + +@pytest.fixture +def patched_server(monkeypatch): + """Patch validate_server's external touchpoints; return a configure() + helper so each test picks the canned log it wants.""" + monkeypatch.setattr(M, "free_local_port", lambda: 18000) + monkeypatch.setattr(M, "binary_env", lambda *a, **k: {}) + monkeypatch.setattr(M.time, "sleep", lambda *a, **k: None) + monkeypatch.setattr(M.urllib.request, "urlopen", lambda *a, **k: _FakeResponse(200)) + + def configure(log_text): + monkeypatch.setattr(M.subprocess, "Popen", _FakeProc(log_text)) + + return configure + + +def _run_validate(tmp_path, host, install_kind): + server = tmp_path / "llama-server" + server.write_text("#!/bin/sh\n") + probe = tmp_path / "probe.gguf" + probe.write_bytes(b"GGUF") + validate_server(server, probe, host, tmp_path, install_kind = install_kind) + + +def test_gpu_intent_cpu_only_rejected(patched_server, tmp_path): + # The #5807 bug: GPU requested, binary ran on CPU -> reject. The exception + # must be GpuOffloadFailure specifically (not just the PrebuiltFallback + # base) so the --smoke-test CLI can map it to EXIT_FALLBACK vs EXIT_ERROR. + patched_server(CPU_ONLY_DEVICE_INFO_LOG) + with pytest.raises(M.GpuOffloadFailure, match = "entirely on CPU"): + _run_validate(tmp_path, windows_nvidia_host(), "windows-cuda") + assert issubclass(M.GpuOffloadFailure, PrebuiltFallback) + + +def test_gpu_intent_gpu_offload_accepted(patched_server, tmp_path): + patched_server(CUDA_DEVICE_INFO_LOG) + _run_validate(tmp_path, windows_nvidia_host(), "windows-cuda") # no raise + + +def test_cpu_kind_not_gpu_gated(patched_server, tmp_path): + # A windows-cpu bundle is the intentional fallback; never GPU-gated even + # if it (obviously) loads on CPU. + patched_server(CPU_ONLY_DEVICE_INFO_LOG) + _run_validate(tmp_path, windows_nvidia_host(), "windows-cpu") # no raise + + +def test_gpu_intent_no_signal_accepted(patched_server, tmp_path): + # No buffer/device signal -> conservative: do not reject on no evidence. + patched_server(NO_SIGNAL_LOG) + _run_validate(tmp_path, nvidia_host(), "linux-cuda") # no raise + + +def test_rocm_cpu_only_rejected(patched_server, tmp_path): + patched_server(CPU_ONLY_DEVICE_INFO_LOG) + with pytest.raises(PrebuiltFallback): + _run_validate(tmp_path, rocm_host(), "linux-rocm") + + +# -- 4. smoke_test_server_binary --------------------------------------------- + + +def test_smoke_test_missing_binary(tmp_path): + with pytest.raises(PrebuiltFallback, match = "not found"): + smoke_test_server_binary( + str(tmp_path / "nope"), nvidia_host(), install_dir = str(tmp_path) + ) + + +def test_smoke_test_delegates_to_validate_server(monkeypatch, tmp_path): + server = tmp_path / "llama-server" + server.write_text("#!/bin/sh\n") + probe = tmp_path / "probe.gguf" + probe.write_bytes(b"GGUF") + seen = {} + + def fake_validate(server_path, probe_path, host, install_dir, **kw): + seen["install_kind"] = kw.get("install_kind") + + monkeypatch.setattr(M, "validate_server", fake_validate) + kind = smoke_test_server_binary( + str(server), nvidia_host(), install_dir = str(tmp_path), probe = str(probe) + ) + assert kind == "linux-cuda" + assert seen["install_kind"] == "linux-cuda" + + +def test_smoke_test_propagates_failure(monkeypatch, tmp_path): + server = tmp_path / "llama-server" + server.write_text("#!/bin/sh\n") + probe = tmp_path / "probe.gguf" + probe.write_bytes(b"GGUF") + + def fake_validate(*a, **k): + raise PrebuiltFallback("loaded the model entirely on CPU") + + monkeypatch.setattr(M, "validate_server", fake_validate) + with pytest.raises(PrebuiltFallback): + smoke_test_server_binary( + str(server), + nvidia_host(), + install_dir = str(tmp_path), + probe = str(probe), + ) + + +# -- 5. --smoke-test CLI exit-code contract (the load-bearing fix contract) -- +# setup.sh / setup.ps1 branch on: 2 = ran on CPU (rebuild CPU), 1 = inconclusive +# (keep GPU build), 0 = offload confirmed. + + +def _run_main_smoke(monkeypatch, smoke_impl): + monkeypatch.setattr(M, "detect_host", lambda: nvidia_host()) + monkeypatch.setattr(M, "smoke_test_server_binary", smoke_impl) + monkeypatch.setattr( + sys, "argv", ["install_llama_prebuilt.py", "--smoke-test", "/x/llama-server"] + ) + return M.main() + + +def test_main_smoke_exit_success(monkeypatch): + rc = _run_main_smoke(monkeypatch, lambda *a, **k: "linux-cuda") + assert rc == M.EXIT_SUCCESS + + +def test_main_smoke_exit_fallback_on_cpu_only(monkeypatch): + def impl(*a, **k): + raise M.GpuOffloadFailure("loaded the model entirely on CPU") + + rc = _run_main_smoke(monkeypatch, impl) + assert rc == M.EXIT_FALLBACK # 2 -> setup scripts rebuild CPU + + +def test_main_smoke_exit_error_on_inconclusive(monkeypatch): + def impl(*a, **k): + raise PrebuiltFallback("llama-server exited during startup") + + rc = _run_main_smoke(monkeypatch, impl) + assert rc == M.EXIT_ERROR # 1 -> setup scripts keep the GPU build