From a742dd49ce57c6cc5b4c38ae337c0fe6a87bb583 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 1 Jun 2026 15:55:04 +0000 Subject: [PATCH] CI: cross-platform GPU-offload spoof on Windows/macOS/Linux runners Adds a GPU-less smoke that drives install_llama_prebuilt.py --smoke-test against a fake llama-server emitting CPU-only vs GPU device_info logs, asserting a CPU-only binary tagged as a GPU install is rejected (exit 2) and a GPU one is accepted (exit 0), plus the selection and classifier unit tests, on all three OSes. This is the coverage gap that let the silent CPU-only path ship. --- .../workflows/studio-gpu-offload-smoke.yml | 65 +++++++++++ tests/studio/install/run_smoke_spoof.py | 101 ++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 .github/workflows/studio-gpu-offload-smoke.yml create mode 100644 tests/studio/install/run_smoke_spoof.py diff --git a/.github/workflows/studio-gpu-offload-smoke.yml b/.github/workflows/studio-gpu-offload-smoke.yml new file mode 100644 index 0000000000..92136e8c3b --- /dev/null +++ b/.github/workflows/studio-gpu-offload-smoke.yml @@ -0,0 +1,65 @@ +# Cross-platform GPU-offload validation smoke (no GPU required). +# +# Reproduces the silent CPU-only GGUF bug (#5807 / #5106 / #5830) with a fake +# llama-server that "starts and serves HTTP 200" while its log reports CPU-only +# or GPU offload, and asserts install_llama_prebuilt.py --smoke-test rejects the +# CPU-only-tagged-GPU case (exit 2) and accepts the GPU case (exit 0). Runs on +# GPU-less Windows / macOS / Linux runners, which is why this regressed +# untested. Also runs the pure-Python selection + classifier unit tests. + +name: Studio GPU Offload Smoke + +on: + pull_request: + paths: + - 'studio/install_llama_prebuilt.py' + - 'studio/setup.sh' + - 'studio/setup.ps1' + - 'studio/backend/core/inference/llama_cpp.py' + - 'tests/studio/install/**' + - 'tests/sh/test_llama_gpu_smoke.sh' + - '.github/workflows/studio-gpu-offload-smoke.yml' + push: + branches: [main, pip] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + gpu-offload-spoof: + name: GPU-offload spoof (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install pytest + run: python -m pip install --upgrade pip pytest + + - name: GPU-offload spoof (end to end, real subprocess + HTTP) + run: python tests/studio/install/run_smoke_spoof.py + + - name: Selection + classifier unit tests + run: > + python -m pytest + tests/studio/install/test_validate_server_gpu_offload.py + tests/studio/install/test_selection_logic.py + tests/studio/install/test_gpu_offload_spoof.py + -q + + - name: setup.sh smoke-exit classifier (POSIX) + if: runner.os != 'Windows' + run: bash tests/sh/test_llama_gpu_smoke.sh diff --git a/tests/studio/install/run_smoke_spoof.py b/tests/studio/install/run_smoke_spoof.py new file mode 100644 index 0000000000..2538e5ef52 --- /dev/null +++ b/tests/studio/install/run_smoke_spoof.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Cross-platform GPU-offload spoof driver for CI (no GPU required). + +Builds an OS-appropriate `llama-server` wrapper around fake_llama_server.py and +drives install_llama_prebuilt.py --smoke-test against it, asserting the exit +code contract the setup scripts depend on: + + FAKE_LLAMA_MODE=cpu , GPU install_kind -> exit 2 (EXIT_FALLBACK) rejected + FAKE_LLAMA_MODE=cuda, GPU install_kind -> exit 0 (EXIT_SUCCESS) accepted + FAKE_LLAMA_MODE=cpu , CPU install_kind -> exit 0 not gated + +Runs on windows-latest / macos-latest / ubuntu-latest. Exits non-zero on any +mismatch so the CI job fails loudly. +""" + +import os +import stat +import subprocess +import sys +import tempfile +from pathlib import Path + + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[2] +INSTALLER = REPO / "studio" / "install_llama_prebuilt.py" +FAKE = HERE / "fake_llama_server.py" +IS_WIN = sys.platform == "win32" + +GPU_KIND = { + "win32": "windows-cuda", + "darwin": "macos-arm64", +}.get(sys.platform, "linux-cuda") +CPU_KIND = { + "win32": "windows-cpu", + "darwin": "macos-cpu", +}.get(sys.platform, "linux-cpu") + + +def make_wrapper(workdir: Path) -> Path: + if IS_WIN: + wrapper = workdir / "llama-server.bat" + wrapper.write_text(f'@"{sys.executable}" "{FAKE}" %*\r\n') + return wrapper + wrapper = workdir / "llama-server" + wrapper.write_text(f'#!/bin/sh\nexec "{sys.executable}" "{FAKE}" "$@"\n') + wrapper.chmod(wrapper.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + return wrapper + + +def run_smoke(wrapper: Path, probe: Path, install_kind: str, mode: str) -> int: + env = dict(os.environ, FAKE_LLAMA_MODE=mode) + proc = subprocess.run( + [ + sys.executable, + str(INSTALLER), + "--smoke-test", str(wrapper), + "--probe", str(probe), + "--install-kind", install_kind, + ], + env=env, + capture_output=True, + text=True, + ) + sys.stdout.write(proc.stdout) + sys.stderr.write(proc.stderr) + return proc.returncode + + +def main() -> int: + failures = [] + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + wrapper = make_wrapper(work) + probe = work / "probe.gguf" + probe.write_bytes(b"GGUF\x00fake") + + cases = [ + ("cpu", GPU_KIND, 2, "CPU-only binary tagged GPU is rejected"), + ("offloaded_zero", GPU_KIND, 2, "offloaded 0/N tagged GPU is rejected"), + ("cuda", GPU_KIND, 0, "GPU binary tagged GPU is accepted"), + ("cuda_buffer", GPU_KIND, 0, "GPU buffer-format binary is accepted"), + ("cpu", CPU_KIND, 0, "CPU binary tagged CPU is not gated"), + ("no_signal", GPU_KIND, 0, "no-signal log is not rejected"), + ] + for mode, kind, expected, label in cases: + rc = run_smoke(wrapper, probe, kind, mode) + ok = rc == expected + print(f"[{'PASS' if ok else 'FAIL'}] {label}: mode={mode} kind={kind} exit={rc} (want {expected})") + if not ok: + failures.append(label) + + if failures: + print(f"\n{len(failures)} spoof case(s) failed: {failures}") + return 1 + print("\nAll GPU-offload spoof cases passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())