From 27bc44c460a96722f251f0858128789c75d94b4d Mon Sep 17 00:00:00 2001
From: Daniel Han <danielhanchen@gmail.com>
Date: Sun, 21 Jun 2026 01:31:37 -0700
Subject: [PATCH] fix(provision): functionally confirm CUDA before the step-0
 rebuild-skip

is_cuda_server() treats a co-located libggml-cuda.so* as proof the server is
CUDA-ready. That's normally true (llama.cpp dlopens the backend from beside the
binary), but an *interrupted* build (thermal/power shutdown -- common on the
NVIDIA-ARM laptops this path targets) can leave a half-linked libggml-cuda.so
next to the server: present, so is_cuda_server() matches, yet the backend fails
to load at runtime. The post-build path already wipes+rebuilds such a partial
.so, but the step-0 early-skip trusted it and never rebuilt -- so Studio could
report GGUF CUDA inference ready while running a broken/non-CUDA backend.

Gate the early-skip with cuda_server_probe(): 'llama-server --list-devices'
enumerates backends and exits (cheap, no server spin-up). Only a definitive
'flag supported, ran, but no CUDA device' triggers a clean rebuild; a timeout or
an old pin without --list-devices stays inconclusive and keeps trusting the .so,
so we never force a needless, thermally-expensive rebuild. Probe logic verified
against healthy/broken/unsupported/timeout stubs (0/1/2/2).

Addresses Codex review P2 (provision_llama_cuda.sh).
---
 studio/scripts/provision_llama_cuda.sh | 30 ++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/studio/scripts/provision_llama_cuda.sh b/studio/scripts/provision_llama_cuda.sh
index a7fc34103c..585f0baf01 100644
--- a/studio/scripts/provision_llama_cuda.sh
+++ b/studio/scripts/provision_llama_cuda.sh
@@ -23,10 +23,36 @@ is_cuda_server() {
     return 1
 }
 
+# Functional confirmation that the server's CUDA backend actually loads, used only
+# to gate the step-0 early-skip. is_cuda_server() trusts a co-located
+# libggml-cuda.so*, but an *interrupted* build (thermal/power shutdown -- common on
+# this hardware) can leave a half-linked libggml-cuda.so beside the binary: present,
+# so is_cuda_server() matches, yet the backend fails to dlopen at runtime. The
+# post-build path already wipes+rebuilds such a partial .so, but the early-skip would
+# trust it and never rebuild. `--list-devices` enumerates backends and exits, so it's
+# a cheap probe (no server spin-up). Returns: 0 = a CUDA device is listed; 1 = the
+# flag is supported and ran but no CUDA device appeared (broken/partial backend ->
+# rebuild); 2 = inconclusive (timed out, or an old pin without --list-devices) -> keep
+# trusting the .so so we never force a needless, thermally-expensive rebuild.
+cuda_server_probe() {
+    local _to="" _out _rc
+    command -v timeout >/dev/null 2>&1 && _to="timeout 60"
+    _out="$( $_to "$1" --list-devices 2>&1 )"; _rc=$?
+    [ "$_rc" -eq 124 ] && return 2                                   # timed out
+    printf '%s\n' "$_out" | grep -qiE 'CUDA[0-9]'        && return 0 # CUDA device listed
+    printf '%s\n' "$_out" | grep -qi  'available devices' && return 1 # ran, but none is CUDA
+    return 2                                                         # flag unsupported / couldn't run
+}
+
 # 0. Already provisioned?
 if is_cuda_server "$SERVER"; then
-    log "CUDA llama-server already present: $SERVER"
-    exit 0
+    cuda_server_probe "$SERVER"; _probe=$?
+    if [ "$_probe" -ne 1 ]; then
+        log "CUDA llama-server already present: $SERVER"
+        exit 0
+    fi
+    log "existing llama-server has libggml-cuda.so but lists no CUDA device (partial/broken build); rebuilding clean"
+    rm -rf "$LLAMA_DIR/build"   # force a clean reconfigure+build below
 fi
 
 # 1. Require an NVIDIA GPU (this script is only meaningful with one).