Fix llama-server: build in-tree, fix path resolution, add LD_LIBRARY_PATH

.gitignore

@@ -24,6 +24,9 @@ unsloth_training_checkpoints/
 *.gguf
 *.safetensors
 
+# llama.cpp build (built by setup.sh, shared with unsloth-zoo export)
+llama.cpp/
+
 # Built binaries (llama-server etc.)
 bin/
 

setup.sh

@@ -207,10 +207,10 @@ else
 fi
 
 # ── 8. Build llama-server for GGUF inference ──
-# Builds in an isolated temp directory to avoid conflicts with unsloth-zoo's
-# own llama.cpp management (used for GGUF export). Only the llama-server
-# binary is extracted to $REPO/bin/.
-LLAMA_SERVER_BIN="$SCRIPT_DIR/bin/llama-server"
+# Builds in-tree at $REPO/llama.cpp/. This directory is shared with
+# unsloth-zoo's GGUF export pipeline — if converter/quantize are missing,
+# unsloth-zoo will rebuild them on first export. We only build llama-server here.
+LLAMA_SERVER_BIN="$SCRIPT_DIR/llama.cpp/build/bin/llama-server"
 if [ -f "$LLAMA_SERVER_BIN" ]; then
     echo ""
     echo "✅ llama-server already exists at $LLAMA_SERVER_BIN"
@@ -226,10 +226,17 @@ else
     else
         echo ""
         echo "Building llama-server for GGUF inference..."
-        LLAMA_BUILD_TMP=$(mktemp -d)
+        LLAMA_CPP_DIR="$SCRIPT_DIR/llama.cpp"
 
         BUILD_OK=true
-        run_quiet "clone llama.cpp" git clone --depth 1 https://github.com/ggml-org/llama.cpp.git "$LLAMA_BUILD_TMP/llama.cpp" || BUILD_OK=false
+        if [ -d "$LLAMA_CPP_DIR/.git" ]; then
+            echo "   llama.cpp repo already cloned, pulling latest..."
+            run_quiet "pull llama.cpp" git -C "$LLAMA_CPP_DIR" pull || true
+        else
+            # Remove any non-git llama.cpp directory (stale build artifacts)
+            rm -rf "$LLAMA_CPP_DIR"
+            run_quiet "clone llama.cpp" git clone --depth 1 https://github.com/ggml-org/llama.cpp.git "$LLAMA_CPP_DIR" || BUILD_OK=false
+        fi
 
         if [ "$BUILD_OK" = true ]; then
             CMAKE_ARGS=""
@@ -258,27 +265,22 @@ else
 
             NCPU=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
 
-            run_quiet "cmake llama.cpp" cmake -S "$LLAMA_BUILD_TMP/llama.cpp" -B "$LLAMA_BUILD_TMP/llama.cpp/build" $CMAKE_ARGS || BUILD_OK=false
+            run_quiet "cmake llama.cpp" cmake -S "$LLAMA_CPP_DIR" -B "$LLAMA_CPP_DIR/build" $CMAKE_ARGS || BUILD_OK=false
         fi
 
         if [ "$BUILD_OK" = true ]; then
-            run_quiet "build llama-server" cmake --build "$LLAMA_BUILD_TMP/llama.cpp/build" --config Release --target llama-server -j"$NCPU" || BUILD_OK=false
+            run_quiet "build llama-server" cmake --build "$LLAMA_CPP_DIR/build" --config Release --target llama-server -j"$NCPU" || BUILD_OK=false
         fi
 
         if [ "$BUILD_OK" = true ]; then
-            mkdir -p "$SCRIPT_DIR/bin"
-            if [ -f "$LLAMA_BUILD_TMP/llama.cpp/build/bin/llama-server" ]; then
-                cp "$LLAMA_BUILD_TMP/llama.cpp/build/bin/llama-server" "$LLAMA_SERVER_BIN"
-                echo "✅ llama-server built and installed to $LLAMA_SERVER_BIN"
+            if [ -f "$LLAMA_SERVER_BIN" ]; then
+                echo "✅ llama-server built at $LLAMA_SERVER_BIN"
             else
                 echo "⚠️  llama-server binary not found after build — GGUF inference won't be available"
             fi
         else
             echo "⚠️  llama-server build failed — GGUF inference won't be available, but everything else works"
         fi
-
-        # Clean up temp build directory
-        rm -rf "$LLAMA_BUILD_TMP"
     fi
 fi
 

studio/backend/core/inference/llama_cpp.py

@@ -65,9 +65,9 @@ class LlamaCppBackend:
 
         Search order:
         1. LLAMA_SERVER_PATH environment variable
-        2. ./bin/llama-server  (built by setup.sh)
+        2. ./llama.cpp/build/bin/llama-server  (built by setup.sh in-tree)
         3. llama-server on PATH  (system install)
-        4. ./llama.cpp/llama-server  (unsloth-zoo build output)
+        4. ./bin/llama-server  (legacy: extracted binary)
         """
         import os
 
@@ -76,21 +76,23 @@ class LlamaCppBackend:
         if env_path and Path(env_path).is_file():
             return env_path
 
-        # 2. Project bin/ directory (setup.sh output)
-        project_root = Path(__file__).resolve().parents[3]  # core/inference/ → backend/ → studio/ → root
-        bin_path = project_root / "bin" / "llama-server"
-        if bin_path.is_file():
-            return str(bin_path)
+        # Project root: llama_cpp.py → inference/ → core/ → backend/ → studio/ → root
+        project_root = Path(__file__).resolve().parents[4]
+
+        # 2. In-tree llama.cpp build (setup.sh builds here)
+        build_path = project_root / "llama.cpp" / "build" / "bin" / "llama-server"
+        if build_path.is_file():
+            return str(build_path)
 
         # 3. System PATH
         system_path = shutil.which("llama-server")
         if system_path:
             return system_path
 
-        # 4. unsloth-zoo build output (from GGUF export)
-        llama_cpp_path = project_root / "llama.cpp" / "llama-server"
-        if llama_cpp_path.is_file():
-            return str(llama_cpp_path)
+        # 4. Legacy: extracted to bin/
+        bin_path = project_root / "bin" / "llama-server"
+        if bin_path.is_file():
+            return str(bin_path)
 
         return None
 
@@ -154,11 +156,20 @@ class LlamaCppBackend:
 
             logger.info(f"Starting llama-server: {' '.join(cmd)}")
 
+            # Set LD_LIBRARY_PATH so llama-server can find its shared libs
+            # (libmtmd.so, libllama.so, etc.) which live next to the binary
+            import os
+            env = os.environ.copy()
+            binary_dir = str(Path(binary).parent)
+            existing_ld = env.get("LD_LIBRARY_PATH", "")
+            env["LD_LIBRARY_PATH"] = f"{binary_dir}:{existing_ld}" if existing_ld else binary_dir
+
             self._process = subprocess.Popen(
                 cmd,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,
                 text=True,
+                env=env,
             )
 
             self._gguf_path = gguf_path