From 08aeeaee4b134a3138b470b50ef08df7e045bb63 Mon Sep 17 00:00:00 2001 From: Roland Tannous <115670425+rolandtannous@users.noreply.github.com> Date: Tue, 24 Feb 2026 18:19:29 +0400 Subject: [PATCH] Fix llama-server: build in-tree, fix path resolution, add LD_LIBRARY_PATH --- .gitignore | 3 ++ setup.sh | 32 +++++++++++---------- studio/backend/core/inference/llama_cpp.py | 33 ++++++++++++++-------- 3 files changed, 42 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index e24c38c2b0..2ede66ec5b 100755 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,9 @@ unsloth_training_checkpoints/ *.gguf *.safetensors +# llama.cpp build (built by setup.sh, shared with unsloth-zoo export) +llama.cpp/ + # Built binaries (llama-server etc.) bin/ diff --git a/setup.sh b/setup.sh index 68c2d010da..36165dac65 100755 --- a/setup.sh +++ b/setup.sh @@ -207,10 +207,10 @@ else fi # ── 8. Build llama-server for GGUF inference ── -# Builds in an isolated temp directory to avoid conflicts with unsloth-zoo's -# own llama.cpp management (used for GGUF export). Only the llama-server -# binary is extracted to $REPO/bin/. -LLAMA_SERVER_BIN="$SCRIPT_DIR/bin/llama-server" +# Builds in-tree at $REPO/llama.cpp/. This directory is shared with +# unsloth-zoo's GGUF export pipeline — if converter/quantize are missing, +# unsloth-zoo will rebuild them on first export. We only build llama-server here. +LLAMA_SERVER_BIN="$SCRIPT_DIR/llama.cpp/build/bin/llama-server" if [ -f "$LLAMA_SERVER_BIN" ]; then echo "" echo "✅ llama-server already exists at $LLAMA_SERVER_BIN" @@ -226,10 +226,17 @@ else else echo "" echo "Building llama-server for GGUF inference..." - LLAMA_BUILD_TMP=$(mktemp -d) + LLAMA_CPP_DIR="$SCRIPT_DIR/llama.cpp" BUILD_OK=true - run_quiet "clone llama.cpp" git clone --depth 1 https://github.com/ggml-org/llama.cpp.git "$LLAMA_BUILD_TMP/llama.cpp" || BUILD_OK=false + if [ -d "$LLAMA_CPP_DIR/.git" ]; then + echo " llama.cpp repo already cloned, pulling latest..." + run_quiet "pull llama.cpp" git -C "$LLAMA_CPP_DIR" pull || true + else + # Remove any non-git llama.cpp directory (stale build artifacts) + rm -rf "$LLAMA_CPP_DIR" + run_quiet "clone llama.cpp" git clone --depth 1 https://github.com/ggml-org/llama.cpp.git "$LLAMA_CPP_DIR" || BUILD_OK=false + fi if [ "$BUILD_OK" = true ]; then CMAKE_ARGS="" @@ -258,27 +265,22 @@ else NCPU=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) - run_quiet "cmake llama.cpp" cmake -S "$LLAMA_BUILD_TMP/llama.cpp" -B "$LLAMA_BUILD_TMP/llama.cpp/build" $CMAKE_ARGS || BUILD_OK=false + run_quiet "cmake llama.cpp" cmake -S "$LLAMA_CPP_DIR" -B "$LLAMA_CPP_DIR/build" $CMAKE_ARGS || BUILD_OK=false fi if [ "$BUILD_OK" = true ]; then - run_quiet "build llama-server" cmake --build "$LLAMA_BUILD_TMP/llama.cpp/build" --config Release --target llama-server -j"$NCPU" || BUILD_OK=false + run_quiet "build llama-server" cmake --build "$LLAMA_CPP_DIR/build" --config Release --target llama-server -j"$NCPU" || BUILD_OK=false fi if [ "$BUILD_OK" = true ]; then - mkdir -p "$SCRIPT_DIR/bin" - if [ -f "$LLAMA_BUILD_TMP/llama.cpp/build/bin/llama-server" ]; then - cp "$LLAMA_BUILD_TMP/llama.cpp/build/bin/llama-server" "$LLAMA_SERVER_BIN" - echo "✅ llama-server built and installed to $LLAMA_SERVER_BIN" + if [ -f "$LLAMA_SERVER_BIN" ]; then + echo "✅ llama-server built at $LLAMA_SERVER_BIN" else echo "⚠️ llama-server binary not found after build — GGUF inference won't be available" fi else echo "⚠️ llama-server build failed — GGUF inference won't be available, but everything else works" fi - - # Clean up temp build directory - rm -rf "$LLAMA_BUILD_TMP" fi fi diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index dae8d20489..1489365686 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -65,9 +65,9 @@ class LlamaCppBackend: Search order: 1. LLAMA_SERVER_PATH environment variable - 2. ./bin/llama-server (built by setup.sh) + 2. ./llama.cpp/build/bin/llama-server (built by setup.sh in-tree) 3. llama-server on PATH (system install) - 4. ./llama.cpp/llama-server (unsloth-zoo build output) + 4. ./bin/llama-server (legacy: extracted binary) """ import os @@ -76,21 +76,23 @@ class LlamaCppBackend: if env_path and Path(env_path).is_file(): return env_path - # 2. Project bin/ directory (setup.sh output) - project_root = Path(__file__).resolve().parents[3] # core/inference/ → backend/ → studio/ → root - bin_path = project_root / "bin" / "llama-server" - if bin_path.is_file(): - return str(bin_path) + # Project root: llama_cpp.py → inference/ → core/ → backend/ → studio/ → root + project_root = Path(__file__).resolve().parents[4] + + # 2. In-tree llama.cpp build (setup.sh builds here) + build_path = project_root / "llama.cpp" / "build" / "bin" / "llama-server" + if build_path.is_file(): + return str(build_path) # 3. System PATH system_path = shutil.which("llama-server") if system_path: return system_path - # 4. unsloth-zoo build output (from GGUF export) - llama_cpp_path = project_root / "llama.cpp" / "llama-server" - if llama_cpp_path.is_file(): - return str(llama_cpp_path) + # 4. Legacy: extracted to bin/ + bin_path = project_root / "bin" / "llama-server" + if bin_path.is_file(): + return str(bin_path) return None @@ -154,11 +156,20 @@ class LlamaCppBackend: logger.info(f"Starting llama-server: {' '.join(cmd)}") + # Set LD_LIBRARY_PATH so llama-server can find its shared libs + # (libmtmd.so, libllama.so, etc.) which live next to the binary + import os + env = os.environ.copy() + binary_dir = str(Path(binary).parent) + existing_ld = env.get("LD_LIBRARY_PATH", "") + env["LD_LIBRARY_PATH"] = f"{binary_dir}:{existing_ld}" if existing_ld else binary_dir + self._process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + env=env, ) self._gguf_path = gguf_path