From 34cdbf42bfd77c73573dc7ec6c11fdb3be7d724f Mon Sep 17 00:00:00 2001
From: Daniel Han <danielhanchen@gmail.com>
Date: Mon, 16 Mar 2026 14:02:20 +0000
Subject: [PATCH 01/10] studio: training progress visibility + deferred
 llama.cpp compilation

Training progress:
- Show row counts in status messages: "Loaded dataset from HuggingFace:
  Open-Orca/OpenOrca (4,233,923 rows)" instead of just the dataset name
- Emit "Formatting dataset (N rows)..." and "Applying chat template
  (N rows)..." status updates so users see progress during the
  preprocessing stages that previously appeared stuck

Deferred llama.cpp compilation:
- Add LlamaCppBuilder that runs cmake build in a background thread
  at server startup if the llama-server binary is missing
- Studio starts immediately and is usable for training/non-GGUF tasks
  while llama.cpp compiles in the background
- GGUF model loads wait for the build to finish with a helpful message
- Add /api/inference/llama-cpp-status endpoint for build status
- Frontend shows "Waiting for llama.cpp to compile..." toast when
  loading a GGUF while build is in progress
---
 .../core/inference/llama_cpp_builder.py       | 248 ++++++++++++++++++
 studio/backend/core/training/trainer.py       |   5 +-
 studio/backend/main.py                        |   6 +
 studio/backend/routes/inference.py            |  26 ++
 .../backend/utils/datasets/dataset_utils.py   |   9 +
 .../src/features/chat/api/chat-api.ts         |   9 +
 .../chat/hooks/use-chat-model-runtime.ts      |  20 ++
 7 files changed, 321 insertions(+), 2 deletions(-)
 create mode 100644 studio/backend/core/inference/llama_cpp_builder.py

diff --git a/studio/backend/core/inference/llama_cpp_builder.py b/studio/backend/core/inference/llama_cpp_builder.py
new file mode 100644
index 0000000000..268bddba8f
--- /dev/null
+++ b/studio/backend/core/inference/llama_cpp_builder.py
@@ -0,0 +1,248 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""
+Background llama.cpp compilation.
+
+On server startup, if the llama-server binary is not found, a background
+thread clones and builds llama.cpp from source. Features that need
+llama.cpp (GGUF chat, export, AI Assist) can await `wait_for_ready()`
+which blocks until compilation finishes or fails.
+"""
+
+import os
+import shutil
+import subprocess
+import threading
+from pathlib import Path
+from typing import Optional
+
+import structlog
+from loggers import get_logger
+
+logger = get_logger(__name__)
+
+
+class LlamaCppBuilder:
+    """Manages background llama.cpp compilation."""
+
+    def __init__(self):
+        self._ready = threading.Event()
+        self._building = False
+        self._error: Optional[str] = None
+        self._thread: Optional[threading.Thread] = None
+        self._binary_path: Optional[str] = None
+
+    @property
+    def is_ready(self) -> bool:
+        return self._ready.is_set()
+
+    @property
+    def is_building(self) -> bool:
+        return self._building
+
+    @property
+    def error(self) -> Optional[str]:
+        return self._error
+
+    @property
+    def binary_path(self) -> Optional[str]:
+        return self._binary_path
+
+    def check_and_build(self) -> None:
+        """Check if llama-server exists. If not, start background build."""
+        from core.inference.llama_cpp import LlamaCppBackend
+
+        binary = LlamaCppBackend._find_llama_server_binary()
+        if binary:
+            self._binary_path = binary
+            self._ready.set()
+            logger.info(f"llama-server binary found: {binary}")
+            return
+
+        logger.info("llama-server binary not found, starting background build...")
+        self._building = True
+        self._thread = threading.Thread(
+            target=self._build, daemon=True, name="llama-cpp-build"
+        )
+        self._thread.start()
+
+    def wait_for_ready(self, timeout: Optional[float] = None) -> bool:
+        """Block until llama.cpp is ready. Returns True if ready, False on timeout."""
+        return self._ready.wait(timeout=timeout)
+
+    def _build(self) -> None:
+        """Clone and build llama.cpp in the background."""
+        try:
+            import sys
+
+            llama_dir = Path.home() / ".unsloth" / "llama.cpp"
+            binary_name = (
+                "llama-server.exe" if sys.platform == "win32" else "llama-server"
+            )
+
+            # Don't rebuild if binary appeared while we were starting
+            from core.inference.llama_cpp import LlamaCppBackend
+
+            binary = LlamaCppBackend._find_llama_server_binary()
+            if binary:
+                self._binary_path = binary
+                self._building = False
+                self._ready.set()
+                return
+
+            # Check prerequisites
+            if not shutil.which("cmake"):
+                self._error = "cmake not found. Install cmake to enable GGUF inference."
+                self._building = False
+                logger.warning(self._error)
+                return
+            if not shutil.which("git"):
+                self._error = "git not found. Install git to enable GGUF inference."
+                self._building = False
+                logger.warning(self._error)
+                return
+
+            # Clone
+            if llama_dir.exists():
+                shutil.rmtree(llama_dir, ignore_errors=True)
+
+            logger.info("Cloning llama.cpp...")
+            subprocess.run(
+                [
+                    "git",
+                    "clone",
+                    "--depth",
+                    "1",
+                    "https://github.com/ggml-org/llama.cpp.git",
+                    str(llama_dir),
+                ],
+                check=True,
+                capture_output=True,
+            )
+
+            # Detect CUDA
+            cmake_args = [
+                "-DBUILD_SHARED_LIBS=OFF",
+            ]
+            nvcc = shutil.which("nvcc")
+            if not nvcc:
+                for cuda_dir in sorted(
+                    Path("/usr/local").glob("cuda-*/bin/nvcc"), reverse=True
+                ):
+                    nvcc = str(cuda_dir)
+                    break
+            if not nvcc and Path("/usr/local/cuda/bin/nvcc").is_file():
+                nvcc = "/usr/local/cuda/bin/nvcc"
+
+            if nvcc:
+                logger.info(f"CUDA detected: {nvcc}")
+                cmake_args.append("-DGGML_CUDA=ON")
+                # Detect compute capabilities
+                try:
+                    result = subprocess.run(
+                        [
+                            "nvidia-smi",
+                            "--query-gpu=compute_cap",
+                            "--format=csv,noheader",
+                        ],
+                        capture_output=True,
+                        text=True,
+                        timeout=10,
+                    )
+                    if result.returncode == 0:
+                        caps = set()
+                        for line in result.stdout.strip().split("\n"):
+                            cap = line.strip().replace(".", "")
+                            if cap:
+                                caps.add(cap)
+                        if caps:
+                            cmake_args.append(
+                                f"-DCMAKE_CUDA_ARCHITECTURES={';'.join(sorted(caps))}"
+                            )
+                except Exception:
+                    pass
+                cmake_args.append("-DCMAKE_CUDA_FLAGS=--threads=0")
+            else:
+                logger.info("No CUDA detected, building CPU-only llama.cpp")
+
+            # Use Ninja if available
+            generator_args = []
+            if shutil.which("ninja"):
+                generator_args = ["-G", "Ninja"]
+
+            # Configure
+            build_dir = llama_dir / "build"
+            logger.info("Configuring llama.cpp build...")
+            subprocess.run(
+                [
+                    "cmake",
+                    "-B",
+                    str(build_dir),
+                    "-S",
+                    str(llama_dir),
+                ]
+                + generator_args
+                + cmake_args,
+                check=True,
+                capture_output=True,
+            )
+
+            # Build
+            logger.info("Building llama.cpp (this may take a few minutes)...")
+            subprocess.run(
+                [
+                    "cmake",
+                    "--build",
+                    str(build_dir),
+                    "--config",
+                    "Release",
+                    "--target",
+                    "llama-server",
+                    "llama-quantize",
+                    "-j",
+                ],
+                check=True,
+                capture_output=True,
+            )
+
+            # Symlink llama-quantize for unsloth-zoo's check_llama_cpp()
+            quantize_src = build_dir / "bin" / "llama-quantize"
+            quantize_link = llama_dir / "llama-quantize"
+            if quantize_src.is_file() and not quantize_link.exists():
+                try:
+                    quantize_link.symlink_to(quantize_src)
+                except Exception:
+                    pass
+
+            # Verify
+            expected = build_dir / "bin" / binary_name
+            if expected.is_file():
+                self._binary_path = str(expected)
+                logger.info(f"llama.cpp build complete: {self._binary_path}")
+            else:
+                self._error = f"Build completed but binary not found at {expected}"
+                logger.error(self._error)
+
+        except subprocess.CalledProcessError as e:
+            stderr = e.stderr.decode() if e.stderr else ""
+            self._error = f"llama.cpp build failed: {stderr[-500:]}"
+            logger.error(self._error)
+        except Exception as e:
+            self._error = f"llama.cpp build error: {e}"
+            logger.error(self._error)
+        finally:
+            self._building = False
+            self._ready.set()  # Unblock waiters even on failure
+
+
+# ── Singleton ────────────────────────────────────────────────────
+
+_builder: Optional[LlamaCppBuilder] = None
+
+
+def get_llama_cpp_builder() -> LlamaCppBuilder:
+    global _builder
+    if _builder is None:
+        _builder = LlamaCppBuilder()
+    return _builder
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py
index ecc11c30d5..af20146a7f 100644
--- a/studio/backend/core/training/trainer.py
+++ b/studio/backend/core/training/trainer.py
@@ -2352,11 +2352,12 @@ class UnslothTrainer:
                     logger.info("Stopped during dataset loading\n")
                     return None
 
+                n_rows = len(dataset) if hasattr(dataset, "__len__") else 0
                 self._update_progress(
-                    status_message = f"Loaded dataset from HuggingFace: {dataset_source}"
+                    status_message = f"Loaded dataset from HuggingFace: {dataset_source} ({n_rows:,} rows)"
                 )
                 logger.info(
-                    f"Loaded dataset from Hugging Face: {dataset_source} ({len(dataset)} rows)\n"
+                    f"Loaded dataset from Hugging Face: {dataset_source} ({n_rows:,} rows)\n"
                 )
 
                 # Resolve eval split from a separate HF split (explicit or auto-detected)
diff --git a/studio/backend/main.py b/studio/backend/main.py
index 3ab846306e..4495c99e2f 100644
--- a/studio/backend/main.py
+++ b/studio/backend/main.py
@@ -75,6 +75,12 @@ async def lifespan(app: FastAPI):
 
     threading.Thread(target = _precache, daemon = True).start()
 
+    # Build llama.cpp in the background if the binary is missing.
+    # Studio starts immediately; GGUF features wait for the build to finish.
+    from core.inference.llama_cpp_builder import get_llama_cpp_builder
+
+    get_llama_cpp_builder().check_and_build()
+
     if storage.ensure_default_admin():
         bootstrap_pw = storage.get_bootstrap_password()
         app.state.bootstrap_password = bootstrap_pw
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index e93ba1d6d6..34a5f345cb 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -111,6 +111,19 @@ async def load_model(
 
         # ── GGUF path: load via llama-server ──────────────────────
         if config.is_gguf:
+            # Wait for llama.cpp to be compiled if a background build is in progress
+            from core.inference.llama_cpp_builder import get_llama_cpp_builder
+
+            builder = get_llama_cpp_builder()
+            if not builder.is_ready:
+                logger.info("Waiting for llama.cpp build to finish...")
+                ready = await asyncio.to_thread(builder.wait_for_ready, timeout = 600)
+                if not ready or builder.error:
+                    raise HTTPException(
+                        status_code = 503,
+                        detail = builder.error or "llama.cpp is still compiling. Please wait.",
+                    )
+
             llama_backend = get_llama_cpp_backend()
             unsloth_backend = get_inference_backend()
 
@@ -538,6 +551,19 @@ async def get_status(
         raise HTTPException(status_code = 500, detail = f"Failed to get status: {str(e)}")
 
 
+@router.get("/llama-cpp-status")
+async def llama_cpp_status():
+    """Check if llama.cpp is ready (built and available)."""
+    from core.inference.llama_cpp_builder import get_llama_cpp_builder
+
+    builder = get_llama_cpp_builder()
+    return {
+        "ready": builder.is_ready and builder.error is None,
+        "building": builder.is_building,
+        "error": builder.error,
+    }
+
+
 # =====================================================================
 # Audio (TTS) Generation  (/audio/generate)
 # =====================================================================
diff --git a/studio/backend/utils/datasets/dataset_utils.py b/studio/backend/utils/datasets/dataset_utils.py
index 9d15b86ca1..1d2e4083ec 100644
--- a/studio/backend/utils/datasets/dataset_utils.py
+++ b/studio/backend/utils/datasets/dataset_utils.py
@@ -1092,6 +1092,11 @@ def format_and_template_dataset(
     # LLM FLOW (Existing code)
     else:
         # Step 1: Format the dataset
+        n_rows = len(dataset) if hasattr(dataset, "__len__") else None
+        if progress_callback and n_rows:
+            progress_callback(
+                status_message = f"Formatting dataset ({n_rows:,} rows)..."
+            )
         dataset_info = format_dataset(
             dataset,
             format_type = format_type,
@@ -1106,6 +1111,10 @@ def format_and_template_dataset(
         )
 
         # Step 2: Apply chat template
+        if progress_callback and n_rows:
+            progress_callback(
+                status_message = f"Applying chat template ({n_rows:,} rows)..."
+            )
         # Gemma emits a leading <bos> that must be stripped for text-only chatml/sharegpt.
         is_alpaca = format_type == "alpaca" or (
             format_type == "auto" and dataset_info["detected_format"] == "alpaca"
diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts
index 01bc762d86..0f63e0d462 100644
--- a/studio/frontend/src/features/chat/api/chat-api.ts
+++ b/studio/frontend/src/features/chat/api/chat-api.ts
@@ -62,6 +62,15 @@ export async function getInferenceStatus(): Promise<InferenceStatusResponse> {
   return parseJsonOrThrow<InferenceStatusResponse>(response);
 }
 
+export async function getLlamaCppStatus(): Promise<{
+  ready: boolean;
+  building: boolean;
+  error: string | null;
+}> {
+  const response = await authFetch("/api/inference/llama-cpp-status");
+  return parseJsonOrThrow(response);
+}
+
 export async function loadModel(
   payload: LoadModelRequest,
 ): Promise<LoadModelResponse> {
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index b79c3073b2..e06418ce6d 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -9,6 +9,7 @@ import {
   getDownloadProgress,
   getGgufDownloadProgress,
   getInferenceStatus,
+  getLlamaCppStatus,
   listLoras,
   listModels,
   loadModel,
@@ -318,6 +319,25 @@ export function useChatModelRuntime() {
       try {
         async function performLoad(): Promise<void> {
           if (abortCtrl.signal.aborted) throw new Error("Cancelled");
+
+          // For GGUF models, check if llama.cpp is still compiling
+          if (ggufVariant) {
+            try {
+              const llamaStatus = await getLlamaCppStatus();
+              if (llamaStatus.building) {
+                toast.info("Waiting for llama.cpp to compile...", {
+                  description: "This is a one-time build. Studio is still usable for non-GGUF tasks.",
+                  duration: 8000,
+                });
+              } else if (llamaStatus.error) {
+                throw new Error(llamaStatus.error);
+              }
+            } catch (e) {
+              if (e instanceof Error && e.message.includes("not found")) throw e;
+              // Endpoint might not exist on older backends, ignore
+            }
+          }
+
           let previousWasUnloaded = false;
           const currentCheckpoint =
             useChatRuntimeStore.getState().params.checkpoint;

From 056be824b2cf04e50806c9430a1275d4f334e83f Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 16 Mar 2026 14:02:48 +0000
Subject: [PATCH 02/10] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .../core/inference/llama_cpp_builder.py       | 26 +++++++++----------
 studio/backend/routes/inference.py            |  3 ++-
 .../backend/utils/datasets/dataset_utils.py   |  4 +--
 3 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/studio/backend/core/inference/llama_cpp_builder.py b/studio/backend/core/inference/llama_cpp_builder.py
index 268bddba8f..4a1de0cf7b 100644
--- a/studio/backend/core/inference/llama_cpp_builder.py
+++ b/studio/backend/core/inference/llama_cpp_builder.py
@@ -63,13 +63,13 @@ class LlamaCppBuilder:
         logger.info("llama-server binary not found, starting background build...")
         self._building = True
         self._thread = threading.Thread(
-            target=self._build, daemon=True, name="llama-cpp-build"
+            target = self._build, daemon = True, name = "llama-cpp-build"
         )
         self._thread.start()
 
     def wait_for_ready(self, timeout: Optional[float] = None) -> bool:
         """Block until llama.cpp is ready. Returns True if ready, False on timeout."""
-        return self._ready.wait(timeout=timeout)
+        return self._ready.wait(timeout = timeout)
 
     def _build(self) -> None:
         """Clone and build llama.cpp in the background."""
@@ -105,7 +105,7 @@ class LlamaCppBuilder:
 
             # Clone
             if llama_dir.exists():
-                shutil.rmtree(llama_dir, ignore_errors=True)
+                shutil.rmtree(llama_dir, ignore_errors = True)
 
             logger.info("Cloning llama.cpp...")
             subprocess.run(
@@ -117,8 +117,8 @@ class LlamaCppBuilder:
                     "https://github.com/ggml-org/llama.cpp.git",
                     str(llama_dir),
                 ],
-                check=True,
-                capture_output=True,
+                check = True,
+                capture_output = True,
             )
 
             # Detect CUDA
@@ -128,7 +128,7 @@ class LlamaCppBuilder:
             nvcc = shutil.which("nvcc")
             if not nvcc:
                 for cuda_dir in sorted(
-                    Path("/usr/local").glob("cuda-*/bin/nvcc"), reverse=True
+                    Path("/usr/local").glob("cuda-*/bin/nvcc"), reverse = True
                 ):
                     nvcc = str(cuda_dir)
                     break
@@ -146,9 +146,9 @@ class LlamaCppBuilder:
                             "--query-gpu=compute_cap",
                             "--format=csv,noheader",
                         ],
-                        capture_output=True,
-                        text=True,
-                        timeout=10,
+                        capture_output = True,
+                        text = True,
+                        timeout = 10,
                     )
                     if result.returncode == 0:
                         caps = set()
@@ -184,8 +184,8 @@ class LlamaCppBuilder:
                 ]
                 + generator_args
                 + cmake_args,
-                check=True,
-                capture_output=True,
+                check = True,
+                capture_output = True,
             )
 
             # Build
@@ -202,8 +202,8 @@ class LlamaCppBuilder:
                     "llama-quantize",
                     "-j",
                 ],
-                check=True,
-                capture_output=True,
+                check = True,
+                capture_output = True,
             )
 
             # Symlink llama-quantize for unsloth-zoo's check_llama_cpp()
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 34a5f345cb..75e6472d18 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -121,7 +121,8 @@ async def load_model(
                 if not ready or builder.error:
                     raise HTTPException(
                         status_code = 503,
-                        detail = builder.error or "llama.cpp is still compiling. Please wait.",
+                        detail = builder.error
+                        or "llama.cpp is still compiling. Please wait.",
                     )
 
             llama_backend = get_llama_cpp_backend()
diff --git a/studio/backend/utils/datasets/dataset_utils.py b/studio/backend/utils/datasets/dataset_utils.py
index 1d2e4083ec..0808509091 100644
--- a/studio/backend/utils/datasets/dataset_utils.py
+++ b/studio/backend/utils/datasets/dataset_utils.py
@@ -1094,9 +1094,7 @@ def format_and_template_dataset(
         # Step 1: Format the dataset
         n_rows = len(dataset) if hasattr(dataset, "__len__") else None
         if progress_callback and n_rows:
-            progress_callback(
-                status_message = f"Formatting dataset ({n_rows:,} rows)..."
-            )
+            progress_callback(status_message = f"Formatting dataset ({n_rows:,} rows)...")
         dataset_info = format_dataset(
             dataset,
             format_type = format_type,

From d7a78599e62290d74b1371d8364e0f475de00521 Mon Sep 17 00:00:00 2001
From: Daniel Han <danielhanchen@gmail.com>
Date: Mon, 16 Mar 2026 14:14:31 +0000
Subject: [PATCH 03/10] studio: optimize llama.cpp build with static linking,
 ccache, ninja

Benchmarked BUILD_SHARED_LIBS ON vs OFF:
- Build time identical (~55s both on 192-core B200)
- Static: 75MB self-contained binary, no LD_LIBRARY_PATH needed
- Shared: 7.4MB + .so deps, needs lib path management
- Both work correctly

Optimizations applied to LlamaCppBuilder:
- BUILD_SHARED_LIBS=OFF: static binary, simpler deployment
- ccache: 27x faster rebuilds (55s -> 2s with warm cache)
- Ninja generator: parallel builds
- CMAKE_CUDA_ARCHITECTURES: build only for detected GPU arch
- CMAKE_CUDA_FLAGS=--threads=0: multi-threaded CUDA compilation
- GGML_NATIVE=ON: native CPU optimizations
- LLAMA_BUILD_TESTS=OFF, LLAMA_BUILD_EXAMPLES=OFF: skip unused targets
- LLAMA_BUILD_SERVER=ON: build only what we need
---
 .../core/inference/llama_cpp_builder.py       | 27 +++++++++++++++----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/studio/backend/core/inference/llama_cpp_builder.py b/studio/backend/core/inference/llama_cpp_builder.py
index 4a1de0cf7b..5f57329003 100644
--- a/studio/backend/core/inference/llama_cpp_builder.py
+++ b/studio/backend/core/inference/llama_cpp_builder.py
@@ -121,10 +121,26 @@ class LlamaCppBuilder:
                 capture_output = True,
             )
 
-            # Detect CUDA
+            # Build configuration: static binary, only needed targets, max parallelism
             cmake_args = [
-                "-DBUILD_SHARED_LIBS=OFF",
+                "-DBUILD_SHARED_LIBS=OFF",     # Self-contained binary, no LD_LIBRARY_PATH needed
+                "-DGGML_NATIVE=ON",            # Native CPU optimizations
+                "-DLLAMA_BUILD_TESTS=OFF",     # Skip tests
+                "-DLLAMA_BUILD_EXAMPLES=OFF",  # Skip examples (we build server explicitly)
+                "-DLLAMA_BUILD_SERVER=ON",     # Ensure server target is available
             ]
+
+            # Use ccache if available (27x faster rebuilds)
+            ccache = shutil.which("ccache")
+            if ccache:
+                cmake_args.extend([
+                    f"-DCMAKE_C_COMPILER_LAUNCHER={ccache}",
+                    f"-DCMAKE_CXX_COMPILER_LAUNCHER={ccache}",
+                    f"-DCMAKE_CUDA_COMPILER_LAUNCHER={ccache}",
+                ])
+                logger.info("Using ccache for faster compilation")
+
+            # Detect CUDA
             nvcc = shutil.which("nvcc")
             if not nvcc:
                 for cuda_dir in sorted(
@@ -138,7 +154,7 @@ class LlamaCppBuilder:
             if nvcc:
                 logger.info(f"CUDA detected: {nvcc}")
                 cmake_args.append("-DGGML_CUDA=ON")
-                # Detect compute capabilities
+                # Detect compute capabilities (build only for this GPU, not all)
                 try:
                     result = subprocess.run(
                         [
@@ -162,11 +178,12 @@ class LlamaCppBuilder:
                             )
                 except Exception:
                     pass
+                # Multi-threaded CUDA compilation
                 cmake_args.append("-DCMAKE_CUDA_FLAGS=--threads=0")
             else:
                 logger.info("No CUDA detected, building CPU-only llama.cpp")
 
-            # Use Ninja if available
+            # Use Ninja if available (faster than Make)
             generator_args = []
             if shutil.which("ninja"):
                 generator_args = ["-G", "Ninja"]
@@ -188,7 +205,7 @@ class LlamaCppBuilder:
                 capture_output = True,
             )
 
-            # Build
+            # Build only the targets we need (llama-server + llama-quantize)
             logger.info("Building llama.cpp (this may take a few minutes)...")
             subprocess.run(
                 [

From fc6110d491148308285e63b582d113442d8efd3e Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 16 Mar 2026 14:15:24 +0000
Subject: [PATCH 04/10] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .../core/inference/llama_cpp_builder.py       | 20 ++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/studio/backend/core/inference/llama_cpp_builder.py b/studio/backend/core/inference/llama_cpp_builder.py
index 5f57329003..c2c95fbd3b 100644
--- a/studio/backend/core/inference/llama_cpp_builder.py
+++ b/studio/backend/core/inference/llama_cpp_builder.py
@@ -123,21 +123,23 @@ class LlamaCppBuilder:
 
             # Build configuration: static binary, only needed targets, max parallelism
             cmake_args = [
-                "-DBUILD_SHARED_LIBS=OFF",     # Self-contained binary, no LD_LIBRARY_PATH needed
-                "-DGGML_NATIVE=ON",            # Native CPU optimizations
-                "-DLLAMA_BUILD_TESTS=OFF",     # Skip tests
+                "-DBUILD_SHARED_LIBS=OFF",  # Self-contained binary, no LD_LIBRARY_PATH needed
+                "-DGGML_NATIVE=ON",  # Native CPU optimizations
+                "-DLLAMA_BUILD_TESTS=OFF",  # Skip tests
                 "-DLLAMA_BUILD_EXAMPLES=OFF",  # Skip examples (we build server explicitly)
-                "-DLLAMA_BUILD_SERVER=ON",     # Ensure server target is available
+                "-DLLAMA_BUILD_SERVER=ON",  # Ensure server target is available
             ]
 
             # Use ccache if available (27x faster rebuilds)
             ccache = shutil.which("ccache")
             if ccache:
-                cmake_args.extend([
-                    f"-DCMAKE_C_COMPILER_LAUNCHER={ccache}",
-                    f"-DCMAKE_CXX_COMPILER_LAUNCHER={ccache}",
-                    f"-DCMAKE_CUDA_COMPILER_LAUNCHER={ccache}",
-                ])
+                cmake_args.extend(
+                    [
+                        f"-DCMAKE_C_COMPILER_LAUNCHER={ccache}",
+                        f"-DCMAKE_CXX_COMPILER_LAUNCHER={ccache}",
+                        f"-DCMAKE_CUDA_COMPILER_LAUNCHER={ccache}",
+                    ]
+                )
                 logger.info("Using ccache for faster compilation")
 
             # Detect CUDA

From de7ddc704ee27f53ca6034be98ebc59beef14117 Mon Sep 17 00:00:00 2001
From: Roland Tannous <115670425+rolandtannous@users.noreply.github.com>
Date: Mon, 16 Mar 2026 18:16:16 +0400
Subject: [PATCH 05/10] fix: Resolve CUDA toolkit mismatch on multi-CUDA
 Windows systems (#4324)

* fix: prefer existing CUDA_PATH toolkit to avoid version mismatch on multi-CUDA systems

* fix: validate GPU arch support before accepting CUDA toolkit (sm_120 + CUDA 12.4 fallback)

* debug: add temporary CUDA compatibility check print

* fix: auto-copy CUDA VS integration files when missing (No CUDA toolset found)

* fix: return false when nvcc --list-gpu-arch unavailable (reject old toolkit, scan for newer)

* fix: re-sanitize CUDA env vars before cmake build (survives Refresh-Environment)

* fix: use --list-gpu-code (sm_*) instead of --list-gpu-arch (compute_*) for arch probing
---
 studio/setup.ps1 | 192 ++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 175 insertions(+), 17 deletions(-)

diff --git a/studio/setup.ps1 b/studio/setup.ps1
index 1cf17b5d27..bc0105bee7 100644
--- a/studio/setup.ps1
+++ b/studio/setup.ps1
@@ -126,6 +126,42 @@ function Get-CudaComputeCapability {
     return $null
 }
 
+# Check if an nvcc binary supports a given sm_ architecture.
+# Uses `nvcc --list-gpu-code` which outputs sm_* tokens (--list-gpu-arch
+# outputs compute_* tokens instead).  Available since CUDA 11.6.
+# Returns $false if the flag isn't supported (old toolkit) — safer to reject
+# and fall back to scanning/PTX than to assume support and fail later.
+function Test-NvccArchSupport {
+    param([string]$NvccExe, [string]$Arch)
+    try {
+        $listCode = & $NvccExe --list-gpu-code 2>&1 | Out-String
+        if ($LASTEXITCODE -ne 0) { return $false }
+        return ($listCode -match "sm_$Arch")
+    } catch {
+        return $false
+    }
+}
+
+# Given an nvcc binary, return the highest sm_ architecture it supports.
+# Returns e.g. "90" for CUDA 12.4. Returns $null if detection fails.
+function Get-NvccMaxArch {
+    param([string]$NvccExe)
+    try {
+        $listCode = & $NvccExe --list-gpu-code 2>&1 | Out-String
+        if ($LASTEXITCODE -ne 0) { return $null }
+        $arches = @()
+        foreach ($line in $listCode -split "`n") {
+            if ($line.Trim() -match '^sm_(\d+)') {
+                $arches += [int]$Matches[1]
+            }
+        }
+        if ($arches.Count -gt 0) {
+            return ($arches | Sort-Object | Select-Object -Last 1).ToString()
+        }
+    } catch { }
+    return $null
+}
+
 # Detect driver's max CUDA version from nvidia-smi and return the highest
 # compatible PyTorch CUDA index tag (e.g. "cu128").
 # PyTorch on Windows ships CPU-only by default from PyPI; CUDA wheels live at
@@ -368,19 +404,76 @@ try {
     }
 } catch {}
 
-# -- Find a toolkit that's compatible with the driver --
+# Detect compute capability early so we can validate toolkit support
+$CudaArch = Get-CudaComputeCapability
+if ($CudaArch) {
+    Write-Host "   GPU Compute Capability = $($CudaArch.Insert($CudaArch.Length-1, '.')) (sm_$CudaArch)" -ForegroundColor Gray
+}
+
+# -- Find a toolkit that's compatible with the driver AND the GPU --
+# Strategy: prefer the toolkit at CUDA_PATH (user's existing setup) if it's
+# compatible with the driver AND supports the GPU architecture.  Only fall back
+# to scanning side-by-side installs if CUDA_PATH is missing, points to an
+# incompatible version, or can't compile for the GPU.  This avoids
+# header/binary mismatches when multiple toolkits are installed.
 $IncompatibleToolkit = $null
+$NvccPath = $null
+
 if ($DriverMaxCuda) {
-    $NvccPath = Find-Nvcc -MaxVersion $DriverMaxCuda
-    if ($NvccPath) {
-        Write-Host "   [OK] Found compatible CUDA Toolkit (nvcc: $NvccPath)" -ForegroundColor Green
-    } else {
-        # Check if there's an incompatible (too new) toolkit installed
-        $AnyNvcc = Find-Nvcc
-        if ($AnyNvcc) {
-            $NvccOut = & $AnyNvcc --version 2>&1 | Out-String
-            if ($NvccOut -match "release\s+([\d]+\.[\d]+)") {
-                $IncompatibleToolkit = $Matches[1]
+    $drMajorCuda = [int]$DriverMaxCuda.Split('.')[0]
+    $drMinorCuda = [int]$DriverMaxCuda.Split('.')[1]
+
+    # --- Step 1: Check existing CUDA_PATH first ---
+    $existingCudaPath = [Environment]::GetEnvironmentVariable('CUDA_PATH', 'Machine')
+    if (-not $existingCudaPath) {
+        $existingCudaPath = [Environment]::GetEnvironmentVariable('CUDA_PATH', 'User')
+    }
+    if ($existingCudaPath -and (Test-Path (Join-Path $existingCudaPath 'bin\nvcc.exe'))) {
+        $candidateNvcc = Join-Path $existingCudaPath 'bin\nvcc.exe'
+        $verOut = & $candidateNvcc --version 2>&1 | Out-String
+        if ($verOut -match 'release\s+(\d+)\.(\d+)') {
+            $tkMaj = [int]$Matches[1]; $tkMin = [int]$Matches[2]
+            $isCompat = ($tkMaj -lt $drMajorCuda) -or ($tkMaj -eq $drMajorCuda -and $tkMin -le $drMinorCuda)
+            if ($isCompat) {
+                # Also verify the toolkit supports our GPU architecture
+                Write-Host "   [DEBUG] Checking CUDA compatibility: toolkit=$tkMaj.$tkMin arch=sm_$CudaArch" -ForegroundColor Magenta
+                $archOk = $true
+                if ($CudaArch) {
+                    $archOk = Test-NvccArchSupport -NvccExe $candidateNvcc -Arch $CudaArch
+                    if (-not $archOk) {
+                        Write-Host "   [INFO] CUDA_PATH toolkit (CUDA $tkMaj.$tkMin) does not support GPU arch sm_$CudaArch" -ForegroundColor Yellow
+                        Write-Host "          Looking for a newer toolkit..." -ForegroundColor Yellow
+                    }
+                }
+                if ($archOk) {
+                    $NvccPath = $candidateNvcc
+                    Write-Host "   [OK] Using existing CUDA Toolkit at CUDA_PATH (nvcc: $NvccPath)" -ForegroundColor Green
+                }
+            } else {
+                Write-Host "   [INFO] CUDA_PATH ($existingCudaPath) has CUDA $tkMaj.$tkMin which exceeds driver max $DriverMaxCuda" -ForegroundColor Yellow
+            }
+        }
+    }
+
+    # --- Step 2: Fall back to scanning side-by-side installs ---
+    if (-not $NvccPath) {
+        $NvccPath = Find-Nvcc -MaxVersion $DriverMaxCuda
+        if ($NvccPath) {
+            Write-Host "   [OK] Found compatible CUDA Toolkit (nvcc: $NvccPath)" -ForegroundColor Green
+            if ($existingCudaPath) {
+                $selectedRoot = Split-Path (Split-Path $NvccPath -Parent) -Parent
+                if ($existingCudaPath.TrimEnd('\') -ne $selectedRoot.TrimEnd('\')) {
+                    Write-Host "   [INFO] Overriding CUDA_PATH from $existingCudaPath to $selectedRoot" -ForegroundColor Yellow
+                }
+            }
+        } else {
+            # Check if there's an incompatible (too new) toolkit installed
+            $AnyNvcc = Find-Nvcc
+            if ($AnyNvcc) {
+                $NvccOut = & $AnyNvcc --version 2>&1 | Out-String
+                if ($NvccOut -match "release\s+([\d]+\.[\d]+)") {
+                    $IncompatibleToolkit = $Matches[1]
+                }
             }
         }
     }
@@ -487,6 +580,19 @@ $CudaToolkitRoot = Split-Path (Split-Path $NvccPath -Parent) -Parent
 # in future sessions (overwrites any existing value pointing to a newer, incompatible version)
 [Environment]::SetEnvironmentVariable('CUDA_PATH', $CudaToolkitRoot, 'User')
 Write-Host "   Persisted CUDA_PATH=$CudaToolkitRoot to user environment" -ForegroundColor Gray
+# Clear all versioned CUDA_PATH_V* env vars in this process to prevent
+# cmake/MSBuild from discovering a conflicting CUDA installation.
+$cudaPathVars = @([Environment]::GetEnvironmentVariables('Process').Keys | Where-Object { $_ -match '^CUDA_PATH_V' })
+foreach ($v in $cudaPathVars) {
+    [Environment]::SetEnvironmentVariable($v, $null, 'Process')
+}
+# Set only the versioned var matching the selected toolkit (e.g. CUDA_PATH_V13_0)
+$tkDirName = Split-Path $CudaToolkitRoot -Leaf
+if ($tkDirName -match '^v(\d+)\.(\d+)') {
+    $cudaPathVerVar = "CUDA_PATH_V$($Matches[1])_$($Matches[2])"
+    [Environment]::SetEnvironmentVariable($cudaPathVerVar, $CudaToolkitRoot, 'Process')
+    Write-Host "   Set $cudaPathVerVar (cleared other CUDA_PATH_V* vars)" -ForegroundColor Gray
+}
 # Ensure nvcc's bin dir is on PATH for this process
 $nvccBinDir = Split-Path $NvccPath -Parent
 if ($env:PATH -notlike "*$nvccBinDir*") {
@@ -503,15 +609,38 @@ if (-not $userPath -or $userPath -notlike "*$nvccBinDir*") {
     Write-Host "   Persisted CUDA bin dir to user PATH" -ForegroundColor Gray
 }
 
+# -- Ensure CUDA ↔ Visual Studio integration files exist --
+# When CUDA is installed before VS Build Tools (or VS is reinstalled after CUDA),
+# the MSBuild .targets/.props files that let VS compile .cu files are missing.
+# cmake fails with "No CUDA toolset found". Fix: copy from CUDA extras dir.
+if ($VsInstallPath -and $CudaToolkitRoot) {
+    $vsCustomizations = Join-Path $VsInstallPath "MSBuild\Microsoft\VC\v170\BuildCustomizations"
+    $cudaExtras = Join-Path $CudaToolkitRoot "extras\visual_studio_integration\MSBuildExtensions"
+    if ((Test-Path $cudaExtras) -and (Test-Path $vsCustomizations)) {
+        $hasTargets = Get-ChildItem $vsCustomizations -Filter "CUDA *.targets" -ErrorAction SilentlyContinue
+        if (-not $hasTargets) {
+            Write-Host "   [INFO] CUDA VS integration missing -- copying .targets files..." -ForegroundColor Yellow
+            try {
+                Copy-Item "$cudaExtras\*" $vsCustomizations -Force -ErrorAction Stop
+                Write-Host "   [OK] CUDA VS integration files installed" -ForegroundColor Green
+            } catch {
+                Write-Host "   [WARN] Could not copy CUDA VS integration files (may need admin)" -ForegroundColor Yellow
+                Write-Host "          Manual fix: copy contents of" -ForegroundColor Yellow
+                Write-Host "            $cudaExtras" -ForegroundColor Cyan
+                Write-Host "          into:" -ForegroundColor Yellow
+                Write-Host "            $vsCustomizations" -ForegroundColor Cyan
+            }
+        }
+    }
+}
+
 Write-Host "[OK] CUDA Toolkit: $NvccPath" -ForegroundColor Green
 Write-Host "   CUDA_PATH      = $CudaToolkitRoot" -ForegroundColor Gray
 Write-Host "   CudaToolkitDir = $CudaToolkitRoot\" -ForegroundColor Gray
 
-# Detect compute capability (used later for llama.cpp cmake)
-$CudaArch = Get-CudaComputeCapability
-if ($CudaArch) {
-    Write-Host "   Compute Capability = $($CudaArch.Insert($CudaArch.Length-1, '.')) (sm_$CudaArch)" -ForegroundColor Gray
-} else {
+# $CudaArch was detected earlier (before toolkit selection) so it could
+# influence which toolkit we picked.  Just log the final state here.
+if (-not $CudaArch) {
     Write-Host "   [WARN] Could not detect compute capability -- cmake will use defaults" -ForegroundColor Yellow
 }
 
@@ -875,6 +1004,21 @@ if (Test-Path $LlamaServerBin) {
     $BuildOk = $true
     $FailedStep = ""
 
+    # Re-sanitize CUDA_PATH_V* vars — Refresh-Environment (called during
+    # Node/Python installs above) may have repopulated conflicting versioned
+    # vars from the Machine registry.
+    $cudaPathVars2 = @([Environment]::GetEnvironmentVariables('Process').Keys | Where-Object { $_ -match '^CUDA_PATH_V' })
+    foreach ($v2 in $cudaPathVars2) {
+        [Environment]::SetEnvironmentVariable($v2, $null, 'Process')
+    }
+    $tkDirName2 = Split-Path $CudaToolkitRoot -Leaf
+    if ($tkDirName2 -match '^v(\d+)\.(\d+)') {
+        [Environment]::SetEnvironmentVariable("CUDA_PATH_V$($Matches[1])_$($Matches[2])", $CudaToolkitRoot, 'Process')
+    }
+    # Also re-assert CUDA_PATH and CudaToolkitDir in case they were overwritten
+    [Environment]::SetEnvironmentVariable('CUDA_PATH', $CudaToolkitRoot, 'Process')
+    [Environment]::SetEnvironmentVariable('CudaToolkitDir', "$CudaToolkitRoot\", 'Process')
+
     # -- Step A: Clone or pull llama.cpp --
 
     if (Test-Path (Join-Path $LlamaCppDir ".git")) {
@@ -921,6 +1065,7 @@ if (Test-Path $LlamaServerBin) {
         # CUDA flags (Unsloth-aligned)
         $CmakeArgs += '-DGGML_CUDA=ON'
         $CmakeArgs += "-DCUDAToolkit_ROOT=$CudaToolkitRoot"
+        $CmakeArgs += "-DCUDA_TOOLKIT_ROOT_DIR=$CudaToolkitRoot"
         $CmakeArgs += "-DCMAKE_CUDA_COMPILER=$NvccPath"
         $CmakeArgs += '-DGGML_CUDA_FA_ALL_QUANTS=ON'
         $CmakeArgs += '-DGGML_CUDA_F16=OFF'
@@ -928,7 +1073,20 @@ if (Test-Path $LlamaServerBin) {
         $CmakeArgs += '-DGGML_CUDA_FORCE_CUBLAS=OFF'
         $CmakeArgs += '-DGGML_CUDA_PEER_MAX_BATCH_SIZE=8192'
         if ($CudaArch) {
-            $CmakeArgs += "-DCMAKE_CUDA_ARCHITECTURES=$CudaArch"
+            # Validate nvcc actually supports this architecture
+            if (Test-NvccArchSupport -NvccExe $NvccPath -Arch $CudaArch) {
+                $CmakeArgs += "-DCMAKE_CUDA_ARCHITECTURES=$CudaArch"
+            } else {
+                # GPU arch too new for this toolkit — fall back to highest supported.
+                # PTX forward-compatibility will JIT-compile for the actual GPU at runtime.
+                $maxArch = Get-NvccMaxArch -NvccExe $NvccPath
+                if ($maxArch) {
+                    $CmakeArgs += "-DCMAKE_CUDA_ARCHITECTURES=$maxArch"
+                    Write-Host "   [WARN] GPU is sm_$CudaArch but nvcc only supports up to sm_$maxArch" -ForegroundColor Yellow
+                    Write-Host "          Building with sm_$maxArch (PTX will JIT for your GPU at runtime)" -ForegroundColor Yellow
+                }
+                # else: omit flag entirely, let cmake pick defaults
+            }
         }
 
         cmake @CmakeArgs 2>&1 | Out-Null

From 2b7a787075eda95e5ad823a74bffaeb97ccb2fe8 Mon Sep 17 00:00:00 2001
From: Daniel Han <danielhanchen@gmail.com>
Date: Mon, 16 Mar 2026 14:21:28 +0000
Subject: [PATCH 06/10] studio: cross-platform llama.cpp builder (Linux, macOS,
 Windows)

- Linux: CUDA detection via /usr/local/cuda*, Ninja preferred
- macOS: Metal backend auto-enabled (llama.cpp default), no CUDA
- Windows: CUDA via CUDA_PATH env and toolkit dirs, VS generator
  fallback, binaries in build/bin/Release/, copy instead of symlink
- Reuse existing source (don't re-clone if CMakeLists.txt present)
- Both llama-server and llama-quantize verified and built
---
 .../core/inference/llama_cpp_builder.py       | 259 +++++++++++-------
 1 file changed, 153 insertions(+), 106 deletions(-)

diff --git a/studio/backend/core/inference/llama_cpp_builder.py b/studio/backend/core/inference/llama_cpp_builder.py
index c2c95fbd3b..e8bb7ffbe9 100644
--- a/studio/backend/core/inference/llama_cpp_builder.py
+++ b/studio/backend/core/inference/llama_cpp_builder.py
@@ -8,11 +8,14 @@ On server startup, if the llama-server binary is not found, a background
 thread clones and builds llama.cpp from source. Features that need
 llama.cpp (GGUF chat, export, AI Assist) can await `wait_for_ready()`
 which blocks until compilation finishes or fails.
+
+Cross-platform: Linux, macOS, Windows.
 """
 
 import os
 import shutil
 import subprocess
+import sys
 import threading
 from pathlib import Path
 from typing import Optional
@@ -22,6 +25,9 @@ from loggers import get_logger
 
 logger = get_logger(__name__)
 
+_IS_WIN = sys.platform == "win32"
+_IS_MAC = sys.platform == "darwin"
+
 
 class LlamaCppBuilder:
     """Manages background llama.cpp compilation."""
@@ -71,15 +77,75 @@ class LlamaCppBuilder:
         """Block until llama.cpp is ready. Returns True if ready, False on timeout."""
         return self._ready.wait(timeout = timeout)
 
+    # ── Platform helpers ─────────────────────────────────────────
+
+    @staticmethod
+    def _find_nvcc() -> Optional[str]:
+        """Find nvcc across platforms."""
+        nvcc = shutil.which("nvcc")
+        if nvcc:
+            return nvcc
+
+        if _IS_WIN:
+            # Windows: check CUDA_PATH env, then standard install dirs
+            cuda_path = os.environ.get("CUDA_PATH", "")
+            if cuda_path:
+                candidate = Path(cuda_path) / "bin" / "nvcc.exe"
+                if candidate.is_file():
+                    return str(candidate)
+            toolkit_base = Path(r"C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA")
+            if toolkit_base.is_dir():
+                for d in sorted(toolkit_base.iterdir(), reverse = True):
+                    candidate = d / "bin" / "nvcc.exe"
+                    if candidate.is_file():
+                        return str(candidate)
+        else:
+            # Linux: check standard locations
+            if Path("/usr/local/cuda/bin/nvcc").is_file():
+                return "/usr/local/cuda/bin/nvcc"
+            for cuda_dir in sorted(
+                Path("/usr/local").glob("cuda-*/bin/nvcc"), reverse = True
+            ):
+                return str(cuda_dir)
+
+        return None
+
+    @staticmethod
+    def _detect_cuda_architectures() -> Optional[str]:
+        """Detect GPU compute capabilities via nvidia-smi."""
+        nvidia_smi = "nvidia-smi"
+        if _IS_WIN:
+            # nvidia-smi is typically in System32 on Windows
+            win_path = Path(r"C:\Windows\System32\nvidia-smi.exe")
+            if win_path.is_file():
+                nvidia_smi = str(win_path)
+
+        try:
+            result = subprocess.run(
+                [nvidia_smi, "--query-gpu=compute_cap", "--format=csv,noheader"],
+                capture_output = True,
+                text = True,
+                timeout = 10,
+            )
+            if result.returncode == 0:
+                caps = set()
+                for line in result.stdout.strip().split("\n"):
+                    cap = line.strip().replace(".", "")
+                    if cap:
+                        caps.add(cap)
+                if caps:
+                    return ";".join(sorted(caps))
+        except Exception:
+            pass
+        return None
+
+    # ── Build ────────────────────────────────────────────────────
+
     def _build(self) -> None:
         """Clone and build llama.cpp in the background."""
         try:
-            import sys
-
             llama_dir = Path.home() / ".unsloth" / "llama.cpp"
-            binary_name = (
-                "llama-server.exe" if sys.platform == "win32" else "llama-server"
-            )
+            binary_name = "llama-server.exe" if _IS_WIN else "llama-server"
 
             # Don't rebuild if binary appeared while we were starting
             from core.inference.llama_cpp import LlamaCppBackend
@@ -103,34 +169,31 @@ class LlamaCppBuilder:
                 logger.warning(self._error)
                 return
 
-            # Clone
-            if llama_dir.exists():
-                shutil.rmtree(llama_dir, ignore_errors = True)
+            # Clone (don't delete if already exists -- might be a partial build)
+            if not (llama_dir / "CMakeLists.txt").is_file():
+                if llama_dir.exists():
+                    shutil.rmtree(llama_dir, ignore_errors = True)
 
-            logger.info("Cloning llama.cpp...")
-            subprocess.run(
-                [
-                    "git",
-                    "clone",
-                    "--depth",
-                    "1",
-                    "https://github.com/ggml-org/llama.cpp.git",
-                    str(llama_dir),
-                ],
-                check = True,
-                capture_output = True,
-            )
+                logger.info("Cloning llama.cpp...")
+                subprocess.run(
+                    ["git", "clone", "--depth", "1",
+                     "https://github.com/ggml-org/llama.cpp.git", str(llama_dir)],
+                    check = True,
+                    capture_output = True,
+                )
+            else:
+                logger.info("llama.cpp source already present, building...")
 
-            # Build configuration: static binary, only needed targets, max parallelism
+            # ── CMake arguments ──────────────────────────────────
             cmake_args = [
-                "-DBUILD_SHARED_LIBS=OFF",  # Self-contained binary, no LD_LIBRARY_PATH needed
+                "-DBUILD_SHARED_LIBS=OFF",  # Self-contained binary
                 "-DGGML_NATIVE=ON",  # Native CPU optimizations
                 "-DLLAMA_BUILD_TESTS=OFF",  # Skip tests
-                "-DLLAMA_BUILD_EXAMPLES=OFF",  # Skip examples (we build server explicitly)
-                "-DLLAMA_BUILD_SERVER=ON",  # Ensure server target is available
+                "-DLLAMA_BUILD_EXAMPLES=OFF",  # Skip examples
+                "-DLLAMA_BUILD_SERVER=ON",  # Ensure server target
             ]
 
-            # Use ccache if available (27x faster rebuilds)
+            # ccache (27x faster rebuilds when available)
             ccache = shutil.which("ccache")
             if ccache:
                 cmake_args.extend(
@@ -142,109 +205,93 @@ class LlamaCppBuilder:
                 )
                 logger.info("Using ccache for faster compilation")
 
-            # Detect CUDA
-            nvcc = shutil.which("nvcc")
-            if not nvcc:
-                for cuda_dir in sorted(
-                    Path("/usr/local").glob("cuda-*/bin/nvcc"), reverse = True
-                ):
-                    nvcc = str(cuda_dir)
-                    break
-            if not nvcc and Path("/usr/local/cuda/bin/nvcc").is_file():
-                nvcc = "/usr/local/cuda/bin/nvcc"
-
-            if nvcc:
-                logger.info(f"CUDA detected: {nvcc}")
-                cmake_args.append("-DGGML_CUDA=ON")
-                # Detect compute capabilities (build only for this GPU, not all)
-                try:
-                    result = subprocess.run(
-                        [
-                            "nvidia-smi",
-                            "--query-gpu=compute_cap",
-                            "--format=csv,noheader",
-                        ],
-                        capture_output = True,
-                        text = True,
-                        timeout = 10,
-                    )
-                    if result.returncode == 0:
-                        caps = set()
-                        for line in result.stdout.strip().split("\n"):
-                            cap = line.strip().replace(".", "")
-                            if cap:
-                                caps.add(cap)
-                        if caps:
-                            cmake_args.append(
-                                f"-DCMAKE_CUDA_ARCHITECTURES={';'.join(sorted(caps))}"
-                            )
-                except Exception:
-                    pass
-                # Multi-threaded CUDA compilation
-                cmake_args.append("-DCMAKE_CUDA_FLAGS=--threads=0")
+            # ── GPU backend ──────────────────────────────────────
+            if _IS_MAC:
+                # macOS: Metal is enabled by default in llama.cpp, no extra flags needed
+                logger.info("macOS detected, Metal backend enabled by default")
             else:
-                logger.info("No CUDA detected, building CPU-only llama.cpp")
+                nvcc = self._find_nvcc()
+                if nvcc:
+                    logger.info(f"CUDA detected: {nvcc}")
+                    cmake_args.append("-DGGML_CUDA=ON")
+                    # Build only for the detected GPU architecture(s)
+                    archs = self._detect_cuda_architectures()
+                    if archs:
+                        cmake_args.append(f"-DCMAKE_CUDA_ARCHITECTURES={archs}")
+                    # Multi-threaded CUDA compilation
+                    cmake_args.append("-DCMAKE_CUDA_FLAGS=--threads=0")
+                else:
+                    logger.info("No CUDA detected, building CPU-only llama.cpp")
 
-            # Use Ninja if available (faster than Make)
+            # ── Generator ────────────────────────────────────────
             generator_args = []
-            if shutil.which("ninja"):
-                generator_args = ["-G", "Ninja"]
+            if _IS_WIN:
+                # Windows: prefer Ninja, fall back to VS generator
+                if shutil.which("ninja"):
+                    generator_args = ["-G", "Ninja"]
+                # else: cmake will use default VS generator
+            else:
+                # Linux/Mac: prefer Ninja
+                if shutil.which("ninja"):
+                    generator_args = ["-G", "Ninja"]
 
-            # Configure
+            # ── Configure ────────────────────────────────────────
             build_dir = llama_dir / "build"
             logger.info("Configuring llama.cpp build...")
             subprocess.run(
-                [
-                    "cmake",
-                    "-B",
-                    str(build_dir),
-                    "-S",
-                    str(llama_dir),
-                ]
+                ["cmake", "-B", str(build_dir), "-S", str(llama_dir)]
                 + generator_args
                 + cmake_args,
                 check = True,
                 capture_output = True,
             )
 
-            # Build only the targets we need (llama-server + llama-quantize)
+            # ── Build ────────────────────────────────────────────
             logger.info("Building llama.cpp (this may take a few minutes)...")
             subprocess.run(
-                [
-                    "cmake",
-                    "--build",
-                    str(build_dir),
-                    "--config",
-                    "Release",
-                    "--target",
-                    "llama-server",
-                    "llama-quantize",
-                    "-j",
-                ],
+                ["cmake", "--build", str(build_dir), "--config", "Release",
+                 "--target", "llama-server", "llama-quantize", "-j"],
                 check = True,
                 capture_output = True,
             )
 
-            # Symlink llama-quantize for unsloth-zoo's check_llama_cpp()
-            quantize_src = build_dir / "bin" / "llama-quantize"
-            quantize_link = llama_dir / "llama-quantize"
-            if quantize_src.is_file() and not quantize_link.exists():
-                try:
-                    quantize_link.symlink_to(quantize_src)
-                except Exception:
-                    pass
-
-            # Verify
-            expected = build_dir / "bin" / binary_name
-            if expected.is_file():
-                self._binary_path = str(expected)
-                logger.info(f"llama.cpp build complete: {self._binary_path}")
+            # ── Verify binaries ──────────────────────────────────
+            # Windows VS generator puts binaries in build/bin/Release/
+            if _IS_WIN:
+                bin_dir = build_dir / "bin" / "Release"
+                if not (bin_dir / binary_name).is_file():
+                    bin_dir = build_dir / "bin"  # Ninja puts them here
             else:
-                self._error = f"Build completed but binary not found at {expected}"
+                bin_dir = build_dir / "bin"
+
+            server_bin = bin_dir / binary_name
+            quantize_name = "llama-quantize.exe" if _IS_WIN else "llama-quantize"
+            quantize_bin = bin_dir / quantize_name
+
+            if server_bin.is_file():
+                self._binary_path = str(server_bin)
+                logger.info(f"llama-server built: {self._binary_path}")
+            else:
+                self._error = f"Build completed but llama-server not found at {server_bin}"
                 logger.error(self._error)
 
+            if quantize_bin.is_file():
+                logger.info(f"llama-quantize built: {quantize_bin}")
+                # Create symlink/copy for unsloth-zoo's check_llama_cpp()
+                quantize_link = llama_dir / quantize_name
+                if not quantize_link.exists():
+                    try:
+                        if _IS_WIN:
+                            shutil.copy2(str(quantize_bin), str(quantize_link))
+                        else:
+                            quantize_link.symlink_to(quantize_bin)
+                    except Exception:
+                        pass
+            else:
+                logger.warning(f"llama-quantize not found at {quantize_bin}")
+
         except subprocess.CalledProcessError as e:
-            stderr = e.stderr.decode() if e.stderr else ""
+            stderr = (e.stderr.decode() if isinstance(e.stderr, bytes) else e.stderr) or ""
             self._error = f"llama.cpp build failed: {stderr[-500:]}"
             logger.error(self._error)
         except Exception as e:

From 251e3d20117777f192e54db5f20511e9ef82ce0c Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 16 Mar 2026 14:22:04 +0000
Subject: [PATCH 07/10] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .../core/inference/llama_cpp_builder.py       | 31 +++++++++++++++----
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/studio/backend/core/inference/llama_cpp_builder.py b/studio/backend/core/inference/llama_cpp_builder.py
index e8bb7ffbe9..c4ae81c7d8 100644
--- a/studio/backend/core/inference/llama_cpp_builder.py
+++ b/studio/backend/core/inference/llama_cpp_builder.py
@@ -176,8 +176,14 @@ class LlamaCppBuilder:
 
                 logger.info("Cloning llama.cpp...")
                 subprocess.run(
-                    ["git", "clone", "--depth", "1",
-                     "https://github.com/ggml-org/llama.cpp.git", str(llama_dir)],
+                    [
+                        "git",
+                        "clone",
+                        "--depth",
+                        "1",
+                        "https://github.com/ggml-org/llama.cpp.git",
+                        str(llama_dir),
+                    ],
                     check = True,
                     capture_output = True,
                 )
@@ -249,8 +255,17 @@ class LlamaCppBuilder:
             # ── Build ────────────────────────────────────────────
             logger.info("Building llama.cpp (this may take a few minutes)...")
             subprocess.run(
-                ["cmake", "--build", str(build_dir), "--config", "Release",
-                 "--target", "llama-server", "llama-quantize", "-j"],
+                [
+                    "cmake",
+                    "--build",
+                    str(build_dir),
+                    "--config",
+                    "Release",
+                    "--target",
+                    "llama-server",
+                    "llama-quantize",
+                    "-j",
+                ],
                 check = True,
                 capture_output = True,
             )
@@ -272,7 +287,9 @@ class LlamaCppBuilder:
                 self._binary_path = str(server_bin)
                 logger.info(f"llama-server built: {self._binary_path}")
             else:
-                self._error = f"Build completed but llama-server not found at {server_bin}"
+                self._error = (
+                    f"Build completed but llama-server not found at {server_bin}"
+                )
                 logger.error(self._error)
 
             if quantize_bin.is_file():
@@ -291,7 +308,9 @@ class LlamaCppBuilder:
                 logger.warning(f"llama-quantize not found at {quantize_bin}")
 
         except subprocess.CalledProcessError as e:
-            stderr = (e.stderr.decode() if isinstance(e.stderr, bytes) else e.stderr) or ""
+            stderr = (
+                e.stderr.decode() if isinstance(e.stderr, bytes) else e.stderr
+            ) or ""
             self._error = f"llama.cpp build failed: {stderr[-500:]}"
             logger.error(self._error)
         except Exception as e:

From eb15b31e3efe14d28cd202294dac82adff3fd356 Mon Sep 17 00:00:00 2001
From: Daniel Han <danielhanchen@gmail.com>
Date: Mon, 16 Mar 2026 15:03:58 +0000
Subject: [PATCH 08/10] studio: improve training progress messages for large
 datasets

Add granular status updates through the full preprocessing pipeline:
- "Downloading dataset: Open-Orca/OpenOrca..." before HF download
- "Downloaded Open-Orca/OpenOrca (4,233,923 rows)" after download
- "Formatting dataset (4,233,923 rows)..." before format step
- "Applying chat template to chatml_conversations (4,233,923 rows)..."
- "Dataset ready (4,233,923 samples, chatml_conversations format)"

Shows detected format name and row counts at each stage so users
can see progress through large dataset preprocessing instead of
a static "Loading and formatting dataset..." for minutes.
---
 studio/backend/core/training/trainer.py        | 12 +++++++++---
 studio/backend/utils/datasets/dataset_utils.py |  3 ++-
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py
index af20146a7f..b3282ef1d9 100644
--- a/studio/backend/core/training/trainer.py
+++ b/studio/backend/core/training/trainer.py
@@ -2345,6 +2345,9 @@ class UnslothTrainer:
                         status_message = f"Streamed {len(dataset)} rows from HuggingFace"
                     )
                 else:
+                    self._update_progress(
+                        status_message = f"Downloading dataset: {dataset_source}..."
+                    )
                     dataset = load_dataset(**load_kwargs)
 
                 # Check if stopped during dataset loading
@@ -2354,7 +2357,7 @@ class UnslothTrainer:
 
                 n_rows = len(dataset) if hasattr(dataset, "__len__") else 0
                 self._update_progress(
-                    status_message = f"Loaded dataset from HuggingFace: {dataset_source} ({n_rows:,} rows)"
+                    status_message = f"Downloaded {dataset_source} ({n_rows:,} rows)"
                 )
                 logger.info(
                     f"Loaded dataset from Hugging Face: {dataset_source} ({n_rows:,} rows)\n"
@@ -2482,10 +2485,13 @@ class UnslothTrainer:
                 self._update_progress(error = error_msg)
                 return None
 
+            detected = dataset_info.get("detected_format", "unknown")
+            final_ds = dataset_info.get("dataset")
+            final_n = len(final_ds) if hasattr(final_ds, "__len__") else "?"
             self._update_progress(
-                status_message = f"Dataset formatted and ready for training"
+                status_message = f"Dataset ready ({final_n:,} samples, {detected} format)"
             )
-            logger.info(f"Dataset formatted successfully\n")
+            logger.info(f"Dataset formatted successfully ({final_n} samples, {detected})\n")
 
             # ========== THEN SPLIT ==========
             if has_separate_eval_source and eval_dataset is not None:
diff --git a/studio/backend/utils/datasets/dataset_utils.py b/studio/backend/utils/datasets/dataset_utils.py
index 0808509091..d4876b6fb8 100644
--- a/studio/backend/utils/datasets/dataset_utils.py
+++ b/studio/backend/utils/datasets/dataset_utils.py
@@ -1109,9 +1109,10 @@ def format_and_template_dataset(
         )
 
         # Step 2: Apply chat template
+        detected = dataset_info.get("detected_format", "unknown")
         if progress_callback and n_rows:
             progress_callback(
-                status_message = f"Applying chat template ({n_rows:,} rows)..."
+                status_message = f"Applying chat template to {detected} ({n_rows:,} rows)..."
             )
         # Gemma emits a leading <bos> that must be stripped for text-only chatml/sharegpt.
         is_alpaca = format_type == "alpaca" or (

From 12bc242ca275acd95fdf58a4973eaa39840639a3 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 16 Mar 2026 15:04:07 +0000
Subject: [PATCH 09/10] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 studio/backend/core/training/trainer.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py
index b3282ef1d9..330fb74747 100644
--- a/studio/backend/core/training/trainer.py
+++ b/studio/backend/core/training/trainer.py
@@ -2491,7 +2491,9 @@ class UnslothTrainer:
             self._update_progress(
                 status_message = f"Dataset ready ({final_n:,} samples, {detected} format)"
             )
-            logger.info(f"Dataset formatted successfully ({final_n} samples, {detected})\n")
+            logger.info(
+                f"Dataset formatted successfully ({final_n} samples, {detected})\n"
+            )
 
             # ========== THEN SPLIT ==========
             if has_separate_eval_source and eval_dataset is not None:

From 67f0ab592ecf918e0a8b7ea3661fbf70e063dcc8 Mon Sep 17 00:00:00 2001
From: Daniel Han <danielhanchen@gmail.com>
Date: Tue, 17 Mar 2026 01:12:55 +0000
Subject: [PATCH 10/10] studio: add CUDA lib paths to LD_LIBRARY_PATH for
 llama-server

When llama-server is built with shared libs (setup.sh default),
it needs libcudart.so.12 and other CUDA runtime libs. Add
/usr/local/cuda/lib64 and targets path to LD_LIBRARY_PATH so
the server starts correctly even when CUDA isn't on the system path.
---
 studio/backend/core/inference/llama_cpp.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 3a137b8b63..fe7c045df1 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -888,9 +888,18 @@ class LlamaCppBackend:
                 env["PATH"] = ";".join(path_dirs) + ";" + existing_path
             else:
                 # Linux: set LD_LIBRARY_PATH for shared libs next to the binary
+                # and CUDA runtime libs (libcudart, libcublas, etc.)
+                lib_dirs = [binary_dir]
+                for cuda_lib in [
+                    "/usr/local/cuda/lib64",
+                    "/usr/local/cuda/targets/x86_64-linux/lib",
+                ]:
+                    if os.path.isdir(cuda_lib):
+                        lib_dirs.append(cuda_lib)
                 existing_ld = env.get("LD_LIBRARY_PATH", "")
+                new_ld = ":".join(lib_dirs)
                 env["LD_LIBRARY_PATH"] = (
-                    f"{binary_dir}:{existing_ld}" if existing_ld else binary_dir
+                    f"{new_ld}:{existing_ld}" if existing_ld else new_ld
                 )
 
             # Pin to selected GPU(s) via CUDA_VISIBLE_DEVICES