diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 3a137b8b63..fe7c045df1 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -888,9 +888,18 @@ class LlamaCppBackend: env["PATH"] = ";".join(path_dirs) + ";" + existing_path else: # Linux: set LD_LIBRARY_PATH for shared libs next to the binary + # and CUDA runtime libs (libcudart, libcublas, etc.) + lib_dirs = [binary_dir] + for cuda_lib in [ + "/usr/local/cuda/lib64", + "/usr/local/cuda/targets/x86_64-linux/lib", + ]: + if os.path.isdir(cuda_lib): + lib_dirs.append(cuda_lib) existing_ld = env.get("LD_LIBRARY_PATH", "") + new_ld = ":".join(lib_dirs) env["LD_LIBRARY_PATH"] = ( - f"{binary_dir}:{existing_ld}" if existing_ld else binary_dir + f"{new_ld}:{existing_ld}" if existing_ld else new_ld ) # Pin to selected GPU(s) via CUDA_VISIBLE_DEVICES diff --git a/studio/backend/core/inference/llama_cpp_builder.py b/studio/backend/core/inference/llama_cpp_builder.py new file mode 100644 index 0000000000..c4ae81c7d8 --- /dev/null +++ b/studio/backend/core/inference/llama_cpp_builder.py @@ -0,0 +1,333 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +""" +Background llama.cpp compilation. + +On server startup, if the llama-server binary is not found, a background +thread clones and builds llama.cpp from source. Features that need +llama.cpp (GGUF chat, export, AI Assist) can await `wait_for_ready()` +which blocks until compilation finishes or fails. + +Cross-platform: Linux, macOS, Windows. +""" + +import os +import shutil +import subprocess +import sys +import threading +from pathlib import Path +from typing import Optional + +import structlog +from loggers import get_logger + +logger = get_logger(__name__) + +_IS_WIN = sys.platform == "win32" +_IS_MAC = sys.platform == "darwin" + + +class LlamaCppBuilder: + """Manages background llama.cpp compilation.""" + + def __init__(self): + self._ready = threading.Event() + self._building = False + self._error: Optional[str] = None + self._thread: Optional[threading.Thread] = None + self._binary_path: Optional[str] = None + + @property + def is_ready(self) -> bool: + return self._ready.is_set() + + @property + def is_building(self) -> bool: + return self._building + + @property + def error(self) -> Optional[str]: + return self._error + + @property + def binary_path(self) -> Optional[str]: + return self._binary_path + + def check_and_build(self) -> None: + """Check if llama-server exists. If not, start background build.""" + from core.inference.llama_cpp import LlamaCppBackend + + binary = LlamaCppBackend._find_llama_server_binary() + if binary: + self._binary_path = binary + self._ready.set() + logger.info(f"llama-server binary found: {binary}") + return + + logger.info("llama-server binary not found, starting background build...") + self._building = True + self._thread = threading.Thread( + target = self._build, daemon = True, name = "llama-cpp-build" + ) + self._thread.start() + + def wait_for_ready(self, timeout: Optional[float] = None) -> bool: + """Block until llama.cpp is ready. Returns True if ready, False on timeout.""" + return self._ready.wait(timeout = timeout) + + # ── Platform helpers ───────────────────────────────────────── + + @staticmethod + def _find_nvcc() -> Optional[str]: + """Find nvcc across platforms.""" + nvcc = shutil.which("nvcc") + if nvcc: + return nvcc + + if _IS_WIN: + # Windows: check CUDA_PATH env, then standard install dirs + cuda_path = os.environ.get("CUDA_PATH", "") + if cuda_path: + candidate = Path(cuda_path) / "bin" / "nvcc.exe" + if candidate.is_file(): + return str(candidate) + toolkit_base = Path(r"C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA") + if toolkit_base.is_dir(): + for d in sorted(toolkit_base.iterdir(), reverse = True): + candidate = d / "bin" / "nvcc.exe" + if candidate.is_file(): + return str(candidate) + else: + # Linux: check standard locations + if Path("/usr/local/cuda/bin/nvcc").is_file(): + return "/usr/local/cuda/bin/nvcc" + for cuda_dir in sorted( + Path("/usr/local").glob("cuda-*/bin/nvcc"), reverse = True + ): + return str(cuda_dir) + + return None + + @staticmethod + def _detect_cuda_architectures() -> Optional[str]: + """Detect GPU compute capabilities via nvidia-smi.""" + nvidia_smi = "nvidia-smi" + if _IS_WIN: + # nvidia-smi is typically in System32 on Windows + win_path = Path(r"C:\Windows\System32\nvidia-smi.exe") + if win_path.is_file(): + nvidia_smi = str(win_path) + + try: + result = subprocess.run( + [nvidia_smi, "--query-gpu=compute_cap", "--format=csv,noheader"], + capture_output = True, + text = True, + timeout = 10, + ) + if result.returncode == 0: + caps = set() + for line in result.stdout.strip().split("\n"): + cap = line.strip().replace(".", "") + if cap: + caps.add(cap) + if caps: + return ";".join(sorted(caps)) + except Exception: + pass + return None + + # ── Build ──────────────────────────────────────────────────── + + def _build(self) -> None: + """Clone and build llama.cpp in the background.""" + try: + llama_dir = Path.home() / ".unsloth" / "llama.cpp" + binary_name = "llama-server.exe" if _IS_WIN else "llama-server" + + # Don't rebuild if binary appeared while we were starting + from core.inference.llama_cpp import LlamaCppBackend + + binary = LlamaCppBackend._find_llama_server_binary() + if binary: + self._binary_path = binary + self._building = False + self._ready.set() + return + + # Check prerequisites + if not shutil.which("cmake"): + self._error = "cmake not found. Install cmake to enable GGUF inference." + self._building = False + logger.warning(self._error) + return + if not shutil.which("git"): + self._error = "git not found. Install git to enable GGUF inference." + self._building = False + logger.warning(self._error) + return + + # Clone (don't delete if already exists -- might be a partial build) + if not (llama_dir / "CMakeLists.txt").is_file(): + if llama_dir.exists(): + shutil.rmtree(llama_dir, ignore_errors = True) + + logger.info("Cloning llama.cpp...") + subprocess.run( + [ + "git", + "clone", + "--depth", + "1", + "https://github.com/ggml-org/llama.cpp.git", + str(llama_dir), + ], + check = True, + capture_output = True, + ) + else: + logger.info("llama.cpp source already present, building...") + + # ── CMake arguments ────────────────────────────────── + cmake_args = [ + "-DBUILD_SHARED_LIBS=OFF", # Self-contained binary + "-DGGML_NATIVE=ON", # Native CPU optimizations + "-DLLAMA_BUILD_TESTS=OFF", # Skip tests + "-DLLAMA_BUILD_EXAMPLES=OFF", # Skip examples + "-DLLAMA_BUILD_SERVER=ON", # Ensure server target + ] + + # ccache (27x faster rebuilds when available) + ccache = shutil.which("ccache") + if ccache: + cmake_args.extend( + [ + f"-DCMAKE_C_COMPILER_LAUNCHER={ccache}", + f"-DCMAKE_CXX_COMPILER_LAUNCHER={ccache}", + f"-DCMAKE_CUDA_COMPILER_LAUNCHER={ccache}", + ] + ) + logger.info("Using ccache for faster compilation") + + # ── GPU backend ────────────────────────────────────── + if _IS_MAC: + # macOS: Metal is enabled by default in llama.cpp, no extra flags needed + logger.info("macOS detected, Metal backend enabled by default") + else: + nvcc = self._find_nvcc() + if nvcc: + logger.info(f"CUDA detected: {nvcc}") + cmake_args.append("-DGGML_CUDA=ON") + # Build only for the detected GPU architecture(s) + archs = self._detect_cuda_architectures() + if archs: + cmake_args.append(f"-DCMAKE_CUDA_ARCHITECTURES={archs}") + # Multi-threaded CUDA compilation + cmake_args.append("-DCMAKE_CUDA_FLAGS=--threads=0") + else: + logger.info("No CUDA detected, building CPU-only llama.cpp") + + # ── Generator ──────────────────────────────────────── + generator_args = [] + if _IS_WIN: + # Windows: prefer Ninja, fall back to VS generator + if shutil.which("ninja"): + generator_args = ["-G", "Ninja"] + # else: cmake will use default VS generator + else: + # Linux/Mac: prefer Ninja + if shutil.which("ninja"): + generator_args = ["-G", "Ninja"] + + # ── Configure ──────────────────────────────────────── + build_dir = llama_dir / "build" + logger.info("Configuring llama.cpp build...") + subprocess.run( + ["cmake", "-B", str(build_dir), "-S", str(llama_dir)] + + generator_args + + cmake_args, + check = True, + capture_output = True, + ) + + # ── Build ──────────────────────────────────────────── + logger.info("Building llama.cpp (this may take a few minutes)...") + subprocess.run( + [ + "cmake", + "--build", + str(build_dir), + "--config", + "Release", + "--target", + "llama-server", + "llama-quantize", + "-j", + ], + check = True, + capture_output = True, + ) + + # ── Verify binaries ────────────────────────────────── + # Windows VS generator puts binaries in build/bin/Release/ + if _IS_WIN: + bin_dir = build_dir / "bin" / "Release" + if not (bin_dir / binary_name).is_file(): + bin_dir = build_dir / "bin" # Ninja puts them here + else: + bin_dir = build_dir / "bin" + + server_bin = bin_dir / binary_name + quantize_name = "llama-quantize.exe" if _IS_WIN else "llama-quantize" + quantize_bin = bin_dir / quantize_name + + if server_bin.is_file(): + self._binary_path = str(server_bin) + logger.info(f"llama-server built: {self._binary_path}") + else: + self._error = ( + f"Build completed but llama-server not found at {server_bin}" + ) + logger.error(self._error) + + if quantize_bin.is_file(): + logger.info(f"llama-quantize built: {quantize_bin}") + # Create symlink/copy for unsloth-zoo's check_llama_cpp() + quantize_link = llama_dir / quantize_name + if not quantize_link.exists(): + try: + if _IS_WIN: + shutil.copy2(str(quantize_bin), str(quantize_link)) + else: + quantize_link.symlink_to(quantize_bin) + except Exception: + pass + else: + logger.warning(f"llama-quantize not found at {quantize_bin}") + + except subprocess.CalledProcessError as e: + stderr = ( + e.stderr.decode() if isinstance(e.stderr, bytes) else e.stderr + ) or "" + self._error = f"llama.cpp build failed: {stderr[-500:]}" + logger.error(self._error) + except Exception as e: + self._error = f"llama.cpp build error: {e}" + logger.error(self._error) + finally: + self._building = False + self._ready.set() # Unblock waiters even on failure + + +# ── Singleton ──────────────────────────────────────────────────── + +_builder: Optional[LlamaCppBuilder] = None + + +def get_llama_cpp_builder() -> LlamaCppBuilder: + global _builder + if _builder is None: + _builder = LlamaCppBuilder() + return _builder diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py index ecc11c30d5..330fb74747 100644 --- a/studio/backend/core/training/trainer.py +++ b/studio/backend/core/training/trainer.py @@ -2345,6 +2345,9 @@ class UnslothTrainer: status_message = f"Streamed {len(dataset)} rows from HuggingFace" ) else: + self._update_progress( + status_message = f"Downloading dataset: {dataset_source}..." + ) dataset = load_dataset(**load_kwargs) # Check if stopped during dataset loading @@ -2352,11 +2355,12 @@ class UnslothTrainer: logger.info("Stopped during dataset loading\n") return None + n_rows = len(dataset) if hasattr(dataset, "__len__") else 0 self._update_progress( - status_message = f"Loaded dataset from HuggingFace: {dataset_source}" + status_message = f"Downloaded {dataset_source} ({n_rows:,} rows)" ) logger.info( - f"Loaded dataset from Hugging Face: {dataset_source} ({len(dataset)} rows)\n" + f"Loaded dataset from Hugging Face: {dataset_source} ({n_rows:,} rows)\n" ) # Resolve eval split from a separate HF split (explicit or auto-detected) @@ -2481,10 +2485,15 @@ class UnslothTrainer: self._update_progress(error = error_msg) return None + detected = dataset_info.get("detected_format", "unknown") + final_ds = dataset_info.get("dataset") + final_n = len(final_ds) if hasattr(final_ds, "__len__") else "?" self._update_progress( - status_message = f"Dataset formatted and ready for training" + status_message = f"Dataset ready ({final_n:,} samples, {detected} format)" + ) + logger.info( + f"Dataset formatted successfully ({final_n} samples, {detected})\n" ) - logger.info(f"Dataset formatted successfully\n") # ========== THEN SPLIT ========== if has_separate_eval_source and eval_dataset is not None: diff --git a/studio/backend/main.py b/studio/backend/main.py index 3ab846306e..4495c99e2f 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -75,6 +75,12 @@ async def lifespan(app: FastAPI): threading.Thread(target = _precache, daemon = True).start() + # Build llama.cpp in the background if the binary is missing. + # Studio starts immediately; GGUF features wait for the build to finish. + from core.inference.llama_cpp_builder import get_llama_cpp_builder + + get_llama_cpp_builder().check_and_build() + if storage.ensure_default_admin(): bootstrap_pw = storage.get_bootstrap_password() app.state.bootstrap_password = bootstrap_pw diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index e93ba1d6d6..75e6472d18 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -111,6 +111,20 @@ async def load_model( # ── GGUF path: load via llama-server ────────────────────── if config.is_gguf: + # Wait for llama.cpp to be compiled if a background build is in progress + from core.inference.llama_cpp_builder import get_llama_cpp_builder + + builder = get_llama_cpp_builder() + if not builder.is_ready: + logger.info("Waiting for llama.cpp build to finish...") + ready = await asyncio.to_thread(builder.wait_for_ready, timeout = 600) + if not ready or builder.error: + raise HTTPException( + status_code = 503, + detail = builder.error + or "llama.cpp is still compiling. Please wait.", + ) + llama_backend = get_llama_cpp_backend() unsloth_backend = get_inference_backend() @@ -538,6 +552,19 @@ async def get_status( raise HTTPException(status_code = 500, detail = f"Failed to get status: {str(e)}") +@router.get("/llama-cpp-status") +async def llama_cpp_status(): + """Check if llama.cpp is ready (built and available).""" + from core.inference.llama_cpp_builder import get_llama_cpp_builder + + builder = get_llama_cpp_builder() + return { + "ready": builder.is_ready and builder.error is None, + "building": builder.is_building, + "error": builder.error, + } + + # ===================================================================== # Audio (TTS) Generation (/audio/generate) # ===================================================================== diff --git a/studio/backend/utils/datasets/dataset_utils.py b/studio/backend/utils/datasets/dataset_utils.py index 9d15b86ca1..d4876b6fb8 100644 --- a/studio/backend/utils/datasets/dataset_utils.py +++ b/studio/backend/utils/datasets/dataset_utils.py @@ -1092,6 +1092,9 @@ def format_and_template_dataset( # LLM FLOW (Existing code) else: # Step 1: Format the dataset + n_rows = len(dataset) if hasattr(dataset, "__len__") else None + if progress_callback and n_rows: + progress_callback(status_message = f"Formatting dataset ({n_rows:,} rows)...") dataset_info = format_dataset( dataset, format_type = format_type, @@ -1106,6 +1109,11 @@ def format_and_template_dataset( ) # Step 2: Apply chat template + detected = dataset_info.get("detected_format", "unknown") + if progress_callback and n_rows: + progress_callback( + status_message = f"Applying chat template to {detected} ({n_rows:,} rows)..." + ) # Gemma emits a leading that must be stripped for text-only chatml/sharegpt. is_alpaca = format_type == "alpaca" or ( format_type == "auto" and dataset_info["detected_format"] == "alpaca" diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts index 01bc762d86..0f63e0d462 100644 --- a/studio/frontend/src/features/chat/api/chat-api.ts +++ b/studio/frontend/src/features/chat/api/chat-api.ts @@ -62,6 +62,15 @@ export async function getInferenceStatus(): Promise { return parseJsonOrThrow(response); } +export async function getLlamaCppStatus(): Promise<{ + ready: boolean; + building: boolean; + error: string | null; +}> { + const response = await authFetch("/api/inference/llama-cpp-status"); + return parseJsonOrThrow(response); +} + export async function loadModel( payload: LoadModelRequest, ): Promise { diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts index b79c3073b2..e06418ce6d 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts @@ -9,6 +9,7 @@ import { getDownloadProgress, getGgufDownloadProgress, getInferenceStatus, + getLlamaCppStatus, listLoras, listModels, loadModel, @@ -318,6 +319,25 @@ export function useChatModelRuntime() { try { async function performLoad(): Promise { if (abortCtrl.signal.aborted) throw new Error("Cancelled"); + + // For GGUF models, check if llama.cpp is still compiling + if (ggufVariant) { + try { + const llamaStatus = await getLlamaCppStatus(); + if (llamaStatus.building) { + toast.info("Waiting for llama.cpp to compile...", { + description: "This is a one-time build. Studio is still usable for non-GGUF tasks.", + duration: 8000, + }); + } else if (llamaStatus.error) { + throw new Error(llamaStatus.error); + } + } catch (e) { + if (e instanceof Error && e.message.includes("not found")) throw e; + // Endpoint might not exist on older backends, ignore + } + } + let previousWasUnloaded = false; const currentCheckpoint = useChatRuntimeStore.getState().params.checkpoint; diff --git a/studio/setup.ps1 b/studio/setup.ps1 index 1cf17b5d27..bc0105bee7 100644 --- a/studio/setup.ps1 +++ b/studio/setup.ps1 @@ -126,6 +126,42 @@ function Get-CudaComputeCapability { return $null } +# Check if an nvcc binary supports a given sm_ architecture. +# Uses `nvcc --list-gpu-code` which outputs sm_* tokens (--list-gpu-arch +# outputs compute_* tokens instead). Available since CUDA 11.6. +# Returns $false if the flag isn't supported (old toolkit) — safer to reject +# and fall back to scanning/PTX than to assume support and fail later. +function Test-NvccArchSupport { + param([string]$NvccExe, [string]$Arch) + try { + $listCode = & $NvccExe --list-gpu-code 2>&1 | Out-String + if ($LASTEXITCODE -ne 0) { return $false } + return ($listCode -match "sm_$Arch") + } catch { + return $false + } +} + +# Given an nvcc binary, return the highest sm_ architecture it supports. +# Returns e.g. "90" for CUDA 12.4. Returns $null if detection fails. +function Get-NvccMaxArch { + param([string]$NvccExe) + try { + $listCode = & $NvccExe --list-gpu-code 2>&1 | Out-String + if ($LASTEXITCODE -ne 0) { return $null } + $arches = @() + foreach ($line in $listCode -split "`n") { + if ($line.Trim() -match '^sm_(\d+)') { + $arches += [int]$Matches[1] + } + } + if ($arches.Count -gt 0) { + return ($arches | Sort-Object | Select-Object -Last 1).ToString() + } + } catch { } + return $null +} + # Detect driver's max CUDA version from nvidia-smi and return the highest # compatible PyTorch CUDA index tag (e.g. "cu128"). # PyTorch on Windows ships CPU-only by default from PyPI; CUDA wheels live at @@ -368,19 +404,76 @@ try { } } catch {} -# -- Find a toolkit that's compatible with the driver -- +# Detect compute capability early so we can validate toolkit support +$CudaArch = Get-CudaComputeCapability +if ($CudaArch) { + Write-Host " GPU Compute Capability = $($CudaArch.Insert($CudaArch.Length-1, '.')) (sm_$CudaArch)" -ForegroundColor Gray +} + +# -- Find a toolkit that's compatible with the driver AND the GPU -- +# Strategy: prefer the toolkit at CUDA_PATH (user's existing setup) if it's +# compatible with the driver AND supports the GPU architecture. Only fall back +# to scanning side-by-side installs if CUDA_PATH is missing, points to an +# incompatible version, or can't compile for the GPU. This avoids +# header/binary mismatches when multiple toolkits are installed. $IncompatibleToolkit = $null +$NvccPath = $null + if ($DriverMaxCuda) { - $NvccPath = Find-Nvcc -MaxVersion $DriverMaxCuda - if ($NvccPath) { - Write-Host " [OK] Found compatible CUDA Toolkit (nvcc: $NvccPath)" -ForegroundColor Green - } else { - # Check if there's an incompatible (too new) toolkit installed - $AnyNvcc = Find-Nvcc - if ($AnyNvcc) { - $NvccOut = & $AnyNvcc --version 2>&1 | Out-String - if ($NvccOut -match "release\s+([\d]+\.[\d]+)") { - $IncompatibleToolkit = $Matches[1] + $drMajorCuda = [int]$DriverMaxCuda.Split('.')[0] + $drMinorCuda = [int]$DriverMaxCuda.Split('.')[1] + + # --- Step 1: Check existing CUDA_PATH first --- + $existingCudaPath = [Environment]::GetEnvironmentVariable('CUDA_PATH', 'Machine') + if (-not $existingCudaPath) { + $existingCudaPath = [Environment]::GetEnvironmentVariable('CUDA_PATH', 'User') + } + if ($existingCudaPath -and (Test-Path (Join-Path $existingCudaPath 'bin\nvcc.exe'))) { + $candidateNvcc = Join-Path $existingCudaPath 'bin\nvcc.exe' + $verOut = & $candidateNvcc --version 2>&1 | Out-String + if ($verOut -match 'release\s+(\d+)\.(\d+)') { + $tkMaj = [int]$Matches[1]; $tkMin = [int]$Matches[2] + $isCompat = ($tkMaj -lt $drMajorCuda) -or ($tkMaj -eq $drMajorCuda -and $tkMin -le $drMinorCuda) + if ($isCompat) { + # Also verify the toolkit supports our GPU architecture + Write-Host " [DEBUG] Checking CUDA compatibility: toolkit=$tkMaj.$tkMin arch=sm_$CudaArch" -ForegroundColor Magenta + $archOk = $true + if ($CudaArch) { + $archOk = Test-NvccArchSupport -NvccExe $candidateNvcc -Arch $CudaArch + if (-not $archOk) { + Write-Host " [INFO] CUDA_PATH toolkit (CUDA $tkMaj.$tkMin) does not support GPU arch sm_$CudaArch" -ForegroundColor Yellow + Write-Host " Looking for a newer toolkit..." -ForegroundColor Yellow + } + } + if ($archOk) { + $NvccPath = $candidateNvcc + Write-Host " [OK] Using existing CUDA Toolkit at CUDA_PATH (nvcc: $NvccPath)" -ForegroundColor Green + } + } else { + Write-Host " [INFO] CUDA_PATH ($existingCudaPath) has CUDA $tkMaj.$tkMin which exceeds driver max $DriverMaxCuda" -ForegroundColor Yellow + } + } + } + + # --- Step 2: Fall back to scanning side-by-side installs --- + if (-not $NvccPath) { + $NvccPath = Find-Nvcc -MaxVersion $DriverMaxCuda + if ($NvccPath) { + Write-Host " [OK] Found compatible CUDA Toolkit (nvcc: $NvccPath)" -ForegroundColor Green + if ($existingCudaPath) { + $selectedRoot = Split-Path (Split-Path $NvccPath -Parent) -Parent + if ($existingCudaPath.TrimEnd('\') -ne $selectedRoot.TrimEnd('\')) { + Write-Host " [INFO] Overriding CUDA_PATH from $existingCudaPath to $selectedRoot" -ForegroundColor Yellow + } + } + } else { + # Check if there's an incompatible (too new) toolkit installed + $AnyNvcc = Find-Nvcc + if ($AnyNvcc) { + $NvccOut = & $AnyNvcc --version 2>&1 | Out-String + if ($NvccOut -match "release\s+([\d]+\.[\d]+)") { + $IncompatibleToolkit = $Matches[1] + } } } } @@ -487,6 +580,19 @@ $CudaToolkitRoot = Split-Path (Split-Path $NvccPath -Parent) -Parent # in future sessions (overwrites any existing value pointing to a newer, incompatible version) [Environment]::SetEnvironmentVariable('CUDA_PATH', $CudaToolkitRoot, 'User') Write-Host " Persisted CUDA_PATH=$CudaToolkitRoot to user environment" -ForegroundColor Gray +# Clear all versioned CUDA_PATH_V* env vars in this process to prevent +# cmake/MSBuild from discovering a conflicting CUDA installation. +$cudaPathVars = @([Environment]::GetEnvironmentVariables('Process').Keys | Where-Object { $_ -match '^CUDA_PATH_V' }) +foreach ($v in $cudaPathVars) { + [Environment]::SetEnvironmentVariable($v, $null, 'Process') +} +# Set only the versioned var matching the selected toolkit (e.g. CUDA_PATH_V13_0) +$tkDirName = Split-Path $CudaToolkitRoot -Leaf +if ($tkDirName -match '^v(\d+)\.(\d+)') { + $cudaPathVerVar = "CUDA_PATH_V$($Matches[1])_$($Matches[2])" + [Environment]::SetEnvironmentVariable($cudaPathVerVar, $CudaToolkitRoot, 'Process') + Write-Host " Set $cudaPathVerVar (cleared other CUDA_PATH_V* vars)" -ForegroundColor Gray +} # Ensure nvcc's bin dir is on PATH for this process $nvccBinDir = Split-Path $NvccPath -Parent if ($env:PATH -notlike "*$nvccBinDir*") { @@ -503,15 +609,38 @@ if (-not $userPath -or $userPath -notlike "*$nvccBinDir*") { Write-Host " Persisted CUDA bin dir to user PATH" -ForegroundColor Gray } +# -- Ensure CUDA ↔ Visual Studio integration files exist -- +# When CUDA is installed before VS Build Tools (or VS is reinstalled after CUDA), +# the MSBuild .targets/.props files that let VS compile .cu files are missing. +# cmake fails with "No CUDA toolset found". Fix: copy from CUDA extras dir. +if ($VsInstallPath -and $CudaToolkitRoot) { + $vsCustomizations = Join-Path $VsInstallPath "MSBuild\Microsoft\VC\v170\BuildCustomizations" + $cudaExtras = Join-Path $CudaToolkitRoot "extras\visual_studio_integration\MSBuildExtensions" + if ((Test-Path $cudaExtras) -and (Test-Path $vsCustomizations)) { + $hasTargets = Get-ChildItem $vsCustomizations -Filter "CUDA *.targets" -ErrorAction SilentlyContinue + if (-not $hasTargets) { + Write-Host " [INFO] CUDA VS integration missing -- copying .targets files..." -ForegroundColor Yellow + try { + Copy-Item "$cudaExtras\*" $vsCustomizations -Force -ErrorAction Stop + Write-Host " [OK] CUDA VS integration files installed" -ForegroundColor Green + } catch { + Write-Host " [WARN] Could not copy CUDA VS integration files (may need admin)" -ForegroundColor Yellow + Write-Host " Manual fix: copy contents of" -ForegroundColor Yellow + Write-Host " $cudaExtras" -ForegroundColor Cyan + Write-Host " into:" -ForegroundColor Yellow + Write-Host " $vsCustomizations" -ForegroundColor Cyan + } + } + } +} + Write-Host "[OK] CUDA Toolkit: $NvccPath" -ForegroundColor Green Write-Host " CUDA_PATH = $CudaToolkitRoot" -ForegroundColor Gray Write-Host " CudaToolkitDir = $CudaToolkitRoot\" -ForegroundColor Gray -# Detect compute capability (used later for llama.cpp cmake) -$CudaArch = Get-CudaComputeCapability -if ($CudaArch) { - Write-Host " Compute Capability = $($CudaArch.Insert($CudaArch.Length-1, '.')) (sm_$CudaArch)" -ForegroundColor Gray -} else { +# $CudaArch was detected earlier (before toolkit selection) so it could +# influence which toolkit we picked. Just log the final state here. +if (-not $CudaArch) { Write-Host " [WARN] Could not detect compute capability -- cmake will use defaults" -ForegroundColor Yellow } @@ -875,6 +1004,21 @@ if (Test-Path $LlamaServerBin) { $BuildOk = $true $FailedStep = "" + # Re-sanitize CUDA_PATH_V* vars — Refresh-Environment (called during + # Node/Python installs above) may have repopulated conflicting versioned + # vars from the Machine registry. + $cudaPathVars2 = @([Environment]::GetEnvironmentVariables('Process').Keys | Where-Object { $_ -match '^CUDA_PATH_V' }) + foreach ($v2 in $cudaPathVars2) { + [Environment]::SetEnvironmentVariable($v2, $null, 'Process') + } + $tkDirName2 = Split-Path $CudaToolkitRoot -Leaf + if ($tkDirName2 -match '^v(\d+)\.(\d+)') { + [Environment]::SetEnvironmentVariable("CUDA_PATH_V$($Matches[1])_$($Matches[2])", $CudaToolkitRoot, 'Process') + } + # Also re-assert CUDA_PATH and CudaToolkitDir in case they were overwritten + [Environment]::SetEnvironmentVariable('CUDA_PATH', $CudaToolkitRoot, 'Process') + [Environment]::SetEnvironmentVariable('CudaToolkitDir', "$CudaToolkitRoot\", 'Process') + # -- Step A: Clone or pull llama.cpp -- if (Test-Path (Join-Path $LlamaCppDir ".git")) { @@ -921,6 +1065,7 @@ if (Test-Path $LlamaServerBin) { # CUDA flags (Unsloth-aligned) $CmakeArgs += '-DGGML_CUDA=ON' $CmakeArgs += "-DCUDAToolkit_ROOT=$CudaToolkitRoot" + $CmakeArgs += "-DCUDA_TOOLKIT_ROOT_DIR=$CudaToolkitRoot" $CmakeArgs += "-DCMAKE_CUDA_COMPILER=$NvccPath" $CmakeArgs += '-DGGML_CUDA_FA_ALL_QUANTS=ON' $CmakeArgs += '-DGGML_CUDA_F16=OFF' @@ -928,7 +1073,20 @@ if (Test-Path $LlamaServerBin) { $CmakeArgs += '-DGGML_CUDA_FORCE_CUBLAS=OFF' $CmakeArgs += '-DGGML_CUDA_PEER_MAX_BATCH_SIZE=8192' if ($CudaArch) { - $CmakeArgs += "-DCMAKE_CUDA_ARCHITECTURES=$CudaArch" + # Validate nvcc actually supports this architecture + if (Test-NvccArchSupport -NvccExe $NvccPath -Arch $CudaArch) { + $CmakeArgs += "-DCMAKE_CUDA_ARCHITECTURES=$CudaArch" + } else { + # GPU arch too new for this toolkit — fall back to highest supported. + # PTX forward-compatibility will JIT-compile for the actual GPU at runtime. + $maxArch = Get-NvccMaxArch -NvccExe $NvccPath + if ($maxArch) { + $CmakeArgs += "-DCMAKE_CUDA_ARCHITECTURES=$maxArch" + Write-Host " [WARN] GPU is sm_$CudaArch but nvcc only supports up to sm_$maxArch" -ForegroundColor Yellow + Write-Host " Building with sm_$maxArch (PTX will JIT for your GPU at runtime)" -ForegroundColor Yellow + } + # else: omit flag entirely, let cmake pick defaults + } } cmake @CmakeArgs 2>&1 | Out-Null