diff --git a/.gitignore b/.gitignore index 08f9d8ee6b..044775e846 100755 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,12 @@ unsloth_training_checkpoints/ *.gguf *.safetensors +# llama.cpp build (built by setup.sh, shared with unsloth-zoo export) +llama.cpp/ + +# Built binaries (llama-server etc.) +bin/ + # IDE / Editors .vscode/ .idea/ diff --git a/setup.sh b/setup.sh index 3e6b41db9f..55fa57c83f 100755 --- a/setup.sh +++ b/setup.sh @@ -211,7 +211,99 @@ else fi fi -# ── 8. Add shell alias (skip in Colab) ── +# ── 8. Build llama.cpp binaries for GGUF inference + export ── +# Builds in-tree at $REPO/llama.cpp/. This directory is shared with +# unsloth-zoo's GGUF export pipeline. We build: +# - llama-server: for GGUF model inference +# - llama-quantize: for GGUF export quantization (symlinked to root for check_llama_cpp()) +LLAMA_SERVER_BIN="$SCRIPT_DIR/llama.cpp/build/bin/llama-server" +if [ -f "$LLAMA_SERVER_BIN" ]; then + echo "" + echo "✅ llama-server already exists at $LLAMA_SERVER_BIN" +else + # Check prerequisites + if ! command -v cmake &>/dev/null; then + echo "" + echo "⚠️ cmake not found — skipping llama-server build (GGUF inference won't be available)" + echo " Install cmake and re-run setup.sh to enable GGUF inference." + elif ! command -v git &>/dev/null; then + echo "" + echo "⚠️ git not found — skipping llama-server build (GGUF inference won't be available)" + else + echo "" + echo "Building llama-server for GGUF inference..." + LLAMA_CPP_DIR="$SCRIPT_DIR/llama.cpp" + + BUILD_OK=true + if [ -d "$LLAMA_CPP_DIR/.git" ]; then + echo " llama.cpp repo already cloned, pulling latest..." + run_quiet "pull llama.cpp" git -C "$LLAMA_CPP_DIR" pull || true + else + # Remove any non-git llama.cpp directory (stale build artifacts) + rm -rf "$LLAMA_CPP_DIR" + run_quiet "clone llama.cpp" git clone --depth 1 https://github.com/ggml-org/llama.cpp.git "$LLAMA_CPP_DIR" || BUILD_OK=false + fi + + if [ "$BUILD_OK" = true ]; then + CMAKE_ARGS="" + # Detect CUDA: check nvcc on PATH, then common install locations + NVCC_PATH="" + if command -v nvcc &>/dev/null; then + NVCC_PATH="$(command -v nvcc)" + elif [ -x /usr/local/cuda/bin/nvcc ]; then + NVCC_PATH="/usr/local/cuda/bin/nvcc" + export PATH="/usr/local/cuda/bin:$PATH" + elif ls /usr/local/cuda-*/bin/nvcc &>/dev/null 2>&1; then + # Pick the newest cuda-XX.X directory + NVCC_PATH="$(ls -d /usr/local/cuda-*/bin/nvcc 2>/dev/null | sort -V | tail -1)" + export PATH="$(dirname "$NVCC_PATH"):$PATH" + fi + + if [ -n "$NVCC_PATH" ]; then + echo " Building with CUDA support (nvcc: $NVCC_PATH)..." + CMAKE_ARGS="-DGGML_CUDA=ON" + elif [ -d /usr/local/cuda ] || nvidia-smi &>/dev/null; then + echo " CUDA driver detected but nvcc not found — building CPU-only" + echo " To enable GPU: install cuda-toolkit or add nvcc to PATH" + else + echo " Building CPU-only (no CUDA detected)..." + fi + + NCPU=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) + + run_quiet "cmake llama.cpp" cmake -S "$LLAMA_CPP_DIR" -B "$LLAMA_CPP_DIR/build" $CMAKE_ARGS || BUILD_OK=false + fi + + if [ "$BUILD_OK" = true ]; then + run_quiet "build llama-server" cmake --build "$LLAMA_CPP_DIR/build" --config Release --target llama-server -j"$NCPU" || BUILD_OK=false + fi + + # Also build llama-quantize (needed by unsloth-zoo's GGUF export pipeline) + if [ "$BUILD_OK" = true ]; then + run_quiet "build llama-quantize" cmake --build "$LLAMA_CPP_DIR/build" --config Release --target llama-quantize -j"$NCPU" || true + # Symlink to llama.cpp root — check_llama_cpp() looks for the binary there + QUANTIZE_BIN="$LLAMA_CPP_DIR/build/bin/llama-quantize" + if [ -f "$QUANTIZE_BIN" ]; then + ln -sf build/bin/llama-quantize "$LLAMA_CPP_DIR/llama-quantize" + fi + fi + + if [ "$BUILD_OK" = true ]; then + if [ -f "$LLAMA_SERVER_BIN" ]; then + echo "✅ llama-server built at $LLAMA_SERVER_BIN" + else + echo "⚠️ llama-server binary not found after build — GGUF inference won't be available" + fi + if [ -f "$LLAMA_CPP_DIR/llama-quantize" ]; then + echo "✅ llama-quantize available for GGUF export" + fi + else + echo "⚠️ llama-server build failed — GGUF inference won't be available, but everything else works" + fi + fi +fi + +# ── 9. Add shell alias (skip in Colab) ── # Note: venv activation does NOT persist across terminal sessions. # This alias hardcodes the venv python path so users don't need to activate. if [ "$IS_COLAB" = false ]; then diff --git a/studio/backend/core/export/export.py b/studio/backend/core/export/export.py index 231566cc22..cb2cb01c22 100644 --- a/studio/backend/core/export/export.py +++ b/studio/backend/core/export/export.py @@ -397,53 +397,30 @@ class ExportBackend: # Save locally if requested if save_directory: - logger.info(f"Saving GGUF model locally to: {save_directory}") + # Resolve to absolute path so unsloth's relative-path internals + # (check_llama_cpp, use_local_gguf, _download_convert_hf_to_gguf) + # all resolve against the repo root cwd, NOT the export directory. + abs_save_dir = os.path.abspath(save_directory) + logger.info(f"Saving GGUF model locally to: {abs_save_dir}") # Create the directory if it doesn't exist - os.makedirs(save_directory, exist_ok=True) + os.makedirs(abs_save_dir, exist_ok=True) - # Get the base filename for the GGUF file - import shutil - original_dir = os.getcwd() + # On WSL, patch out sudo check before llama.cpp build + _apply_wsl_sudo_patch() - try: - # Change to target directory - os.chdir(save_directory) - logger.info(f"Changed directory to: {save_directory}") + # Pass absolute path — no os.chdir needed. + # unsloth saves model files into this directory, while + # check_llama_cpp("llama.cpp") resolves against cwd (repo root) + # where setup.sh already built llama.cpp with quantizer. + model_save_path = os.path.join(abs_save_dir, "model") + self.current_model.save_pretrained_gguf( + model_save_path, + self.current_tokenizer, + quantization_method=quant_method + ) - # On WSL, patch out sudo check before llama.cpp build - _apply_wsl_sudo_patch() - - # Now save (will save in current directory) - self.current_model.save_pretrained_gguf( - "model", # Base filename - self.current_tokenizer, - quantization_method=quant_method - ) - - logger.info(f"GGUF model saved successfully in {save_directory}") - - # Check if llama.cpp directory was created here - llama_cpp_in_target = os.path.join(save_directory, "llama.cpp") - llama_cpp_in_original = os.path.join(original_dir, "llama.cpp") - - if os.path.exists(llama_cpp_in_target): - logger.info(f"Found llama.cpp directory in {save_directory}") - - # Remove llama.cpp from original directory if it exists - if os.path.exists(llama_cpp_in_original): - logger.info(f"Removing existing llama.cpp in {original_dir}") - shutil.rmtree(llama_cpp_in_original) - - # Move llama.cpp back to original directory - logger.info(f"Moving llama.cpp to {original_dir}") - shutil.move(llama_cpp_in_target, llama_cpp_in_original) - logger.info(f"Successfully moved llama.cpp back to original directory") - - finally: - # Always change back to original directory - os.chdir(original_dir) - logger.info(f"Changed back to original directory: {original_dir}") + logger.info(f"GGUF model saved successfully in {abs_save_dir}") # Push to hub if requested if push_to_hub: diff --git a/studio/backend/core/inference/__init__.py b/studio/backend/core/inference/__init__.py index 494229a087..ff8b75d36a 100644 --- a/studio/backend/core/inference/__init__.py +++ b/studio/backend/core/inference/__init__.py @@ -2,8 +2,10 @@ Inference submodule - Inference backend for model loading and generation """ from .inference import InferenceBackend, get_inference_backend +from .llama_cpp import LlamaCppBackend __all__ = [ 'InferenceBackend', 'get_inference_backend', + 'LlamaCppBackend', ] diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py new file mode 100644 index 0000000000..30fe660404 --- /dev/null +++ b/studio/backend/core/inference/llama_cpp.py @@ -0,0 +1,416 @@ +""" +llama-server inference backend for GGUF models. + +Manages a llama-server subprocess and proxies chat completions +through its OpenAI-compatible /v1/chat/completions endpoint. +""" +import atexit +import json +import logging +import shutil +import signal +import socket +import subprocess +import threading +import time +from pathlib import Path +from typing import Generator, Optional + +import httpx + +logger = logging.getLogger(__name__) + + +class LlamaCppBackend: + """ + Manages a llama-server subprocess for GGUF model inference. + + Lifecycle: + 1. load_model() — starts llama-server with the GGUF file + 2. generate_chat_completion() — proxies to /v1/chat/completions, streams back + 3. unload_model() — terminates llama-server subprocess + """ + + def __init__(self): + self._process: Optional[subprocess.Popen] = None + self._port: Optional[int] = None + self._model_identifier: Optional[str] = None + self._gguf_path: Optional[str] = None + self._hf_repo: Optional[str] = None + self._hf_variant: Optional[str] = None + self._is_vision: bool = False + self._healthy = False + self._lock = threading.Lock() + + atexit.register(self._cleanup) + + # ── Properties ──────────────────────────────────────────────── + + @property + def is_loaded(self) -> bool: + return self._process is not None and self._healthy + + @property + def base_url(self) -> str: + return f"http://127.0.0.1:{self._port}" + + @property + def model_identifier(self) -> Optional[str]: + return self._model_identifier + + @property + def is_vision(self) -> bool: + return self._is_vision + + # ── Binary discovery ────────────────────────────────────────── + + @staticmethod + def _find_llama_server_binary() -> Optional[str]: + """ + Locate the llama-server binary. + + Search order: + 1. LLAMA_SERVER_PATH environment variable + 2. ./llama.cpp/build/bin/llama-server (built by setup.sh in-tree) + 3. llama-server on PATH (system install) + 4. ./bin/llama-server (legacy: extracted binary) + """ + import os + + # 1. Env var + env_path = os.environ.get("LLAMA_SERVER_PATH") + if env_path and Path(env_path).is_file(): + return env_path + + # Project root: llama_cpp.py → inference/ → core/ → backend/ → studio/ → root + project_root = Path(__file__).resolve().parents[4] + + # 2. In-tree llama.cpp build (setup.sh builds here) + build_path = project_root / "llama.cpp" / "build" / "bin" / "llama-server" + if build_path.is_file(): + return str(build_path) + + # 3. System PATH + system_path = shutil.which("llama-server") + if system_path: + return system_path + + # 4. Legacy: extracted to bin/ + bin_path = project_root / "bin" / "llama-server" + if bin_path.is_file(): + return str(bin_path) + + return None + + # ── Port allocation ─────────────────────────────────────────── + + @staticmethod + def _find_free_port() -> int: + """Find an available TCP port.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + + # ── Lifecycle ───────────────────────────────────────────────── + + def load_model( + self, + *, + # Local mode: pass a path to a .gguf file + gguf_path: Optional[str] = None, + # HF mode: let llama-server download via -hf "repo:quant" + hf_repo: Optional[str] = None, + hf_variant: Optional[str] = None, + hf_token: Optional[str] = None, + # Common + model_identifier: str, + is_vision: bool = False, + n_ctx: int = 4096, + n_gpu_layers: int = -1, + n_threads: Optional[int] = None, + ) -> bool: + """ + Start llama-server with a GGUF model. + + Two modes: + - Local: ``gguf_path="/path/to/model.gguf"`` → uses ``-m`` + - HF: ``hf_repo="unsloth/gemma-3-4b-it-GGUF", hf_variant="Q4_K_M"`` → uses ``-hf`` + + In HF mode, llama-server handles downloading, caching, and + auto-loading mmproj files for vision models. + + Returns True if server started and health check passed. + """ + with self._lock: + self._kill_process() + + binary = self._find_llama_server_binary() + if not binary: + raise RuntimeError( + "llama-server binary not found. " + "Run setup.sh to build it, install llama.cpp, " + "or set LLAMA_SERVER_PATH environment variable." + ) + + self._port = self._find_free_port() + + # Build command based on mode + if hf_repo: + hf_spec = f"{hf_repo}:{hf_variant}" if hf_variant else hf_repo + cmd = [ + binary, + "-hf", hf_spec, + "--port", str(self._port), + "-c", str(n_ctx), + "-ngl", str(n_gpu_layers), + ] + if hf_token: + cmd.extend(["--hf-token", hf_token]) + elif gguf_path: + if not Path(gguf_path).is_file(): + raise FileNotFoundError(f"GGUF file not found: {gguf_path}") + cmd = [ + binary, + "-m", gguf_path, + "--port", str(self._port), + "-c", str(n_ctx), + "-ngl", str(n_gpu_layers), + ] + else: + raise ValueError("Either gguf_path or hf_repo must be provided") + + if n_threads is not None: + cmd.extend(["--threads", str(n_threads)]) + + logger.info(f"Starting llama-server: {' '.join(cmd)}") + + # Set LD_LIBRARY_PATH so llama-server can find its shared libs + # (libmtmd.so, libllama.so, etc.) which live next to the binary + import os + env = os.environ.copy() + binary_dir = str(Path(binary).parent) + existing_ld = env.get("LD_LIBRARY_PATH", "") + env["LD_LIBRARY_PATH"] = f"{binary_dir}:{existing_ld}" if existing_ld else binary_dir + + self._process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + env=env, + ) + + self._gguf_path = gguf_path + self._hf_repo = hf_repo + self._hf_variant = hf_variant + self._is_vision = is_vision + self._model_identifier = model_identifier + + # HF mode: llama-server downloads before becoming healthy — need longer timeout + timeout = 600.0 if hf_repo else 120.0 + if not self._wait_for_health(timeout=timeout): + self._kill_process() + raise RuntimeError( + "llama-server failed to start. " + "Check that the GGUF file is valid and you have enough memory." + ) + + self._healthy = True + + logger.info( + f"llama-server ready on port {self._port} " + f"for model '{model_identifier}'" + ) + return True + + def unload_model(self) -> bool: + """Terminate the llama-server subprocess and clean up state.""" + with self._lock: + self._kill_process() + logger.info(f"Unloaded GGUF model: {self._model_identifier}") + self._model_identifier = None + self._gguf_path = None + self._hf_repo = None + self._hf_variant = None + self._is_vision = False + self._port = None + self._healthy = False + return True + + def _kill_process(self): + """Terminate the subprocess if running.""" + if self._process is None: + return + try: + self._process.terminate() + self._process.wait(timeout=5) + except subprocess.TimeoutExpired: + logger.warning("llama-server did not exit on SIGTERM, sending SIGKILL") + self._process.kill() + self._process.wait(timeout=5) + except Exception as e: + logger.warning(f"Error killing llama-server process: {e}") + finally: + self._process = None + + def _cleanup(self): + """atexit handler to ensure llama-server is terminated.""" + self._kill_process() + + def _wait_for_health(self, timeout: float = 120.0, interval: float = 0.5) -> bool: + """ + Poll llama-server's /health endpoint until it responds 200. + + Also monitors subprocess for early exit/crash. + """ + deadline = time.monotonic() + timeout + url = f"http://127.0.0.1:{self._port}/health" + + while time.monotonic() < deadline: + # Check if process crashed + if self._process.poll() is not None: + # Read remaining output for error info + output = self._process.stdout.read() if self._process.stdout else "" + logger.error( + f"llama-server exited with code {self._process.returncode}. " + f"Output: {output[:2000]}" + ) + return False + + try: + resp = httpx.get(url, timeout=2.0) + if resp.status_code == 200: + return True + except (httpx.ConnectError, httpx.TimeoutException): + pass + + time.sleep(interval) + + logger.error(f"llama-server health check timed out after {timeout}s") + return False + + # ── Message building (OpenAI format) ────────────────────────── + + @staticmethod + def _build_openai_messages( + messages: list[dict], + image_b64: Optional[str] = None, + ) -> list[dict]: + """ + Build OpenAI-format messages, optionally injecting an image_url + content part into the last user message for vision models. + + If no image is provided, returns messages as-is. + """ + if not image_b64: + return messages + + # Find the last user message and convert to multimodal content parts + result = [msg.copy() for msg in messages] + last_user_idx = None + for i, msg in enumerate(result): + if msg["role"] == "user": + last_user_idx = i + + if last_user_idx is not None: + text_content = result[last_user_idx].get("content", "") + result[last_user_idx]["content"] = [ + {"type": "text", "text": text_content}, + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{image_b64}", + }, + }, + ] + + return result + + # ── Generation (proxy to llama-server) ──────────────────────── + + def generate_chat_completion( + self, + messages: list[dict], + image_b64: Optional[str] = None, + temperature: float = 0.7, + top_p: float = 0.9, + top_k: int = 40, + min_p: float = 0.0, + max_tokens: int = 512, + repetition_penalty: float = 1.1, + stop: Optional[list[str]] = None, + cancel_event: Optional[threading.Event] = None, + ) -> Generator[str, None, None]: + """ + Send a chat completion request to llama-server and stream tokens back. + + Uses /v1/chat/completions — llama-server handles chat template + application and vision (multimodal image_url parts) natively. + + Yields cumulative text (matching InferenceBackend's convention). + """ + if not self.is_loaded: + raise RuntimeError("llama-server is not loaded") + + openai_messages = self._build_openai_messages(messages, image_b64) + + payload = { + "messages": openai_messages, + "stream": True, + "temperature": temperature, + "top_p": top_p, + "top_k": top_k if top_k >= 0 else 0, + "min_p": min_p, + "max_tokens": max_tokens, + "repeat_penalty": repetition_penalty, + } + if stop: + payload["stop"] = stop + + url = f"{self.base_url}/v1/chat/completions" + cumulative = "" + + try: + with httpx.Client(timeout=None) as client: + with client.stream("POST", url, json=payload) as response: + if response.status_code != 200: + error_body = response.read().decode() + raise RuntimeError( + f"llama-server returned {response.status_code}: {error_body}" + ) + + buffer = "" + for raw_chunk in response.iter_text(): + if cancel_event is not None and cancel_event.is_set(): + break + + buffer += raw_chunk + while "\n" in buffer: + line, buffer = buffer.split("\n", 1) + line = line.strip() + + if not line: + continue + if line == "data: [DONE]": + return + if not line.startswith("data: "): + continue + + try: + data = json.loads(line[6:]) + choices = data.get("choices", []) + if choices: + delta = choices[0].get("delta", {}) + token = delta.get("content", "") + if token: + cumulative += token + yield cumulative + except json.JSONDecodeError: + logger.debug(f"Skipping malformed SSE line: {line[:100]}") + + except httpx.ConnectError: + raise RuntimeError("Lost connection to llama-server") + except Exception as e: + if cancel_event is not None and cancel_event.is_set(): + return + raise diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py index 43cb5ff1a1..e4e7a474be 100644 --- a/studio/backend/core/training/trainer.py +++ b/studio/backend/core/training/trainer.py @@ -173,6 +173,15 @@ class UnslothTrainer: token=hf_token, ) logger.info("Loaded vision model") + + # Diagnostic: check if FastVisionModel returned a real Processor or a raw tokenizer + from transformers import ProcessorMixin + tok = self.tokenizer + has_image_proc = isinstance(tok, ProcessorMixin) or hasattr(tok, "image_processor") + print(f"\n[VLM Diagnostic] FastVisionModel returned: {type(tok).__name__}") + print(f"[VLM Diagnostic] Is ProcessorMixin: {isinstance(tok, ProcessorMixin)}") + print(f"[VLM Diagnostic] Has image_processor: {hasattr(tok, 'image_processor')}") + print(f"[VLM Diagnostic] Usable as vision processor: {has_image_proc}\n") else: # Load text model - returns (model, tokenizer) self.model, self.tokenizer = FastLanguageModel.from_pretrained( diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index d2d98d7944..fc3f788ac6 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -17,6 +17,7 @@ class LoadRequest(BaseModel): max_seq_length: int = Field(2048, ge=128, le=32768, description="Maximum sequence length") load_in_4bit: bool = Field(True, description="Load model in 4-bit quantization") is_lora: bool = Field(False, description="Whether this is a LoRA adapter") + gguf_variant: Optional[str] = Field(None, description="GGUF quantization variant (e.g. 'Q4_K_M')") class UnloadRequest(BaseModel): @@ -43,6 +44,7 @@ class LoadResponse(BaseModel): display_name: str = Field(..., description="Display name of the model") is_vision: bool = Field(False, description="Whether model is a vision model") is_lora: bool = Field(False, description="Whether model is a LoRA adapter") + is_gguf: bool = Field(False, description="Whether model is a GGUF model (llama.cpp)") inference: dict = Field(..., description="Inference parameters (temperature, top_p, top_k, min_p)") @@ -56,6 +58,7 @@ class InferenceStatusResponse(BaseModel): """Current inference backend status""" active_model: Optional[str] = Field(None, description="Currently active model identifier") is_vision: bool = Field(False, description="Whether the active model is a vision model") + is_gguf: bool = Field(False, description="Whether the active model is a GGUF model (llama.cpp)") loading: List[str] = Field(default_factory=list, description="Models currently being loaded") loaded: List[str] = Field(default_factory=list, description="Models currently loaded") diff --git a/studio/backend/models/models.py b/studio/backend/models/models.py index 10d87d7825..39034f8ce2 100644 --- a/studio/backend/models/models.py +++ b/studio/backend/models/models.py @@ -53,6 +53,7 @@ class ModelDetails(BaseModel): config: Optional[Dict[str, Any]] = Field(None, description="Model configuration dictionary") is_vision: bool = Field(False, description="Whether model is a vision model") is_lora: bool = Field(False, description="Whether model is a LoRA adapter") + is_gguf: bool = Field(False, description="Whether model is a GGUF model (llama.cpp format)") base_model: Optional[str] = Field(None, description="Base model if this is a LoRA adapter") @@ -77,6 +78,21 @@ class ModelListResponse(BaseModel): default_models: List[str] = Field(default_factory=list, description="List of default model IDs") +class GgufVariantDetail(BaseModel): + """A single GGUF quantization variant in a HuggingFace repo.""" + filename: str = Field(..., description="GGUF filename (e.g., 'gemma-3-4b-it-Q4_K_M.gguf')") + quant: str = Field(..., description="Quantization label (e.g., 'Q4_K_M')") + size_bytes: int = Field(0, description="File size in bytes") + + +class GgufVariantsResponse(BaseModel): + """Response for listing GGUF quantization variants in a HuggingFace repo.""" + repo_id: str = Field(..., description="HuggingFace repo ID") + variants: List[GgufVariantDetail] = Field(default_factory=list, description="Available GGUF variants") + has_vision: bool = Field(False, description="Whether the model has vision support (mmproj files)") + default_variant: Optional[str] = Field(None, description="Recommended default quantization variant") + + class LocalModelInfo(BaseModel): """Discovered local model candidate.""" id: str = Field(..., description="Identifier to use for loading/training") diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index c30a1638d4..bc450add0d 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -23,6 +23,7 @@ if str(backend_path) not in sys.path: # Import backend functions try: from core.inference import get_inference_backend + from core.inference.llama_cpp import LlamaCppBackend from utils.models import ModelConfig from utils.inference import load_inference_config except ImportError: @@ -30,6 +31,7 @@ except ImportError: if str(parent_backend) not in sys.path: sys.path.insert(0, str(parent_backend)) from core.inference import get_inference_backend + from core.inference.llama_cpp import LlamaCppBackend from utils.models import ModelConfig from utils.inference import load_inference_config @@ -61,60 +63,126 @@ if not logger.handlers: logger.addHandler(handler) logger.setLevel(logging.INFO) +# GGUF inference backend (llama-server) +_llama_cpp_backend = LlamaCppBackend() + +def get_llama_cpp_backend() -> LlamaCppBackend: + return _llama_cpp_backend + @router.post("/load", response_model=LoadResponse) async def load_model(request: LoadRequest): """ Load a model for inference. - + The model_path should be a clean identifier from GET /models/list. Returns inference configuration parameters (temperature, top_p, top_k, min_p) from the model's YAML config, falling back to default.yaml for missing values. + + GGUF models are loaded via llama-server (llama.cpp) instead of Unsloth. """ try: - backend = get_inference_backend() - # Create config using clean factory method # is_lora is auto-detected from adapter_config.json on disk/HF config = ModelConfig.from_identifier( model_id=request.model_path, hf_token=request.hf_token, + gguf_variant=request.gguf_variant, ) - + if not config: raise HTTPException( status_code=400, detail=f"Invalid model identifier: {request.model_path}" ) - - # Load the model + + # ── GGUF path: load via llama-server ────────────────────── + if config.is_gguf: + llama_backend = get_llama_cpp_backend() + unsloth_backend = get_inference_backend() + + # Unload any active Unsloth model first to free VRAM + if unsloth_backend.active_model_name: + logger.info(f"Unloading Unsloth model '{unsloth_backend.active_model_name}' before loading GGUF") + unsloth_backend.unload_model(unsloth_backend.active_model_name) + + # Route to HF mode or local mode based on config + if config.gguf_hf_repo: + # HF mode: llama-server downloads via -hf "repo:quant" + success = llama_backend.load_model( + hf_repo=config.gguf_hf_repo, + hf_variant=config.gguf_variant, + hf_token=request.hf_token, + model_identifier=config.identifier, + is_vision=config.is_vision, + n_ctx=request.max_seq_length, + ) + else: + # Local mode: llama-server loads via -m + success = llama_backend.load_model( + gguf_path=config.gguf_file, + model_identifier=config.identifier, + is_vision=config.is_vision, + n_ctx=request.max_seq_length, + ) + + if not success: + raise HTTPException( + status_code=500, + detail=f"Failed to load GGUF model: {config.display_name}" + ) + + logger.info(f"Loaded GGUF model via llama-server: {config.identifier}") + + inference_config = load_inference_config(config.identifier) + + return LoadResponse( + status="loaded", + model=config.identifier, + display_name=config.display_name, + is_vision=config.is_vision, + is_lora=False, + is_gguf=True, + inference=inference_config, + ) + + # ── Standard path: load via Unsloth/transformers ────────── + backend = get_inference_backend() + + # Unload any active GGUF model first + llama_backend = get_llama_cpp_backend() + if llama_backend.is_loaded: + logger.info("Unloading GGUF model before loading Unsloth model") + llama_backend.unload_model() + success = backend.load_model( config=config, max_seq_length=request.max_seq_length, load_in_4bit=request.load_in_4bit, hf_token=request.hf_token, ) - + if not success: raise HTTPException( status_code=500, detail=f"Failed to load model: {config.display_name}" ) - + logger.info(f"Loaded model: {config.identifier}") - + # Load inference configuration parameters inference_config = load_inference_config(config.identifier) - + return LoadResponse( status="loaded", model=config.identifier, display_name=config.display_name, is_vision=config.is_vision, is_lora=config.is_lora, + is_gguf=False, inference=inference_config, ) - + except HTTPException: raise except Exception as e: @@ -129,13 +197,22 @@ async def load_model(request: LoadRequest): async def unload_model(request: UnloadRequest): """ Unload a model from memory. + Routes to the correct backend (llama-server for GGUF, Unsloth otherwise). """ try: + # Check if the GGUF backend has this model loaded + llama_backend = get_llama_cpp_backend() + if llama_backend.is_loaded and llama_backend.model_identifier == request.model_path: + llama_backend.unload_model() + logger.info(f"Unloaded GGUF model: {request.model_path}") + return UnloadResponse(status="unloaded", model=request.model_path) + + # Otherwise, unload from Unsloth backend backend = get_inference_backend() backend.unload_model(request.model_path) logger.info(f"Unloaded model: {request.model_path}") return UnloadResponse(status="unloaded", model=request.model_path) - + except Exception as e: logger.error(f"Error unloading model: {e}", exc_info=True) raise HTTPException( @@ -221,22 +298,37 @@ async def generate_stream(request: GenerateRequest): async def get_status(): """ Get current inference backend status. + Reports whichever backend (Unsloth or llama-server) is currently active. """ try: + llama_backend = get_llama_cpp_backend() + + # If a GGUF model is loaded via llama-server, report that + if llama_backend.is_loaded: + return InferenceStatusResponse( + active_model=llama_backend.model_identifier, + is_vision=llama_backend.is_vision, + is_gguf=True, + loading=[], + loaded=[llama_backend.model_identifier], + ) + + # Otherwise, report Unsloth backend status backend = get_inference_backend() - + is_vision = False if backend.active_model_name: model_info = backend.models.get(backend.active_model_name, {}) is_vision = model_info.get("is_vision", False) - + return InferenceStatusResponse( active_model=backend.active_model_name, is_vision=is_vision, + is_gguf=False, loading=list(getattr(backend, 'loading_models', set())), loaded=list(backend.models.keys()), ) - + except Exception as e: logger.error(f"Error getting status: {e}", exc_info=True) raise HTTPException( @@ -315,29 +407,163 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque Streaming (default): returns SSE chunks matching OpenAI's format. Non-streaming: returns a single ChatCompletion JSON object. - """ - backend = get_inference_backend() - if not backend.active_model_name: - raise HTTPException( - status_code=400, - detail="No model loaded. Call POST /inference/load first.", - ) + Automatically routes to the correct backend: + - GGUF models → llama-server via LlamaCppBackend + - Other models → Unsloth/transformers via InferenceBackend + """ + llama_backend = get_llama_cpp_backend() + using_gguf = llama_backend.is_loaded + + # ── Determine which backend is active ───────────────────── + if using_gguf: + model_name = llama_backend.model_identifier or payload.model + else: + backend = get_inference_backend() + if not backend.active_model_name: + raise HTTPException( + status_code=400, + detail="No model loaded. Call POST /inference/load first.", + ) + model_name = backend.active_model_name or payload.model # ── Parse messages (handles multimodal content parts) ───── system_prompt, chat_messages, extracted_image_b64 = _extract_content_parts( payload.messages ) - # If no non-system messages were provided, error out if not chat_messages: raise HTTPException( status_code=400, detail="At least one non-system message is required.", ) - # ── Decode image (from content parts OR legacy field) ───── - # Content-part images take priority; fall back to legacy field + # ── GGUF path: proxy to llama-server /v1/chat/completions ── + if using_gguf: + # Reject images if this GGUF model doesn't support vision + image_b64 = extracted_image_b64 or payload.image_base64 + if image_b64 and not llama_backend.is_vision: + raise HTTPException( + status_code=400, + detail="Image provided but current GGUF model does not support vision.", + ) + + # Build message list with system prompt prepended + gguf_messages = [] + if system_prompt: + gguf_messages.append({"role": "system", "content": system_prompt}) + gguf_messages.extend(chat_messages) + + cancel_event = threading.Event() + + completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}" + created = int(time.time()) + + def gguf_generate(): + return llama_backend.generate_chat_completion( + messages=gguf_messages, + image_b64=image_b64, + temperature=payload.temperature, + top_p=payload.top_p, + top_k=payload.top_k, + min_p=payload.min_p, + max_tokens=payload.max_tokens or 512, + repetition_penalty=payload.repetition_penalty, + cancel_event=cancel_event, + ) + + if payload.stream: + async def gguf_stream_chunks(): + try: + # First chunk: role + first_chunk = ChatCompletionChunk( + id=completion_id, + created=created, + model=model_name, + choices=[ChunkChoice( + delta=ChoiceDelta(role="assistant"), + finish_reason=None, + )], + ) + yield f"data: {first_chunk.model_dump_json(exclude_none=True)}\n\n" + + # Content chunks — llama backend yields cumulative text + prev_text = "" + for cumulative in gguf_generate(): + if await request.is_disconnected(): + cancel_event.set() + return + new_text = cumulative[len(prev_text):] + prev_text = cumulative + if not new_text: + continue + chunk = ChatCompletionChunk( + id=completion_id, + created=created, + model=model_name, + choices=[ChunkChoice( + delta=ChoiceDelta(content=new_text), + finish_reason=None, + )], + ) + yield f"data: {chunk.model_dump_json(exclude_none=True)}\n\n" + + # Final chunk + final_chunk = ChatCompletionChunk( + id=completion_id, + created=created, + model=model_name, + choices=[ChunkChoice( + delta=ChoiceDelta(), + finish_reason="stop", + )], + ) + yield f"data: {final_chunk.model_dump_json(exclude_none=True)}\n\n" + yield "data: [DONE]\n\n" + + except asyncio.CancelledError: + cancel_event.set() + raise + except Exception as e: + logger.error(f"Error during GGUF streaming: {e}", exc_info=True) + error_chunk = { + "error": {"message": str(e), "type": "server_error"}, + } + yield f"data: {json.dumps(error_chunk)}\n\n" + + return StreamingResponse( + gguf_stream_chunks(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + }, + ) + else: + try: + full_text = "" + for token in gguf_generate(): + full_text = token + + response = ChatCompletion( + id=completion_id, + created=created, + model=model_name, + choices=[CompletionChoice( + message=CompletionMessage(content=full_text), + finish_reason="stop", + )], + ) + return JSONResponse(content=response.model_dump()) + + except Exception as e: + logger.error(f"Error during GGUF completion: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + # ── Standard Unsloth path ───────────────────────────────── + + # Decode image (from content parts OR legacy field) image_b64 = extracted_image_b64 or payload.image_base64 image = None @@ -363,7 +589,7 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque except Exception as e: raise HTTPException(status_code=400, detail=f"Failed to decode image: {e}") - # ── Shared generation kwargs ────────────────────────────── + # Shared generation kwargs gen_kwargs = dict( messages=chat_messages, system_prompt=system_prompt, @@ -376,11 +602,10 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque repetition_penalty=payload.repetition_penalty, ) - # ── Choose generation path (adapter-controlled or standard) ── + # Choose generation path (adapter-controlled or standard) cancel_event = threading.Event() if payload.use_adapter is not None: - # Compare mode: toggle adapter state atomically with generation def generate(): return backend.generate_with_adapter_control( use_adapter=payload.use_adapter, @@ -388,11 +613,9 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque **gen_kwargs, ) else: - # Standard path: no adapter toggling def generate(): return backend.generate_chat_response(cancel_event=cancel_event, **gen_kwargs) - model_name = backend.active_model_name or payload.model completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}" created = int(time.time()) @@ -400,7 +623,6 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque if payload.stream: async def stream_chunks(): try: - # First chunk: send the role first_chunk = ChatCompletionChunk( id=completion_id, created=created, @@ -412,8 +634,6 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque ) yield f"data: {first_chunk.model_dump_json(exclude_none=True)}\n\n" - # Content chunks — generate_chat_response yields cumulative - # text, so we diff to get incremental deltas. prev_text = "" for cumulative in generate(): if await request.is_disconnected(): @@ -435,7 +655,6 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque ) yield f"data: {chunk.model_dump_json(exclude_none=True)}\n\n" - # Final chunk: finish_reason = stop final_chunk = ChatCompletionChunk( id=completion_id, created=created, @@ -475,7 +694,7 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque try: full_text = "" for token in generate(): - full_text = token # generate_stream yields cumulative text + full_text = token response = ChatCompletion( id=completion_id, diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py index 8c96ee5f28..2c7f0e846f 100644 --- a/studio/backend/routes/models.py +++ b/studio/backend/routes/models.py @@ -23,8 +23,10 @@ try: get_base_model_from_lora, is_vision_model, scan_checkpoints, + list_gguf_variants, ModelConfig, ) + from utils.models.model_config import _pick_best_gguf, _extract_quant_label from core.inference import get_inference_backend except ImportError: # Fallback: try to import from parent directory @@ -38,8 +40,10 @@ except ImportError: get_base_model_from_lora, is_vision_model, scan_checkpoints, + list_gguf_variants, ModelConfig, ) + from utils.models.model_config import _pick_best_gguf, _extract_quant_label from core.inference import get_inference_backend from models import ( @@ -53,6 +57,7 @@ from models import ( LoRAInfo, ModelListResponse, ) +from models.models import GgufVariantDetail, GgufVariantsResponse from models.responses import LoRABaseModelResponse, VisionCheckResponse router = APIRouter() @@ -90,6 +95,7 @@ def _scan_models_dir(models_dir: Path) -> List[LocalModelInfo]: or (child / "adapter_config.json").exists() or any(child.glob("*.safetensors")) or any(child.glob("*.bin")) + or any(child.glob("*.gguf")) ) if not has_model_files: continue @@ -106,6 +112,23 @@ def _scan_models_dir(models_dir: Path) -> List[LocalModelInfo]: updated_at=updated_at, ), ) + # Also scan for standalone .gguf files directly in the models directory + for gguf_file in models_dir.glob("*.gguf"): + if gguf_file.is_file(): + try: + updated_at = gguf_file.stat().st_mtime + except OSError: + updated_at = None + found.append( + LocalModelInfo( + id=str(gguf_file), + display_name=gguf_file.stem, + path=str(gguf_file), + source="models_dir", + updated_at=updated_at, + ), + ) + return found @@ -399,6 +422,49 @@ async def check_vision_model( detail=f"Failed to check vision model: {str(e)}" ) +@router.get("/gguf-variants", response_model=GgufVariantsResponse) +async def get_gguf_variants( + repo_id: str = Query(..., description="HuggingFace repo ID (e.g. 'unsloth/gemma-3-4b-it-GGUF')"), + hf_token: Optional[str] = Query(None, description="HuggingFace token for private repos"), + current_subject: str = Depends(get_current_subject), +): + """ + List available GGUF quantization variants for a HuggingFace repo. + + Returns all available quantization variants (Q4_K_M, Q8_0, BF16, etc.) + with file sizes, whether the model supports vision, and the recommended + default variant. + """ + try: + variants, has_vision = list_gguf_variants(repo_id, hf_token=hf_token) + + # Determine default variant + filenames = [v.filename for v in variants] + best = _pick_best_gguf(filenames) + default_variant = _extract_quant_label(best) if best else None + + return GgufVariantsResponse( + repo_id=repo_id, + variants=[ + GgufVariantDetail( + filename=v.filename, + quant=v.quant, + size_bytes=v.size_bytes, + ) + for v in variants + ], + has_vision=has_vision, + default_variant=default_variant, + ) + + except Exception as e: + logger.error(f"Error listing GGUF variants for '{repo_id}': {e}", exc_info=True) + raise HTTPException( + status_code=500, + detail=f"Failed to list GGUF variants: {str(e)}", + ) + + @router.get("/checkpoints", response_model=CheckpointListResponse) async def list_checkpoints( outputs_dir: str = Query( diff --git a/studio/backend/utils/models/__init__.py b/studio/backend/utils/models/__init__.py index 006deb99c0..92e65cf67c 100644 --- a/studio/backend/utils/models/__init__.py +++ b/studio/backend/utils/models/__init__.py @@ -3,12 +3,14 @@ Model and LoRA configuration handling """ from .model_config import ( ModelConfig, + GgufVariantInfo, is_vision_model, scan_trained_loras, scan_exported_models, load_model_defaults, get_base_model_from_lora, load_model_config, + list_gguf_variants, MODEL_NAME_MAPPING, UI_STATUS_INDICATORS, ) @@ -16,12 +18,14 @@ from .checkpoints import scan_checkpoints __all__ = [ 'ModelConfig', + 'GgufVariantInfo', 'is_vision_model', 'scan_trained_loras', 'scan_exported_models', 'load_model_defaults', 'get_base_model_from_lora', 'load_model_config', + 'list_gguf_variants', 'MODEL_NAME_MAPPING', 'UI_STATUS_INDICATORS', 'scan_checkpoints', diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index f7b95fad11..98e84c7d6c 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -422,6 +422,179 @@ def is_vision_model(model_name: str, hf_token: Optional[str] = None) -> bool: pass +def detect_gguf_model(path: str) -> Optional[str]: + """ + Check if the given local path is or contains a GGUF model file. + + Handles two cases: + 1. path is a direct .gguf file path + 2. path is a directory containing .gguf files + + Returns the full path to the .gguf file if found, None otherwise. + For HuggingFace repo detection, use detect_gguf_model_remote() instead. + """ + p = Path(path) + + # Case 1: direct .gguf file + if p.suffix == ".gguf" and p.is_file(): + return str(p.resolve()) + + # Case 2: directory containing .gguf files + if p.is_dir(): + gguf_files = sorted(p.glob("*.gguf"), key=lambda f: f.stat().st_size, reverse=True) + if gguf_files: + return str(gguf_files[0].resolve()) + + return None + + +# Preferred GGUF quantization levels, in descending priority. +# Q4_K_M is a good default: small, fast, acceptable quality. +_GGUF_QUANT_PREFERENCE = [ + "Q4_K_M", "Q4_K_S", "Q5_K_M", "Q5_K_S", + "Q6_K", "Q8_0", "Q3_K_M", "Q3_K_L", "Q2_K", + "F16", "BF16", "F32", +] + + +def _pick_best_gguf(filenames: list[str]) -> Optional[str]: + """ + Pick the best GGUF file from a list of filenames. + + Prefers quantization levels in _GGUF_QUANT_PREFERENCE order. + Falls back to the first .gguf file found. + """ + gguf_files = [f for f in filenames if f.endswith(".gguf")] + if not gguf_files: + return None + + # Try preferred quantization levels + for quant in _GGUF_QUANT_PREFERENCE: + for f in gguf_files: + if quant in f: + return f + + # Fallback: first GGUF file + return gguf_files[0] + + +@dataclass +class GgufVariantInfo: + """A single GGUF quantization variant from a HuggingFace repo.""" + filename: str # e.g., "gemma-3-4b-it-Q4_K_M.gguf" + quant: str # e.g., "Q4_K_M" (extracted from filename) + size_bytes: int # file size + + +def _extract_quant_label(filename: str) -> str: + """ + Extract quantization label like Q4_K_M, IQ4_XS, BF16 from a GGUF filename. + + Examples: + "gemma-3-4b-it-Q4_K_M.gguf" → "Q4_K_M" + "model-IQ4_NL.gguf" → "IQ4_NL" + "model-BF16.gguf" → "BF16" + "model-UD-IQ1_S.gguf" → "UD-IQ1_S" + """ + import re + stem = filename.rsplit(".", 1)[0] # Remove .gguf + # Match known quantization patterns (UD- prefix, IQ, Q, BF/F variants) + match = re.search( + r'(UD-)?' # Optional UD- prefix (Ultra Discrete) + r'(IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?' # IQ variants: IQ4_XS, IQ4_NL, IQ1_S + r'|Q[0-9]+_K_[A-Z]+' # K-quant: Q4_K_M, Q3_K_S + r'|Q[0-9]+_[0-9]+' # Standard: Q8_0, Q5_1 + r'|Q[0-9]+_K' # Short K-quant: Q6_K + r'|BF16|F16|F32)', # Full precision + stem, re.IGNORECASE, + ) + if match: + prefix = match.group(1) or "" + return f"{prefix}{match.group(2)}" + # Fallback: last segment after hyphen + return stem.split("-")[-1] + + +def list_gguf_variants( + repo_id: str, + hf_token: Optional[str] = None, +) -> tuple[list[GgufVariantInfo], bool]: + """ + List all GGUF quantization variants in a HuggingFace repo. + + Separates main model files from mmproj (vision projection) files. + The presence of mmproj files indicates a vision-capable model. + + Returns: + (variants, has_vision): list of non-mmproj GGUF variants + vision flag. + """ + from huggingface_hub import model_info as hf_model_info + + info = hf_model_info(repo_id, token=hf_token) + variants: list[GgufVariantInfo] = [] + has_vision = False + + for sibling in info.siblings: + fname = sibling.rfilename + if not fname.endswith(".gguf"): + continue + size = sibling.size or 0 + + # mmproj files are vision projection models, not main model files + if "mmproj" in fname.lower(): + has_vision = True + continue + + quant = _extract_quant_label(fname) + variants.append(GgufVariantInfo( + filename=fname, + quant=quant, + size_bytes=size, + )) + + return variants, has_vision + + +def detect_gguf_model_remote( + repo_id: str, + hf_token: Optional[str] = None, +) -> Optional[str]: + """ + Check if a HuggingFace repo contains GGUF files. + + Returns the filename of the best GGUF file in the repo, or None. + """ + try: + from huggingface_hub import model_info as hf_model_info + + info = hf_model_info(repo_id, token=hf_token) + repo_files = [s.rfilename for s in info.siblings] + return _pick_best_gguf(repo_files) + except Exception as e: + logger.debug(f"Could not check GGUF files for '{repo_id}': {e}") + return None + + +def download_gguf_file( + repo_id: str, + filename: str, + hf_token: Optional[str] = None, +) -> str: + """ + Download a specific GGUF file from a HuggingFace repo. + + Returns the local path to the downloaded file. + """ + from huggingface_hub import hf_hub_download + + local_path = hf_hub_download( + repo_id=repo_id, + filename=filename, + token=hf_token, + ) + return local_path + + def scan_trained_loras(outputs_dir: str = "./outputs") -> List[Tuple[str, str]]: """ Scan outputs folder for trained LoRA adapters. @@ -679,6 +852,10 @@ class ModelConfig: is_cached: bool # Is this already in HF cache? is_vision: bool # Is this a vision model? is_lora: bool # Is this a lora adapter? + is_gguf: bool = False # Is this a GGUF model? + gguf_file: Optional[str] = None # Full path to the .gguf file (local mode) + gguf_hf_repo: Optional[str] = None # HF repo ID for -hf mode (e.g. "unsloth/gemma-3-4b-it-GGUF") + gguf_variant: Optional[str] = None # Quantization variant (e.g. "Q4_K_M") base_model: Optional[str] = None # Base model (for LoRAs) @classmethod @@ -734,37 +911,102 @@ class ModelConfig: cls, model_id: str, hf_token: Optional[str] = None, - is_lora: bool = False + is_lora: bool = False, + gguf_variant: Optional[str] = None, ) -> Optional['ModelConfig']: """ Create ModelConfig from a clean model identifier. - + For FastAPI routes where the frontend sends sanitized model paths. No Gradio dropdown parsing - expects clean identifiers like: - "unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit" - "./outputs/my_lora_adapter" - "/absolute/path/to/model" - + Args: model_id: Clean model identifier (HF repo name or local path) hf_token: Optional HF token for vision detection on gated models is_lora: Whether this is a LoRA adapter - + gguf_variant: Optional GGUF quantization variant (e.g. "Q4_K_M"). + For remote GGUF repos, specifies which quant to load via -hf. + If None, auto-selects using _pick_best_gguf(). + Returns: ModelConfig or None if configuration cannot be created """ if not model_id or not model_id.strip(): return None - + identifier = model_id.strip() is_local = is_local_path(identifier) path = normalize_path(identifier) if is_local else identifier - + # Add unsloth/ prefix for shorthand HF models if not is_local and "/" not in identifier: identifier = f"unsloth/{identifier}" path = identifier - + + # Auto-detect GGUF models (check before LoRA/vision detection) + if is_local: + gguf_file = detect_gguf_model(path) + if gguf_file: + display_name = Path(gguf_file).stem + logger.info(f"Detected local GGUF model: {gguf_file}") + return cls( + identifier=identifier, + display_name=display_name, + path=path, + is_local=True, + is_cached=True, + is_vision=False, + is_lora=False, + is_gguf=True, + gguf_file=gguf_file, + ) + else: + # Check if the HF repo contains GGUF files + gguf_filename = detect_gguf_model_remote(identifier, hf_token=hf_token) + if gguf_filename: + # Preflight: verify llama-server binary exists BEFORE user waits + # for a multi-GB download that llama-server handles natively + from core.inference.llama_cpp import LlamaCppBackend + if not LlamaCppBackend._find_llama_server_binary(): + raise RuntimeError( + "llama-server binary not found — cannot load GGUF models. " + "Run setup.sh to build it, or set LLAMA_SERVER_PATH." + ) + + # Use list_gguf_variants() to detect vision & resolve variant + variants, has_vision = list_gguf_variants(identifier, hf_token=hf_token) + variant = gguf_variant + if not variant: + # Auto-select best quantization + variant_filenames = [v.filename for v in variants] + best = _pick_best_gguf(variant_filenames) + if best: + variant = _extract_quant_label(best) + else: + variant = "Q4_K_M" # Fallback — llama-server's own default + + display_name = f"{identifier.split('/')[-1]} ({variant})" + logger.info( + f"Detected remote GGUF repo '{identifier}', " + f"variant={variant}, vision={has_vision}" + ) + return cls( + identifier=identifier, + display_name=display_name, + path=identifier, + is_local=False, + is_cached=False, + is_vision=has_vision, + is_lora=False, + is_gguf=True, + gguf_file=None, + gguf_hf_repo=identifier, + gguf_variant=variant, + ) + # Auto-detect LoRA for local paths (check adapter_config.json on disk) if not is_lora and is_local: detected_base = get_base_model_from_lora(path) diff --git a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx index 441fdf0f02..02205248ea 100644 --- a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx +++ b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx @@ -5,6 +5,8 @@ import { TooltipContent, TooltipTrigger, } from "@/components/ui/tooltip"; +import { listGgufVariants } from "@/features/chat/api/chat-api"; +import type { GgufVariantDetail } from "@/features/chat/types/api"; import { useDebouncedValue, useGpuInfo, @@ -17,7 +19,7 @@ import type { VramFitStatus } from "@/lib/vram"; import { checkVramFit, estimateLoadingVram } from "@/lib/vram"; import { Search01Icon } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; -import { useMemo, useState, type ReactNode } from "react"; +import { useCallback, useEffect, useMemo, useState, type ReactNode } from "react"; import type { LoraModelOption, ModelOption, @@ -36,6 +38,15 @@ function ListLabel({ children }: { children: ReactNode }) { ); } +/** Format bytes to a human-readable size string. */ +function formatBytes(bytes: number): string { + if (bytes === 0) return "0 B"; + const units = ["B", "KB", "MB", "GB", "TB"]; + const i = Math.floor(Math.log(bytes) / Math.log(1024)); + const value = bytes / 1024 ** i; + return `${value.toFixed(value < 10 ? 1 : 0)} ${units[i]}`; +} + function ModelRow({ label, meta, @@ -114,6 +125,124 @@ function ModelRow({ return content; } +// ── GGUF Variant Expander ──────────────────────────────────── + +function GgufVariantExpander({ + repoId, + onSelect, +}: { + repoId: string; + onSelect: (id: string, meta: ModelSelectorChangeMeta) => void; +}) { + const [variants, setVariants] = useState(null); + const [defaultVariant, setDefaultVariant] = useState(null); + const [hasVision, setHasVision] = useState(false); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + let canceled = false; + setLoading(true); + setError(null); + + listGgufVariants(repoId) + .then((res) => { + if (canceled) return; + setVariants(res.variants); + setDefaultVariant(res.default_variant); + setHasVision(res.has_vision); + }) + .catch((err) => { + if (canceled) return; + setError(err instanceof Error ? err.message : "Failed to load variants"); + }) + .finally(() => { + if (!canceled) setLoading(false); + }); + + return () => { + canceled = true; + }; + }, [repoId]); + + const handleVariantClick = useCallback( + (quant: string) => { + onSelect(repoId, { + source: "hub", + isLora: false, + ggufVariant: quant, + }); + }, + [repoId, onSelect], + ); + + if (loading) { + return ( +
+ + Loading variants… +
+ ); + } + + if (error) { + return ( +
{error}
+ ); + } + + if (!variants || variants.length === 0) { + return ( +
+ No GGUF variants found. +
+ ); + } + + return ( +
+
+ + Quantizations + + {hasVision && ( + Vision + )} +
+ {variants.map((v) => ( + + ))} +
+ ); +} + +// ── Detect GGUF repos by naming convention ──────────────────── + +function isGgufRepo(id: string): boolean { + return id.toUpperCase().includes("-GGUF"); +} + +// ── Hub Model Picker ────────────────────────────────────────── + export function HubModelPicker({ models, value, @@ -130,6 +259,9 @@ export function HubModelPicker({ debouncedQuery, ); + // Track which GGUF repo is expanded for variant selection + const [expandedGguf, setExpandedGguf] = useState(null); + const recommendedIds = useMemo( () => dedupe([...models.map((model) => model.id), value ?? ""]), [models, value], @@ -199,6 +331,19 @@ export function HubModelPicker({ const { scrollRef, sentinelRef } = useInfiniteScroll(fetchMore, results.length); + /** Handle clicking a model row — GGUF repos expand, others load directly. */ + const handleModelClick = useCallback( + (id: string) => { + if (isGgufRepo(id)) { + // Toggle GGUF variant expander + setExpandedGguf((prev) => (prev === id ? null : id)); + } else { + onSelect(id, { source: "hub", isLora: false }); + } + }, + [onSelect], + ); + return (
@@ -230,18 +375,24 @@ export function HubModelPicker({ recommendedIds.map((id) => { const vram = recommendedVramMap.get(id); return ( - - onSelect(id, { source: "hub", isLora: false }) - } - vramStatus={vram?.status ?? null} - vramEst={vram?.est} - gpuGb={gpu.available ? gpu.memoryTotalGb : undefined} - /> +
+ handleModelClick(id)} + vramStatus={isGgufRepo(id) ? null : vram?.status ?? null} + vramEst={isGgufRepo(id) ? undefined : vram?.est} + gpuGb={gpu.available ? gpu.memoryTotalGb : undefined} + /> + {expandedGguf === id && ( + + )} +
); }) )} @@ -259,18 +410,24 @@ export function HubModelPicker({ hfIds.map((id) => { const vram = vramMap.get(id); return ( - - onSelect(id, { source: "hub", isLora: false }) - } - vramStatus={vram?.status ?? null} - vramEst={vram?.est} - gpuGb={gpu.available ? gpu.memoryTotalGb : undefined} - /> +
+ handleModelClick(id)} + vramStatus={isGgufRepo(id) ? null : vram?.status ?? null} + vramEst={isGgufRepo(id) ? undefined : vram?.est} + gpuGb={gpu.available ? gpu.memoryTotalGb : undefined} + /> + {expandedGguf === id && ( + + )} +
); }) )} @@ -393,4 +550,3 @@ export function LoraModelPicker({
); } - diff --git a/studio/frontend/src/components/assistant-ui/model-selector/types.ts b/studio/frontend/src/components/assistant-ui/model-selector/types.ts index b8df0f6c6c..43f5e935b3 100644 --- a/studio/frontend/src/components/assistant-ui/model-selector/types.ts +++ b/studio/frontend/src/components/assistant-ui/model-selector/types.ts @@ -17,5 +17,6 @@ export interface LoraModelOption extends ModelOption { export interface ModelSelectorChangeMeta { source: "hub" | "lora" | "exported"; isLora: boolean; + ggufVariant?: string; } diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts index 72baf9a6f6..5d5a9551ef 100644 --- a/studio/frontend/src/features/chat/api/chat-api.ts +++ b/studio/frontend/src/features/chat/api/chat-api.ts @@ -1,5 +1,6 @@ import { authFetch } from "@/features/auth"; import type { + GgufVariantsResponse, InferenceStatusResponse, ListLorasResponse, ListModelsResponse, @@ -74,6 +75,16 @@ export async function unloadModel(payload: UnloadModelRequest): Promise { await parseJsonOrThrow(response); } +export async function listGgufVariants( + repoId: string, + hfToken?: string, +): Promise { + const params = new URLSearchParams({ repo_id: repoId }); + if (hfToken) params.set("hf_token", hfToken); + const response = await authFetch(`/api/models/gguf-variants?${params}`); + return parseJsonOrThrow(response); +} + function parseSseEvent(rawEvent: string): string[] { const dataLines: string[] = []; for (const line of rawEvent.split(/\r?\n/)) { diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx index 1ef3c6bec5..7e986721ab 100644 --- a/studio/frontend/src/features/chat/chat-page.tsx +++ b/studio/frontend/src/features/chat/chat-page.tsx @@ -335,7 +335,7 @@ export function ChatPage(): ReactElement { }, [inferenceParams.checkpoint, lorasFromStore]); const handleCheckpointChange = useCallback( - (value: string, meta?: { isLora: boolean }) => { + (value: string, meta?: { isLora: boolean; ggufVariant?: string }) => { const currentCheckpoint = useChatRuntimeStore.getState().params.checkpoint; if (!value || value === currentCheckpoint) return; @@ -367,10 +367,10 @@ export function ChatPage(): ReactElement { duration: 6000, }); } - await selectModel({ id: value, isLora: meta?.isLora, + ggufVariant: meta?.ggufVariant, }); })(); }, diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts index aafbf00310..c3464f9071 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts @@ -20,6 +20,7 @@ const DEFAULT_MODEL_MAX_SEQ_LENGTH = 2048; type SelectedModelInput = { id: string; isLora?: boolean; + ggufVariant?: string; loadingDescription?: string; }; @@ -42,11 +43,13 @@ function stripTrailingEpoch(input: string): string { function describeModel(model: { is_lora?: boolean; is_vision?: boolean; + is_gguf?: boolean; }): string | undefined { const tags: string[] = []; + if (model.is_gguf) tags.push("GGUF"); if (model.is_lora) tags.push("LoRA"); if (model.is_vision) tags.push("Vision"); - if (!model.is_lora && !model.is_vision) tags.push("Base"); + if (!model.is_lora && !model.is_vision && !model.is_gguf) tags.push("Base"); return tags.join(" · "); } @@ -55,6 +58,7 @@ function toChatModelSummary(model: { name?: string | null; is_lora?: boolean; is_vision?: boolean; + is_gguf?: boolean; }): ChatModelSummary { return { id: model.id, @@ -62,6 +66,7 @@ function toChatModelSummary(model: { description: describeModel(model), isLora: Boolean(model.is_lora), isVision: Boolean(model.is_vision), + isGguf: Boolean(model.is_gguf), }; } @@ -160,6 +165,8 @@ export function useChatModelRuntime() { const explicitIsLora = typeof selection === "string" ? undefined : selection.isLora; + const ggufVariant = + typeof selection === "string" ? undefined : selection.ggufVariant; const extraLoadingDescription = typeof selection === "string" ? undefined : selection.loadingDescription; const model = models.find((entry) => entry.id === modelId); @@ -193,6 +200,7 @@ export function useChatModelRuntime() { max_seq_length: DEFAULT_MODEL_MAX_SEQ_LENGTH, load_in_4bit: true, is_lora: isLora, + gguf_variant: ggufVariant ?? null, }); const currentParams = useChatRuntimeStore.getState().params; diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index 003c3b3629..5dd0cd7a6b 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -3,6 +3,7 @@ export interface BackendModelDetails { name?: string | null; is_vision?: boolean; is_lora?: boolean; + is_gguf?: boolean; } export interface ListModelsResponse { @@ -29,6 +30,20 @@ export interface LoadModelRequest { max_seq_length: number; load_in_4bit: boolean; is_lora: boolean; + gguf_variant?: string | null; +} + +export interface GgufVariantDetail { + filename: string; + quant: string; + size_bytes: number; +} + +export interface GgufVariantsResponse { + repo_id: string; + variants: GgufVariantDetail[]; + has_vision: boolean; + default_variant: string | null; } export interface LoadModelResponse { @@ -37,6 +52,7 @@ export interface LoadModelResponse { display_name: string; is_vision: boolean; is_lora: boolean; + is_gguf?: boolean; inference?: { temperature?: number; top_p?: number; @@ -52,6 +68,7 @@ export interface UnloadModelRequest { export interface InferenceStatusResponse { active_model: string | null; is_vision: boolean; + is_gguf?: boolean; loading: string[]; loaded: string[]; } diff --git a/studio/frontend/src/features/chat/types/runtime.ts b/studio/frontend/src/features/chat/types/runtime.ts index 48f147d9bc..953f4ebbaa 100644 --- a/studio/frontend/src/features/chat/types/runtime.ts +++ b/studio/frontend/src/features/chat/types/runtime.ts @@ -26,6 +26,7 @@ export interface ChatModelSummary { description?: string; isVision: boolean; isLora: boolean; + isGguf?: boolean; } export interface ChatLoraSummary { diff --git a/studio/frontend/src/features/studio/sections/model-section.tsx b/studio/frontend/src/features/studio/sections/model-section.tsx index 5157e9f8fc..5b88664fd1 100644 --- a/studio/frontend/src/features/studio/sections/model-section.tsx +++ b/studio/frontend/src/features/studio/sections/model-section.tsx @@ -172,14 +172,25 @@ export function ModelSection() { return ids; }, [hfResults, selectedModel]); + // Filter out GGUF models — they can't be used for training + const trainableLocalModels = useMemo( + () => + localModels.filter((m) => { + if (m.path.endsWith(".gguf")) return false; + if (m.id.toLowerCase().includes("-gguf")) return false; + return true; + }), + [localModels], + ); + const localMetaById = useMemo(() => { const map = new Map(); - for (const model of localModels) map.set(model.id, model); + for (const model of trainableLocalModels) map.set(model.id, model); return map; - }, [localModels]); + }, [trainableLocalModels]); const localResultIds = useMemo(() => { - const ids = localModels.map((model) => model.id); + const ids = trainableLocalModels.map((model) => model.id); const manual = localModelInput.trim(); if (manual && !ids.includes(manual)) { ids.unshift(manual); @@ -346,8 +357,8 @@ export function ModelSection() {

{localModelsError}

) : (

- {localModels.length > 0 - ? `${localModels.length} local/cached models found` + {trainableLocalModels.length > 0 + ? `${trainableLocalModels.length} local/cached models found` : "No local models found. Enter path manually."}

)}