Merge pull request #261 from unslothai/feat/gguf-llama-cpp-inference
Add GGUF model inference via llama-server with quantization variant selection
This commit is contained in:
Roland Tannous
GitHub
commit
c5f0da7873
20 changed files with 1377 additions and 120 deletions
6
.gitignore
vendored
6
.gitignore
vendored
|
|
@ -25,6 +25,12 @@ unsloth_training_checkpoints/
|
|||
*.gguf
|
||||
*.safetensors
|
||||
|
||||
# llama.cpp build (built by setup.sh, shared with unsloth-zoo export)
|
||||
llama.cpp/
|
||||
|
||||
# Built binaries (llama-server etc.)
|
||||
bin/
|
||||
|
||||
# IDE / Editors
|
||||
.vscode/
|
||||
.idea/
|
||||
|
|
|
|||
94
setup.sh
94
setup.sh
|
|
@ -211,7 +211,99 @@ else
|
|||
fi
|
||||
fi
|
||||
|
||||
# ── 8. Add shell alias (skip in Colab) ──
|
||||
# ── 8. Build llama.cpp binaries for GGUF inference + export ──
|
||||
# Builds in-tree at $REPO/llama.cpp/. This directory is shared with
|
||||
# unsloth-zoo's GGUF export pipeline. We build:
|
||||
# - llama-server: for GGUF model inference
|
||||
# - llama-quantize: for GGUF export quantization (symlinked to root for check_llama_cpp())
|
||||
LLAMA_SERVER_BIN="$SCRIPT_DIR/llama.cpp/build/bin/llama-server"
|
||||
if [ -f "$LLAMA_SERVER_BIN" ]; then
|
||||
echo ""
|
||||
echo "✅ llama-server already exists at $LLAMA_SERVER_BIN"
|
||||
else
|
||||
# Check prerequisites
|
||||
if ! command -v cmake &>/dev/null; then
|
||||
echo ""
|
||||
echo "⚠️ cmake not found — skipping llama-server build (GGUF inference won't be available)"
|
||||
echo " Install cmake and re-run setup.sh to enable GGUF inference."
|
||||
elif ! command -v git &>/dev/null; then
|
||||
echo ""
|
||||
echo "⚠️ git not found — skipping llama-server build (GGUF inference won't be available)"
|
||||
else
|
||||
echo ""
|
||||
echo "Building llama-server for GGUF inference..."
|
||||
LLAMA_CPP_DIR="$SCRIPT_DIR/llama.cpp"
|
||||
|
||||
BUILD_OK=true
|
||||
if [ -d "$LLAMA_CPP_DIR/.git" ]; then
|
||||
echo " llama.cpp repo already cloned, pulling latest..."
|
||||
run_quiet "pull llama.cpp" git -C "$LLAMA_CPP_DIR" pull || true
|
||||
else
|
||||
# Remove any non-git llama.cpp directory (stale build artifacts)
|
||||
rm -rf "$LLAMA_CPP_DIR"
|
||||
run_quiet "clone llama.cpp" git clone --depth 1 https://github.com/ggml-org/llama.cpp.git "$LLAMA_CPP_DIR" || BUILD_OK=false
|
||||
fi
|
||||
|
||||
if [ "$BUILD_OK" = true ]; then
|
||||
CMAKE_ARGS=""
|
||||
# Detect CUDA: check nvcc on PATH, then common install locations
|
||||
NVCC_PATH=""
|
||||
if command -v nvcc &>/dev/null; then
|
||||
NVCC_PATH="$(command -v nvcc)"
|
||||
elif [ -x /usr/local/cuda/bin/nvcc ]; then
|
||||
NVCC_PATH="/usr/local/cuda/bin/nvcc"
|
||||
export PATH="/usr/local/cuda/bin:$PATH"
|
||||
elif ls /usr/local/cuda-*/bin/nvcc &>/dev/null 2>&1; then
|
||||
# Pick the newest cuda-XX.X directory
|
||||
NVCC_PATH="$(ls -d /usr/local/cuda-*/bin/nvcc 2>/dev/null | sort -V | tail -1)"
|
||||
export PATH="$(dirname "$NVCC_PATH"):$PATH"
|
||||
fi
|
||||
|
||||
if [ -n "$NVCC_PATH" ]; then
|
||||
echo " Building with CUDA support (nvcc: $NVCC_PATH)..."
|
||||
CMAKE_ARGS="-DGGML_CUDA=ON"
|
||||
elif [ -d /usr/local/cuda ] || nvidia-smi &>/dev/null; then
|
||||
echo " CUDA driver detected but nvcc not found — building CPU-only"
|
||||
echo " To enable GPU: install cuda-toolkit or add nvcc to PATH"
|
||||
else
|
||||
echo " Building CPU-only (no CUDA detected)..."
|
||||
fi
|
||||
|
||||
NCPU=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
|
||||
|
||||
run_quiet "cmake llama.cpp" cmake -S "$LLAMA_CPP_DIR" -B "$LLAMA_CPP_DIR/build" $CMAKE_ARGS || BUILD_OK=false
|
||||
fi
|
||||
|
||||
if [ "$BUILD_OK" = true ]; then
|
||||
run_quiet "build llama-server" cmake --build "$LLAMA_CPP_DIR/build" --config Release --target llama-server -j"$NCPU" || BUILD_OK=false
|
||||
fi
|
||||
|
||||
# Also build llama-quantize (needed by unsloth-zoo's GGUF export pipeline)
|
||||
if [ "$BUILD_OK" = true ]; then
|
||||
run_quiet "build llama-quantize" cmake --build "$LLAMA_CPP_DIR/build" --config Release --target llama-quantize -j"$NCPU" || true
|
||||
# Symlink to llama.cpp root — check_llama_cpp() looks for the binary there
|
||||
QUANTIZE_BIN="$LLAMA_CPP_DIR/build/bin/llama-quantize"
|
||||
if [ -f "$QUANTIZE_BIN" ]; then
|
||||
ln -sf build/bin/llama-quantize "$LLAMA_CPP_DIR/llama-quantize"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$BUILD_OK" = true ]; then
|
||||
if [ -f "$LLAMA_SERVER_BIN" ]; then
|
||||
echo "✅ llama-server built at $LLAMA_SERVER_BIN"
|
||||
else
|
||||
echo "⚠️ llama-server binary not found after build — GGUF inference won't be available"
|
||||
fi
|
||||
if [ -f "$LLAMA_CPP_DIR/llama-quantize" ]; then
|
||||
echo "✅ llama-quantize available for GGUF export"
|
||||
fi
|
||||
else
|
||||
echo "⚠️ llama-server build failed — GGUF inference won't be available, but everything else works"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── 9. Add shell alias (skip in Colab) ──
|
||||
# Note: venv activation does NOT persist across terminal sessions.
|
||||
# This alias hardcodes the venv python path so users don't need to activate.
|
||||
if [ "$IS_COLAB" = false ]; then
|
||||
|
|
|
|||
|
|
@ -397,53 +397,30 @@ class ExportBackend:
|
|||
|
||||
# Save locally if requested
|
||||
if save_directory:
|
||||
logger.info(f"Saving GGUF model locally to: {save_directory}")
|
||||
# Resolve to absolute path so unsloth's relative-path internals
|
||||
# (check_llama_cpp, use_local_gguf, _download_convert_hf_to_gguf)
|
||||
# all resolve against the repo root cwd, NOT the export directory.
|
||||
abs_save_dir = os.path.abspath(save_directory)
|
||||
logger.info(f"Saving GGUF model locally to: {abs_save_dir}")
|
||||
|
||||
# Create the directory if it doesn't exist
|
||||
os.makedirs(save_directory, exist_ok=True)
|
||||
os.makedirs(abs_save_dir, exist_ok=True)
|
||||
|
||||
# Get the base filename for the GGUF file
|
||||
import shutil
|
||||
original_dir = os.getcwd()
|
||||
# On WSL, patch out sudo check before llama.cpp build
|
||||
_apply_wsl_sudo_patch()
|
||||
|
||||
try:
|
||||
# Change to target directory
|
||||
os.chdir(save_directory)
|
||||
logger.info(f"Changed directory to: {save_directory}")
|
||||
# Pass absolute path — no os.chdir needed.
|
||||
# unsloth saves model files into this directory, while
|
||||
# check_llama_cpp("llama.cpp") resolves against cwd (repo root)
|
||||
# where setup.sh already built llama.cpp with quantizer.
|
||||
model_save_path = os.path.join(abs_save_dir, "model")
|
||||
self.current_model.save_pretrained_gguf(
|
||||
model_save_path,
|
||||
self.current_tokenizer,
|
||||
quantization_method=quant_method
|
||||
)
|
||||
|
||||
# On WSL, patch out sudo check before llama.cpp build
|
||||
_apply_wsl_sudo_patch()
|
||||
|
||||
# Now save (will save in current directory)
|
||||
self.current_model.save_pretrained_gguf(
|
||||
"model", # Base filename
|
||||
self.current_tokenizer,
|
||||
quantization_method=quant_method
|
||||
)
|
||||
|
||||
logger.info(f"GGUF model saved successfully in {save_directory}")
|
||||
|
||||
# Check if llama.cpp directory was created here
|
||||
llama_cpp_in_target = os.path.join(save_directory, "llama.cpp")
|
||||
llama_cpp_in_original = os.path.join(original_dir, "llama.cpp")
|
||||
|
||||
if os.path.exists(llama_cpp_in_target):
|
||||
logger.info(f"Found llama.cpp directory in {save_directory}")
|
||||
|
||||
# Remove llama.cpp from original directory if it exists
|
||||
if os.path.exists(llama_cpp_in_original):
|
||||
logger.info(f"Removing existing llama.cpp in {original_dir}")
|
||||
shutil.rmtree(llama_cpp_in_original)
|
||||
|
||||
# Move llama.cpp back to original directory
|
||||
logger.info(f"Moving llama.cpp to {original_dir}")
|
||||
shutil.move(llama_cpp_in_target, llama_cpp_in_original)
|
||||
logger.info(f"Successfully moved llama.cpp back to original directory")
|
||||
|
||||
finally:
|
||||
# Always change back to original directory
|
||||
os.chdir(original_dir)
|
||||
logger.info(f"Changed back to original directory: {original_dir}")
|
||||
logger.info(f"GGUF model saved successfully in {abs_save_dir}")
|
||||
|
||||
# Push to hub if requested
|
||||
if push_to_hub:
|
||||
|
|
|
|||
|
|
@ -2,8 +2,10 @@
|
|||
Inference submodule - Inference backend for model loading and generation
|
||||
"""
|
||||
from .inference import InferenceBackend, get_inference_backend
|
||||
from .llama_cpp import LlamaCppBackend
|
||||
|
||||
__all__ = [
|
||||
'InferenceBackend',
|
||||
'get_inference_backend',
|
||||
'LlamaCppBackend',
|
||||
]
|
||||
|
|
|
|||
416
studio/backend/core/inference/llama_cpp.py
Normal file
416
studio/backend/core/inference/llama_cpp.py
Normal file
|
|
@ -0,0 +1,416 @@
|
|||
"""
|
||||
llama-server inference backend for GGUF models.
|
||||
|
||||
Manages a llama-server subprocess and proxies chat completions
|
||||
through its OpenAI-compatible /v1/chat/completions endpoint.
|
||||
"""
|
||||
import atexit
|
||||
import json
|
||||
import logging
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Generator, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LlamaCppBackend:
|
||||
"""
|
||||
Manages a llama-server subprocess for GGUF model inference.
|
||||
|
||||
Lifecycle:
|
||||
1. load_model() — starts llama-server with the GGUF file
|
||||
2. generate_chat_completion() — proxies to /v1/chat/completions, streams back
|
||||
3. unload_model() — terminates llama-server subprocess
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._process: Optional[subprocess.Popen] = None
|
||||
self._port: Optional[int] = None
|
||||
self._model_identifier: Optional[str] = None
|
||||
self._gguf_path: Optional[str] = None
|
||||
self._hf_repo: Optional[str] = None
|
||||
self._hf_variant: Optional[str] = None
|
||||
self._is_vision: bool = False
|
||||
self._healthy = False
|
||||
self._lock = threading.Lock()
|
||||
|
||||
atexit.register(self._cleanup)
|
||||
|
||||
# ── Properties ────────────────────────────────────────────────
|
||||
|
||||
@property
|
||||
def is_loaded(self) -> bool:
|
||||
return self._process is not None and self._healthy
|
||||
|
||||
@property
|
||||
def base_url(self) -> str:
|
||||
return f"http://127.0.0.1:{self._port}"
|
||||
|
||||
@property
|
||||
def model_identifier(self) -> Optional[str]:
|
||||
return self._model_identifier
|
||||
|
||||
@property
|
||||
def is_vision(self) -> bool:
|
||||
return self._is_vision
|
||||
|
||||
# ── Binary discovery ──────────────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
def _find_llama_server_binary() -> Optional[str]:
|
||||
"""
|
||||
Locate the llama-server binary.
|
||||
|
||||
Search order:
|
||||
1. LLAMA_SERVER_PATH environment variable
|
||||
2. ./llama.cpp/build/bin/llama-server (built by setup.sh in-tree)
|
||||
3. llama-server on PATH (system install)
|
||||
4. ./bin/llama-server (legacy: extracted binary)
|
||||
"""
|
||||
import os
|
||||
|
||||
# 1. Env var
|
||||
env_path = os.environ.get("LLAMA_SERVER_PATH")
|
||||
if env_path and Path(env_path).is_file():
|
||||
return env_path
|
||||
|
||||
# Project root: llama_cpp.py → inference/ → core/ → backend/ → studio/ → root
|
||||
project_root = Path(__file__).resolve().parents[4]
|
||||
|
||||
# 2. In-tree llama.cpp build (setup.sh builds here)
|
||||
build_path = project_root / "llama.cpp" / "build" / "bin" / "llama-server"
|
||||
if build_path.is_file():
|
||||
return str(build_path)
|
||||
|
||||
# 3. System PATH
|
||||
system_path = shutil.which("llama-server")
|
||||
if system_path:
|
||||
return system_path
|
||||
|
||||
# 4. Legacy: extracted to bin/
|
||||
bin_path = project_root / "bin" / "llama-server"
|
||||
if bin_path.is_file():
|
||||
return str(bin_path)
|
||||
|
||||
return None
|
||||
|
||||
# ── Port allocation ───────────────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
def _find_free_port() -> int:
|
||||
"""Find an available TCP port."""
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("", 0))
|
||||
return s.getsockname()[1]
|
||||
|
||||
# ── Lifecycle ─────────────────────────────────────────────────
|
||||
|
||||
def load_model(
|
||||
self,
|
||||
*,
|
||||
# Local mode: pass a path to a .gguf file
|
||||
gguf_path: Optional[str] = None,
|
||||
# HF mode: let llama-server download via -hf "repo:quant"
|
||||
hf_repo: Optional[str] = None,
|
||||
hf_variant: Optional[str] = None,
|
||||
hf_token: Optional[str] = None,
|
||||
# Common
|
||||
model_identifier: str,
|
||||
is_vision: bool = False,
|
||||
n_ctx: int = 4096,
|
||||
n_gpu_layers: int = -1,
|
||||
n_threads: Optional[int] = None,
|
||||
) -> bool:
|
||||
"""
|
||||
Start llama-server with a GGUF model.
|
||||
|
||||
Two modes:
|
||||
- Local: ``gguf_path="/path/to/model.gguf"`` → uses ``-m``
|
||||
- HF: ``hf_repo="unsloth/gemma-3-4b-it-GGUF", hf_variant="Q4_K_M"`` → uses ``-hf``
|
||||
|
||||
In HF mode, llama-server handles downloading, caching, and
|
||||
auto-loading mmproj files for vision models.
|
||||
|
||||
Returns True if server started and health check passed.
|
||||
"""
|
||||
with self._lock:
|
||||
self._kill_process()
|
||||
|
||||
binary = self._find_llama_server_binary()
|
||||
if not binary:
|
||||
raise RuntimeError(
|
||||
"llama-server binary not found. "
|
||||
"Run setup.sh to build it, install llama.cpp, "
|
||||
"or set LLAMA_SERVER_PATH environment variable."
|
||||
)
|
||||
|
||||
self._port = self._find_free_port()
|
||||
|
||||
# Build command based on mode
|
||||
if hf_repo:
|
||||
hf_spec = f"{hf_repo}:{hf_variant}" if hf_variant else hf_repo
|
||||
cmd = [
|
||||
binary,
|
||||
"-hf", hf_spec,
|
||||
"--port", str(self._port),
|
||||
"-c", str(n_ctx),
|
||||
"-ngl", str(n_gpu_layers),
|
||||
]
|
||||
if hf_token:
|
||||
cmd.extend(["--hf-token", hf_token])
|
||||
elif gguf_path:
|
||||
if not Path(gguf_path).is_file():
|
||||
raise FileNotFoundError(f"GGUF file not found: {gguf_path}")
|
||||
cmd = [
|
||||
binary,
|
||||
"-m", gguf_path,
|
||||
"--port", str(self._port),
|
||||
"-c", str(n_ctx),
|
||||
"-ngl", str(n_gpu_layers),
|
||||
]
|
||||
else:
|
||||
raise ValueError("Either gguf_path or hf_repo must be provided")
|
||||
|
||||
if n_threads is not None:
|
||||
cmd.extend(["--threads", str(n_threads)])
|
||||
|
||||
logger.info(f"Starting llama-server: {' '.join(cmd)}")
|
||||
|
||||
# Set LD_LIBRARY_PATH so llama-server can find its shared libs
|
||||
# (libmtmd.so, libllama.so, etc.) which live next to the binary
|
||||
import os
|
||||
env = os.environ.copy()
|
||||
binary_dir = str(Path(binary).parent)
|
||||
existing_ld = env.get("LD_LIBRARY_PATH", "")
|
||||
env["LD_LIBRARY_PATH"] = f"{binary_dir}:{existing_ld}" if existing_ld else binary_dir
|
||||
|
||||
self._process = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
|
||||
self._gguf_path = gguf_path
|
||||
self._hf_repo = hf_repo
|
||||
self._hf_variant = hf_variant
|
||||
self._is_vision = is_vision
|
||||
self._model_identifier = model_identifier
|
||||
|
||||
# HF mode: llama-server downloads before becoming healthy — need longer timeout
|
||||
timeout = 600.0 if hf_repo else 120.0
|
||||
if not self._wait_for_health(timeout=timeout):
|
||||
self._kill_process()
|
||||
raise RuntimeError(
|
||||
"llama-server failed to start. "
|
||||
"Check that the GGUF file is valid and you have enough memory."
|
||||
)
|
||||
|
||||
self._healthy = True
|
||||
|
||||
logger.info(
|
||||
f"llama-server ready on port {self._port} "
|
||||
f"for model '{model_identifier}'"
|
||||
)
|
||||
return True
|
||||
|
||||
def unload_model(self) -> bool:
|
||||
"""Terminate the llama-server subprocess and clean up state."""
|
||||
with self._lock:
|
||||
self._kill_process()
|
||||
logger.info(f"Unloaded GGUF model: {self._model_identifier}")
|
||||
self._model_identifier = None
|
||||
self._gguf_path = None
|
||||
self._hf_repo = None
|
||||
self._hf_variant = None
|
||||
self._is_vision = False
|
||||
self._port = None
|
||||
self._healthy = False
|
||||
return True
|
||||
|
||||
def _kill_process(self):
|
||||
"""Terminate the subprocess if running."""
|
||||
if self._process is None:
|
||||
return
|
||||
try:
|
||||
self._process.terminate()
|
||||
self._process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("llama-server did not exit on SIGTERM, sending SIGKILL")
|
||||
self._process.kill()
|
||||
self._process.wait(timeout=5)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error killing llama-server process: {e}")
|
||||
finally:
|
||||
self._process = None
|
||||
|
||||
def _cleanup(self):
|
||||
"""atexit handler to ensure llama-server is terminated."""
|
||||
self._kill_process()
|
||||
|
||||
def _wait_for_health(self, timeout: float = 120.0, interval: float = 0.5) -> bool:
|
||||
"""
|
||||
Poll llama-server's /health endpoint until it responds 200.
|
||||
|
||||
Also monitors subprocess for early exit/crash.
|
||||
"""
|
||||
deadline = time.monotonic() + timeout
|
||||
url = f"http://127.0.0.1:{self._port}/health"
|
||||
|
||||
while time.monotonic() < deadline:
|
||||
# Check if process crashed
|
||||
if self._process.poll() is not None:
|
||||
# Read remaining output for error info
|
||||
output = self._process.stdout.read() if self._process.stdout else ""
|
||||
logger.error(
|
||||
f"llama-server exited with code {self._process.returncode}. "
|
||||
f"Output: {output[:2000]}"
|
||||
)
|
||||
return False
|
||||
|
||||
try:
|
||||
resp = httpx.get(url, timeout=2.0)
|
||||
if resp.status_code == 200:
|
||||
return True
|
||||
except (httpx.ConnectError, httpx.TimeoutException):
|
||||
pass
|
||||
|
||||
time.sleep(interval)
|
||||
|
||||
logger.error(f"llama-server health check timed out after {timeout}s")
|
||||
return False
|
||||
|
||||
# ── Message building (OpenAI format) ──────────────────────────
|
||||
|
||||
@staticmethod
|
||||
def _build_openai_messages(
|
||||
messages: list[dict],
|
||||
image_b64: Optional[str] = None,
|
||||
) -> list[dict]:
|
||||
"""
|
||||
Build OpenAI-format messages, optionally injecting an image_url
|
||||
content part into the last user message for vision models.
|
||||
|
||||
If no image is provided, returns messages as-is.
|
||||
"""
|
||||
if not image_b64:
|
||||
return messages
|
||||
|
||||
# Find the last user message and convert to multimodal content parts
|
||||
result = [msg.copy() for msg in messages]
|
||||
last_user_idx = None
|
||||
for i, msg in enumerate(result):
|
||||
if msg["role"] == "user":
|
||||
last_user_idx = i
|
||||
|
||||
if last_user_idx is not None:
|
||||
text_content = result[last_user_idx].get("content", "")
|
||||
result[last_user_idx]["content"] = [
|
||||
{"type": "text", "text": text_content},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": f"data:image/png;base64,{image_b64}",
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
return result
|
||||
|
||||
# ── Generation (proxy to llama-server) ────────────────────────
|
||||
|
||||
def generate_chat_completion(
|
||||
self,
|
||||
messages: list[dict],
|
||||
image_b64: Optional[str] = None,
|
||||
temperature: float = 0.7,
|
||||
top_p: float = 0.9,
|
||||
top_k: int = 40,
|
||||
min_p: float = 0.0,
|
||||
max_tokens: int = 512,
|
||||
repetition_penalty: float = 1.1,
|
||||
stop: Optional[list[str]] = None,
|
||||
cancel_event: Optional[threading.Event] = None,
|
||||
) -> Generator[str, None, None]:
|
||||
"""
|
||||
Send a chat completion request to llama-server and stream tokens back.
|
||||
|
||||
Uses /v1/chat/completions — llama-server handles chat template
|
||||
application and vision (multimodal image_url parts) natively.
|
||||
|
||||
Yields cumulative text (matching InferenceBackend's convention).
|
||||
"""
|
||||
if not self.is_loaded:
|
||||
raise RuntimeError("llama-server is not loaded")
|
||||
|
||||
openai_messages = self._build_openai_messages(messages, image_b64)
|
||||
|
||||
payload = {
|
||||
"messages": openai_messages,
|
||||
"stream": True,
|
||||
"temperature": temperature,
|
||||
"top_p": top_p,
|
||||
"top_k": top_k if top_k >= 0 else 0,
|
||||
"min_p": min_p,
|
||||
"max_tokens": max_tokens,
|
||||
"repeat_penalty": repetition_penalty,
|
||||
}
|
||||
if stop:
|
||||
payload["stop"] = stop
|
||||
|
||||
url = f"{self.base_url}/v1/chat/completions"
|
||||
cumulative = ""
|
||||
|
||||
try:
|
||||
with httpx.Client(timeout=None) as client:
|
||||
with client.stream("POST", url, json=payload) as response:
|
||||
if response.status_code != 200:
|
||||
error_body = response.read().decode()
|
||||
raise RuntimeError(
|
||||
f"llama-server returned {response.status_code}: {error_body}"
|
||||
)
|
||||
|
||||
buffer = ""
|
||||
for raw_chunk in response.iter_text():
|
||||
if cancel_event is not None and cancel_event.is_set():
|
||||
break
|
||||
|
||||
buffer += raw_chunk
|
||||
while "\n" in buffer:
|
||||
line, buffer = buffer.split("\n", 1)
|
||||
line = line.strip()
|
||||
|
||||
if not line:
|
||||
continue
|
||||
if line == "data: [DONE]":
|
||||
return
|
||||
if not line.startswith("data: "):
|
||||
continue
|
||||
|
||||
try:
|
||||
data = json.loads(line[6:])
|
||||
choices = data.get("choices", [])
|
||||
if choices:
|
||||
delta = choices[0].get("delta", {})
|
||||
token = delta.get("content", "")
|
||||
if token:
|
||||
cumulative += token
|
||||
yield cumulative
|
||||
except json.JSONDecodeError:
|
||||
logger.debug(f"Skipping malformed SSE line: {line[:100]}")
|
||||
|
||||
except httpx.ConnectError:
|
||||
raise RuntimeError("Lost connection to llama-server")
|
||||
except Exception as e:
|
||||
if cancel_event is not None and cancel_event.is_set():
|
||||
return
|
||||
raise
|
||||
|
|
@ -173,6 +173,15 @@ class UnslothTrainer:
|
|||
token=hf_token,
|
||||
)
|
||||
logger.info("Loaded vision model")
|
||||
|
||||
# Diagnostic: check if FastVisionModel returned a real Processor or a raw tokenizer
|
||||
from transformers import ProcessorMixin
|
||||
tok = self.tokenizer
|
||||
has_image_proc = isinstance(tok, ProcessorMixin) or hasattr(tok, "image_processor")
|
||||
print(f"\n[VLM Diagnostic] FastVisionModel returned: {type(tok).__name__}")
|
||||
print(f"[VLM Diagnostic] Is ProcessorMixin: {isinstance(tok, ProcessorMixin)}")
|
||||
print(f"[VLM Diagnostic] Has image_processor: {hasattr(tok, 'image_processor')}")
|
||||
print(f"[VLM Diagnostic] Usable as vision processor: {has_image_proc}\n")
|
||||
else:
|
||||
# Load text model - returns (model, tokenizer)
|
||||
self.model, self.tokenizer = FastLanguageModel.from_pretrained(
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ class LoadRequest(BaseModel):
|
|||
max_seq_length: int = Field(2048, ge=128, le=32768, description="Maximum sequence length")
|
||||
load_in_4bit: bool = Field(True, description="Load model in 4-bit quantization")
|
||||
is_lora: bool = Field(False, description="Whether this is a LoRA adapter")
|
||||
gguf_variant: Optional[str] = Field(None, description="GGUF quantization variant (e.g. 'Q4_K_M')")
|
||||
|
||||
|
||||
class UnloadRequest(BaseModel):
|
||||
|
|
@ -43,6 +44,7 @@ class LoadResponse(BaseModel):
|
|||
display_name: str = Field(..., description="Display name of the model")
|
||||
is_vision: bool = Field(False, description="Whether model is a vision model")
|
||||
is_lora: bool = Field(False, description="Whether model is a LoRA adapter")
|
||||
is_gguf: bool = Field(False, description="Whether model is a GGUF model (llama.cpp)")
|
||||
inference: dict = Field(..., description="Inference parameters (temperature, top_p, top_k, min_p)")
|
||||
|
||||
|
||||
|
|
@ -56,6 +58,7 @@ class InferenceStatusResponse(BaseModel):
|
|||
"""Current inference backend status"""
|
||||
active_model: Optional[str] = Field(None, description="Currently active model identifier")
|
||||
is_vision: bool = Field(False, description="Whether the active model is a vision model")
|
||||
is_gguf: bool = Field(False, description="Whether the active model is a GGUF model (llama.cpp)")
|
||||
loading: List[str] = Field(default_factory=list, description="Models currently being loaded")
|
||||
loaded: List[str] = Field(default_factory=list, description="Models currently loaded")
|
||||
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ class ModelDetails(BaseModel):
|
|||
config: Optional[Dict[str, Any]] = Field(None, description="Model configuration dictionary")
|
||||
is_vision: bool = Field(False, description="Whether model is a vision model")
|
||||
is_lora: bool = Field(False, description="Whether model is a LoRA adapter")
|
||||
is_gguf: bool = Field(False, description="Whether model is a GGUF model (llama.cpp format)")
|
||||
base_model: Optional[str] = Field(None, description="Base model if this is a LoRA adapter")
|
||||
|
||||
|
||||
|
|
@ -77,6 +78,21 @@ class ModelListResponse(BaseModel):
|
|||
default_models: List[str] = Field(default_factory=list, description="List of default model IDs")
|
||||
|
||||
|
||||
class GgufVariantDetail(BaseModel):
|
||||
"""A single GGUF quantization variant in a HuggingFace repo."""
|
||||
filename: str = Field(..., description="GGUF filename (e.g., 'gemma-3-4b-it-Q4_K_M.gguf')")
|
||||
quant: str = Field(..., description="Quantization label (e.g., 'Q4_K_M')")
|
||||
size_bytes: int = Field(0, description="File size in bytes")
|
||||
|
||||
|
||||
class GgufVariantsResponse(BaseModel):
|
||||
"""Response for listing GGUF quantization variants in a HuggingFace repo."""
|
||||
repo_id: str = Field(..., description="HuggingFace repo ID")
|
||||
variants: List[GgufVariantDetail] = Field(default_factory=list, description="Available GGUF variants")
|
||||
has_vision: bool = Field(False, description="Whether the model has vision support (mmproj files)")
|
||||
default_variant: Optional[str] = Field(None, description="Recommended default quantization variant")
|
||||
|
||||
|
||||
class LocalModelInfo(BaseModel):
|
||||
"""Discovered local model candidate."""
|
||||
id: str = Field(..., description="Identifier to use for loading/training")
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ if str(backend_path) not in sys.path:
|
|||
# Import backend functions
|
||||
try:
|
||||
from core.inference import get_inference_backend
|
||||
from core.inference.llama_cpp import LlamaCppBackend
|
||||
from utils.models import ModelConfig
|
||||
from utils.inference import load_inference_config
|
||||
except ImportError:
|
||||
|
|
@ -30,6 +31,7 @@ except ImportError:
|
|||
if str(parent_backend) not in sys.path:
|
||||
sys.path.insert(0, str(parent_backend))
|
||||
from core.inference import get_inference_backend
|
||||
from core.inference.llama_cpp import LlamaCppBackend
|
||||
from utils.models import ModelConfig
|
||||
from utils.inference import load_inference_config
|
||||
|
||||
|
|
@ -61,60 +63,126 @@ if not logger.handlers:
|
|||
logger.addHandler(handler)
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
# GGUF inference backend (llama-server)
|
||||
_llama_cpp_backend = LlamaCppBackend()
|
||||
|
||||
def get_llama_cpp_backend() -> LlamaCppBackend:
|
||||
return _llama_cpp_backend
|
||||
|
||||
|
||||
@router.post("/load", response_model=LoadResponse)
|
||||
async def load_model(request: LoadRequest):
|
||||
"""
|
||||
Load a model for inference.
|
||||
|
||||
|
||||
The model_path should be a clean identifier from GET /models/list.
|
||||
Returns inference configuration parameters (temperature, top_p, top_k, min_p)
|
||||
from the model's YAML config, falling back to default.yaml for missing values.
|
||||
|
||||
GGUF models are loaded via llama-server (llama.cpp) instead of Unsloth.
|
||||
"""
|
||||
try:
|
||||
backend = get_inference_backend()
|
||||
|
||||
# Create config using clean factory method
|
||||
# is_lora is auto-detected from adapter_config.json on disk/HF
|
||||
config = ModelConfig.from_identifier(
|
||||
model_id=request.model_path,
|
||||
hf_token=request.hf_token,
|
||||
gguf_variant=request.gguf_variant,
|
||||
)
|
||||
|
||||
|
||||
if not config:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid model identifier: {request.model_path}"
|
||||
)
|
||||
|
||||
# Load the model
|
||||
|
||||
# ── GGUF path: load via llama-server ──────────────────────
|
||||
if config.is_gguf:
|
||||
llama_backend = get_llama_cpp_backend()
|
||||
unsloth_backend = get_inference_backend()
|
||||
|
||||
# Unload any active Unsloth model first to free VRAM
|
||||
if unsloth_backend.active_model_name:
|
||||
logger.info(f"Unloading Unsloth model '{unsloth_backend.active_model_name}' before loading GGUF")
|
||||
unsloth_backend.unload_model(unsloth_backend.active_model_name)
|
||||
|
||||
# Route to HF mode or local mode based on config
|
||||
if config.gguf_hf_repo:
|
||||
# HF mode: llama-server downloads via -hf "repo:quant"
|
||||
success = llama_backend.load_model(
|
||||
hf_repo=config.gguf_hf_repo,
|
||||
hf_variant=config.gguf_variant,
|
||||
hf_token=request.hf_token,
|
||||
model_identifier=config.identifier,
|
||||
is_vision=config.is_vision,
|
||||
n_ctx=request.max_seq_length,
|
||||
)
|
||||
else:
|
||||
# Local mode: llama-server loads via -m <path>
|
||||
success = llama_backend.load_model(
|
||||
gguf_path=config.gguf_file,
|
||||
model_identifier=config.identifier,
|
||||
is_vision=config.is_vision,
|
||||
n_ctx=request.max_seq_length,
|
||||
)
|
||||
|
||||
if not success:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Failed to load GGUF model: {config.display_name}"
|
||||
)
|
||||
|
||||
logger.info(f"Loaded GGUF model via llama-server: {config.identifier}")
|
||||
|
||||
inference_config = load_inference_config(config.identifier)
|
||||
|
||||
return LoadResponse(
|
||||
status="loaded",
|
||||
model=config.identifier,
|
||||
display_name=config.display_name,
|
||||
is_vision=config.is_vision,
|
||||
is_lora=False,
|
||||
is_gguf=True,
|
||||
inference=inference_config,
|
||||
)
|
||||
|
||||
# ── Standard path: load via Unsloth/transformers ──────────
|
||||
backend = get_inference_backend()
|
||||
|
||||
# Unload any active GGUF model first
|
||||
llama_backend = get_llama_cpp_backend()
|
||||
if llama_backend.is_loaded:
|
||||
logger.info("Unloading GGUF model before loading Unsloth model")
|
||||
llama_backend.unload_model()
|
||||
|
||||
success = backend.load_model(
|
||||
config=config,
|
||||
max_seq_length=request.max_seq_length,
|
||||
load_in_4bit=request.load_in_4bit,
|
||||
hf_token=request.hf_token,
|
||||
)
|
||||
|
||||
|
||||
if not success:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Failed to load model: {config.display_name}"
|
||||
)
|
||||
|
||||
|
||||
logger.info(f"Loaded model: {config.identifier}")
|
||||
|
||||
|
||||
# Load inference configuration parameters
|
||||
inference_config = load_inference_config(config.identifier)
|
||||
|
||||
|
||||
return LoadResponse(
|
||||
status="loaded",
|
||||
model=config.identifier,
|
||||
display_name=config.display_name,
|
||||
is_vision=config.is_vision,
|
||||
is_lora=config.is_lora,
|
||||
is_gguf=False,
|
||||
inference=inference_config,
|
||||
)
|
||||
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
|
|
@ -129,13 +197,22 @@ async def load_model(request: LoadRequest):
|
|||
async def unload_model(request: UnloadRequest):
|
||||
"""
|
||||
Unload a model from memory.
|
||||
Routes to the correct backend (llama-server for GGUF, Unsloth otherwise).
|
||||
"""
|
||||
try:
|
||||
# Check if the GGUF backend has this model loaded
|
||||
llama_backend = get_llama_cpp_backend()
|
||||
if llama_backend.is_loaded and llama_backend.model_identifier == request.model_path:
|
||||
llama_backend.unload_model()
|
||||
logger.info(f"Unloaded GGUF model: {request.model_path}")
|
||||
return UnloadResponse(status="unloaded", model=request.model_path)
|
||||
|
||||
# Otherwise, unload from Unsloth backend
|
||||
backend = get_inference_backend()
|
||||
backend.unload_model(request.model_path)
|
||||
logger.info(f"Unloaded model: {request.model_path}")
|
||||
return UnloadResponse(status="unloaded", model=request.model_path)
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error unloading model: {e}", exc_info=True)
|
||||
raise HTTPException(
|
||||
|
|
@ -221,22 +298,37 @@ async def generate_stream(request: GenerateRequest):
|
|||
async def get_status():
|
||||
"""
|
||||
Get current inference backend status.
|
||||
Reports whichever backend (Unsloth or llama-server) is currently active.
|
||||
"""
|
||||
try:
|
||||
llama_backend = get_llama_cpp_backend()
|
||||
|
||||
# If a GGUF model is loaded via llama-server, report that
|
||||
if llama_backend.is_loaded:
|
||||
return InferenceStatusResponse(
|
||||
active_model=llama_backend.model_identifier,
|
||||
is_vision=llama_backend.is_vision,
|
||||
is_gguf=True,
|
||||
loading=[],
|
||||
loaded=[llama_backend.model_identifier],
|
||||
)
|
||||
|
||||
# Otherwise, report Unsloth backend status
|
||||
backend = get_inference_backend()
|
||||
|
||||
|
||||
is_vision = False
|
||||
if backend.active_model_name:
|
||||
model_info = backend.models.get(backend.active_model_name, {})
|
||||
is_vision = model_info.get("is_vision", False)
|
||||
|
||||
|
||||
return InferenceStatusResponse(
|
||||
active_model=backend.active_model_name,
|
||||
is_vision=is_vision,
|
||||
is_gguf=False,
|
||||
loading=list(getattr(backend, 'loading_models', set())),
|
||||
loaded=list(backend.models.keys()),
|
||||
)
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting status: {e}", exc_info=True)
|
||||
raise HTTPException(
|
||||
|
|
@ -315,29 +407,163 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque
|
|||
|
||||
Streaming (default): returns SSE chunks matching OpenAI's format.
|
||||
Non-streaming: returns a single ChatCompletion JSON object.
|
||||
"""
|
||||
backend = get_inference_backend()
|
||||
|
||||
if not backend.active_model_name:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="No model loaded. Call POST /inference/load first.",
|
||||
)
|
||||
Automatically routes to the correct backend:
|
||||
- GGUF models → llama-server via LlamaCppBackend
|
||||
- Other models → Unsloth/transformers via InferenceBackend
|
||||
"""
|
||||
llama_backend = get_llama_cpp_backend()
|
||||
using_gguf = llama_backend.is_loaded
|
||||
|
||||
# ── Determine which backend is active ─────────────────────
|
||||
if using_gguf:
|
||||
model_name = llama_backend.model_identifier or payload.model
|
||||
else:
|
||||
backend = get_inference_backend()
|
||||
if not backend.active_model_name:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="No model loaded. Call POST /inference/load first.",
|
||||
)
|
||||
model_name = backend.active_model_name or payload.model
|
||||
|
||||
# ── Parse messages (handles multimodal content parts) ─────
|
||||
system_prompt, chat_messages, extracted_image_b64 = _extract_content_parts(
|
||||
payload.messages
|
||||
)
|
||||
|
||||
# If no non-system messages were provided, error out
|
||||
if not chat_messages:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="At least one non-system message is required.",
|
||||
)
|
||||
|
||||
# ── Decode image (from content parts OR legacy field) ─────
|
||||
# Content-part images take priority; fall back to legacy field
|
||||
# ── GGUF path: proxy to llama-server /v1/chat/completions ──
|
||||
if using_gguf:
|
||||
# Reject images if this GGUF model doesn't support vision
|
||||
image_b64 = extracted_image_b64 or payload.image_base64
|
||||
if image_b64 and not llama_backend.is_vision:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Image provided but current GGUF model does not support vision.",
|
||||
)
|
||||
|
||||
# Build message list with system prompt prepended
|
||||
gguf_messages = []
|
||||
if system_prompt:
|
||||
gguf_messages.append({"role": "system", "content": system_prompt})
|
||||
gguf_messages.extend(chat_messages)
|
||||
|
||||
cancel_event = threading.Event()
|
||||
|
||||
completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"
|
||||
created = int(time.time())
|
||||
|
||||
def gguf_generate():
|
||||
return llama_backend.generate_chat_completion(
|
||||
messages=gguf_messages,
|
||||
image_b64=image_b64,
|
||||
temperature=payload.temperature,
|
||||
top_p=payload.top_p,
|
||||
top_k=payload.top_k,
|
||||
min_p=payload.min_p,
|
||||
max_tokens=payload.max_tokens or 512,
|
||||
repetition_penalty=payload.repetition_penalty,
|
||||
cancel_event=cancel_event,
|
||||
)
|
||||
|
||||
if payload.stream:
|
||||
async def gguf_stream_chunks():
|
||||
try:
|
||||
# First chunk: role
|
||||
first_chunk = ChatCompletionChunk(
|
||||
id=completion_id,
|
||||
created=created,
|
||||
model=model_name,
|
||||
choices=[ChunkChoice(
|
||||
delta=ChoiceDelta(role="assistant"),
|
||||
finish_reason=None,
|
||||
)],
|
||||
)
|
||||
yield f"data: {first_chunk.model_dump_json(exclude_none=True)}\n\n"
|
||||
|
||||
# Content chunks — llama backend yields cumulative text
|
||||
prev_text = ""
|
||||
for cumulative in gguf_generate():
|
||||
if await request.is_disconnected():
|
||||
cancel_event.set()
|
||||
return
|
||||
new_text = cumulative[len(prev_text):]
|
||||
prev_text = cumulative
|
||||
if not new_text:
|
||||
continue
|
||||
chunk = ChatCompletionChunk(
|
||||
id=completion_id,
|
||||
created=created,
|
||||
model=model_name,
|
||||
choices=[ChunkChoice(
|
||||
delta=ChoiceDelta(content=new_text),
|
||||
finish_reason=None,
|
||||
)],
|
||||
)
|
||||
yield f"data: {chunk.model_dump_json(exclude_none=True)}\n\n"
|
||||
|
||||
# Final chunk
|
||||
final_chunk = ChatCompletionChunk(
|
||||
id=completion_id,
|
||||
created=created,
|
||||
model=model_name,
|
||||
choices=[ChunkChoice(
|
||||
delta=ChoiceDelta(),
|
||||
finish_reason="stop",
|
||||
)],
|
||||
)
|
||||
yield f"data: {final_chunk.model_dump_json(exclude_none=True)}\n\n"
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
except asyncio.CancelledError:
|
||||
cancel_event.set()
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error during GGUF streaming: {e}", exc_info=True)
|
||||
error_chunk = {
|
||||
"error": {"message": str(e), "type": "server_error"},
|
||||
}
|
||||
yield f"data: {json.dumps(error_chunk)}\n\n"
|
||||
|
||||
return StreamingResponse(
|
||||
gguf_stream_chunks(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
else:
|
||||
try:
|
||||
full_text = ""
|
||||
for token in gguf_generate():
|
||||
full_text = token
|
||||
|
||||
response = ChatCompletion(
|
||||
id=completion_id,
|
||||
created=created,
|
||||
model=model_name,
|
||||
choices=[CompletionChoice(
|
||||
message=CompletionMessage(content=full_text),
|
||||
finish_reason="stop",
|
||||
)],
|
||||
)
|
||||
return JSONResponse(content=response.model_dump())
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during GGUF completion: {e}", exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
# ── Standard Unsloth path ─────────────────────────────────
|
||||
|
||||
# Decode image (from content parts OR legacy field)
|
||||
image_b64 = extracted_image_b64 or payload.image_base64
|
||||
image = None
|
||||
|
||||
|
|
@ -363,7 +589,7 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque
|
|||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=f"Failed to decode image: {e}")
|
||||
|
||||
# ── Shared generation kwargs ──────────────────────────────
|
||||
# Shared generation kwargs
|
||||
gen_kwargs = dict(
|
||||
messages=chat_messages,
|
||||
system_prompt=system_prompt,
|
||||
|
|
@ -376,11 +602,10 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque
|
|||
repetition_penalty=payload.repetition_penalty,
|
||||
)
|
||||
|
||||
# ── Choose generation path (adapter-controlled or standard) ──
|
||||
# Choose generation path (adapter-controlled or standard)
|
||||
cancel_event = threading.Event()
|
||||
|
||||
if payload.use_adapter is not None:
|
||||
# Compare mode: toggle adapter state atomically with generation
|
||||
def generate():
|
||||
return backend.generate_with_adapter_control(
|
||||
use_adapter=payload.use_adapter,
|
||||
|
|
@ -388,11 +613,9 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque
|
|||
**gen_kwargs,
|
||||
)
|
||||
else:
|
||||
# Standard path: no adapter toggling
|
||||
def generate():
|
||||
return backend.generate_chat_response(cancel_event=cancel_event, **gen_kwargs)
|
||||
|
||||
model_name = backend.active_model_name or payload.model
|
||||
completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"
|
||||
created = int(time.time())
|
||||
|
||||
|
|
@ -400,7 +623,6 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque
|
|||
if payload.stream:
|
||||
async def stream_chunks():
|
||||
try:
|
||||
# First chunk: send the role
|
||||
first_chunk = ChatCompletionChunk(
|
||||
id=completion_id,
|
||||
created=created,
|
||||
|
|
@ -412,8 +634,6 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque
|
|||
)
|
||||
yield f"data: {first_chunk.model_dump_json(exclude_none=True)}\n\n"
|
||||
|
||||
# Content chunks — generate_chat_response yields cumulative
|
||||
# text, so we diff to get incremental deltas.
|
||||
prev_text = ""
|
||||
for cumulative in generate():
|
||||
if await request.is_disconnected():
|
||||
|
|
@ -435,7 +655,6 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque
|
|||
)
|
||||
yield f"data: {chunk.model_dump_json(exclude_none=True)}\n\n"
|
||||
|
||||
# Final chunk: finish_reason = stop
|
||||
final_chunk = ChatCompletionChunk(
|
||||
id=completion_id,
|
||||
created=created,
|
||||
|
|
@ -475,7 +694,7 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque
|
|||
try:
|
||||
full_text = ""
|
||||
for token in generate():
|
||||
full_text = token # generate_stream yields cumulative text
|
||||
full_text = token
|
||||
|
||||
response = ChatCompletion(
|
||||
id=completion_id,
|
||||
|
|
|
|||
|
|
@ -23,8 +23,10 @@ try:
|
|||
get_base_model_from_lora,
|
||||
is_vision_model,
|
||||
scan_checkpoints,
|
||||
list_gguf_variants,
|
||||
ModelConfig,
|
||||
)
|
||||
from utils.models.model_config import _pick_best_gguf, _extract_quant_label
|
||||
from core.inference import get_inference_backend
|
||||
except ImportError:
|
||||
# Fallback: try to import from parent directory
|
||||
|
|
@ -38,8 +40,10 @@ except ImportError:
|
|||
get_base_model_from_lora,
|
||||
is_vision_model,
|
||||
scan_checkpoints,
|
||||
list_gguf_variants,
|
||||
ModelConfig,
|
||||
)
|
||||
from utils.models.model_config import _pick_best_gguf, _extract_quant_label
|
||||
from core.inference import get_inference_backend
|
||||
|
||||
from models import (
|
||||
|
|
@ -53,6 +57,7 @@ from models import (
|
|||
LoRAInfo,
|
||||
ModelListResponse,
|
||||
)
|
||||
from models.models import GgufVariantDetail, GgufVariantsResponse
|
||||
from models.responses import LoRABaseModelResponse, VisionCheckResponse
|
||||
|
||||
router = APIRouter()
|
||||
|
|
@ -90,6 +95,7 @@ def _scan_models_dir(models_dir: Path) -> List[LocalModelInfo]:
|
|||
or (child / "adapter_config.json").exists()
|
||||
or any(child.glob("*.safetensors"))
|
||||
or any(child.glob("*.bin"))
|
||||
or any(child.glob("*.gguf"))
|
||||
)
|
||||
if not has_model_files:
|
||||
continue
|
||||
|
|
@ -106,6 +112,23 @@ def _scan_models_dir(models_dir: Path) -> List[LocalModelInfo]:
|
|||
updated_at=updated_at,
|
||||
),
|
||||
)
|
||||
# Also scan for standalone .gguf files directly in the models directory
|
||||
for gguf_file in models_dir.glob("*.gguf"):
|
||||
if gguf_file.is_file():
|
||||
try:
|
||||
updated_at = gguf_file.stat().st_mtime
|
||||
except OSError:
|
||||
updated_at = None
|
||||
found.append(
|
||||
LocalModelInfo(
|
||||
id=str(gguf_file),
|
||||
display_name=gguf_file.stem,
|
||||
path=str(gguf_file),
|
||||
source="models_dir",
|
||||
updated_at=updated_at,
|
||||
),
|
||||
)
|
||||
|
||||
return found
|
||||
|
||||
|
||||
|
|
@ -399,6 +422,49 @@ async def check_vision_model(
|
|||
detail=f"Failed to check vision model: {str(e)}"
|
||||
)
|
||||
|
||||
@router.get("/gguf-variants", response_model=GgufVariantsResponse)
|
||||
async def get_gguf_variants(
|
||||
repo_id: str = Query(..., description="HuggingFace repo ID (e.g. 'unsloth/gemma-3-4b-it-GGUF')"),
|
||||
hf_token: Optional[str] = Query(None, description="HuggingFace token for private repos"),
|
||||
current_subject: str = Depends(get_current_subject),
|
||||
):
|
||||
"""
|
||||
List available GGUF quantization variants for a HuggingFace repo.
|
||||
|
||||
Returns all available quantization variants (Q4_K_M, Q8_0, BF16, etc.)
|
||||
with file sizes, whether the model supports vision, and the recommended
|
||||
default variant.
|
||||
"""
|
||||
try:
|
||||
variants, has_vision = list_gguf_variants(repo_id, hf_token=hf_token)
|
||||
|
||||
# Determine default variant
|
||||
filenames = [v.filename for v in variants]
|
||||
best = _pick_best_gguf(filenames)
|
||||
default_variant = _extract_quant_label(best) if best else None
|
||||
|
||||
return GgufVariantsResponse(
|
||||
repo_id=repo_id,
|
||||
variants=[
|
||||
GgufVariantDetail(
|
||||
filename=v.filename,
|
||||
quant=v.quant,
|
||||
size_bytes=v.size_bytes,
|
||||
)
|
||||
for v in variants
|
||||
],
|
||||
has_vision=has_vision,
|
||||
default_variant=default_variant,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error listing GGUF variants for '{repo_id}': {e}", exc_info=True)
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Failed to list GGUF variants: {str(e)}",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/checkpoints", response_model=CheckpointListResponse)
|
||||
async def list_checkpoints(
|
||||
outputs_dir: str = Query(
|
||||
|
|
|
|||
|
|
@ -3,12 +3,14 @@ Model and LoRA configuration handling
|
|||
"""
|
||||
from .model_config import (
|
||||
ModelConfig,
|
||||
GgufVariantInfo,
|
||||
is_vision_model,
|
||||
scan_trained_loras,
|
||||
scan_exported_models,
|
||||
load_model_defaults,
|
||||
get_base_model_from_lora,
|
||||
load_model_config,
|
||||
list_gguf_variants,
|
||||
MODEL_NAME_MAPPING,
|
||||
UI_STATUS_INDICATORS,
|
||||
)
|
||||
|
|
@ -16,12 +18,14 @@ from .checkpoints import scan_checkpoints
|
|||
|
||||
__all__ = [
|
||||
'ModelConfig',
|
||||
'GgufVariantInfo',
|
||||
'is_vision_model',
|
||||
'scan_trained_loras',
|
||||
'scan_exported_models',
|
||||
'load_model_defaults',
|
||||
'get_base_model_from_lora',
|
||||
'load_model_config',
|
||||
'list_gguf_variants',
|
||||
'MODEL_NAME_MAPPING',
|
||||
'UI_STATUS_INDICATORS',
|
||||
'scan_checkpoints',
|
||||
|
|
|
|||
|
|
@ -422,6 +422,179 @@ def is_vision_model(model_name: str, hf_token: Optional[str] = None) -> bool:
|
|||
pass
|
||||
|
||||
|
||||
def detect_gguf_model(path: str) -> Optional[str]:
|
||||
"""
|
||||
Check if the given local path is or contains a GGUF model file.
|
||||
|
||||
Handles two cases:
|
||||
1. path is a direct .gguf file path
|
||||
2. path is a directory containing .gguf files
|
||||
|
||||
Returns the full path to the .gguf file if found, None otherwise.
|
||||
For HuggingFace repo detection, use detect_gguf_model_remote() instead.
|
||||
"""
|
||||
p = Path(path)
|
||||
|
||||
# Case 1: direct .gguf file
|
||||
if p.suffix == ".gguf" and p.is_file():
|
||||
return str(p.resolve())
|
||||
|
||||
# Case 2: directory containing .gguf files
|
||||
if p.is_dir():
|
||||
gguf_files = sorted(p.glob("*.gguf"), key=lambda f: f.stat().st_size, reverse=True)
|
||||
if gguf_files:
|
||||
return str(gguf_files[0].resolve())
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# Preferred GGUF quantization levels, in descending priority.
|
||||
# Q4_K_M is a good default: small, fast, acceptable quality.
|
||||
_GGUF_QUANT_PREFERENCE = [
|
||||
"Q4_K_M", "Q4_K_S", "Q5_K_M", "Q5_K_S",
|
||||
"Q6_K", "Q8_0", "Q3_K_M", "Q3_K_L", "Q2_K",
|
||||
"F16", "BF16", "F32",
|
||||
]
|
||||
|
||||
|
||||
def _pick_best_gguf(filenames: list[str]) -> Optional[str]:
|
||||
"""
|
||||
Pick the best GGUF file from a list of filenames.
|
||||
|
||||
Prefers quantization levels in _GGUF_QUANT_PREFERENCE order.
|
||||
Falls back to the first .gguf file found.
|
||||
"""
|
||||
gguf_files = [f for f in filenames if f.endswith(".gguf")]
|
||||
if not gguf_files:
|
||||
return None
|
||||
|
||||
# Try preferred quantization levels
|
||||
for quant in _GGUF_QUANT_PREFERENCE:
|
||||
for f in gguf_files:
|
||||
if quant in f:
|
||||
return f
|
||||
|
||||
# Fallback: first GGUF file
|
||||
return gguf_files[0]
|
||||
|
||||
|
||||
@dataclass
|
||||
class GgufVariantInfo:
|
||||
"""A single GGUF quantization variant from a HuggingFace repo."""
|
||||
filename: str # e.g., "gemma-3-4b-it-Q4_K_M.gguf"
|
||||
quant: str # e.g., "Q4_K_M" (extracted from filename)
|
||||
size_bytes: int # file size
|
||||
|
||||
|
||||
def _extract_quant_label(filename: str) -> str:
|
||||
"""
|
||||
Extract quantization label like Q4_K_M, IQ4_XS, BF16 from a GGUF filename.
|
||||
|
||||
Examples:
|
||||
"gemma-3-4b-it-Q4_K_M.gguf" → "Q4_K_M"
|
||||
"model-IQ4_NL.gguf" → "IQ4_NL"
|
||||
"model-BF16.gguf" → "BF16"
|
||||
"model-UD-IQ1_S.gguf" → "UD-IQ1_S"
|
||||
"""
|
||||
import re
|
||||
stem = filename.rsplit(".", 1)[0] # Remove .gguf
|
||||
# Match known quantization patterns (UD- prefix, IQ, Q, BF/F variants)
|
||||
match = re.search(
|
||||
r'(UD-)?' # Optional UD- prefix (Ultra Discrete)
|
||||
r'(IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?' # IQ variants: IQ4_XS, IQ4_NL, IQ1_S
|
||||
r'|Q[0-9]+_K_[A-Z]+' # K-quant: Q4_K_M, Q3_K_S
|
||||
r'|Q[0-9]+_[0-9]+' # Standard: Q8_0, Q5_1
|
||||
r'|Q[0-9]+_K' # Short K-quant: Q6_K
|
||||
r'|BF16|F16|F32)', # Full precision
|
||||
stem, re.IGNORECASE,
|
||||
)
|
||||
if match:
|
||||
prefix = match.group(1) or ""
|
||||
return f"{prefix}{match.group(2)}"
|
||||
# Fallback: last segment after hyphen
|
||||
return stem.split("-")[-1]
|
||||
|
||||
|
||||
def list_gguf_variants(
|
||||
repo_id: str,
|
||||
hf_token: Optional[str] = None,
|
||||
) -> tuple[list[GgufVariantInfo], bool]:
|
||||
"""
|
||||
List all GGUF quantization variants in a HuggingFace repo.
|
||||
|
||||
Separates main model files from mmproj (vision projection) files.
|
||||
The presence of mmproj files indicates a vision-capable model.
|
||||
|
||||
Returns:
|
||||
(variants, has_vision): list of non-mmproj GGUF variants + vision flag.
|
||||
"""
|
||||
from huggingface_hub import model_info as hf_model_info
|
||||
|
||||
info = hf_model_info(repo_id, token=hf_token)
|
||||
variants: list[GgufVariantInfo] = []
|
||||
has_vision = False
|
||||
|
||||
for sibling in info.siblings:
|
||||
fname = sibling.rfilename
|
||||
if not fname.endswith(".gguf"):
|
||||
continue
|
||||
size = sibling.size or 0
|
||||
|
||||
# mmproj files are vision projection models, not main model files
|
||||
if "mmproj" in fname.lower():
|
||||
has_vision = True
|
||||
continue
|
||||
|
||||
quant = _extract_quant_label(fname)
|
||||
variants.append(GgufVariantInfo(
|
||||
filename=fname,
|
||||
quant=quant,
|
||||
size_bytes=size,
|
||||
))
|
||||
|
||||
return variants, has_vision
|
||||
|
||||
|
||||
def detect_gguf_model_remote(
|
||||
repo_id: str,
|
||||
hf_token: Optional[str] = None,
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
Check if a HuggingFace repo contains GGUF files.
|
||||
|
||||
Returns the filename of the best GGUF file in the repo, or None.
|
||||
"""
|
||||
try:
|
||||
from huggingface_hub import model_info as hf_model_info
|
||||
|
||||
info = hf_model_info(repo_id, token=hf_token)
|
||||
repo_files = [s.rfilename for s in info.siblings]
|
||||
return _pick_best_gguf(repo_files)
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not check GGUF files for '{repo_id}': {e}")
|
||||
return None
|
||||
|
||||
|
||||
def download_gguf_file(
|
||||
repo_id: str,
|
||||
filename: str,
|
||||
hf_token: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Download a specific GGUF file from a HuggingFace repo.
|
||||
|
||||
Returns the local path to the downloaded file.
|
||||
"""
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
local_path = hf_hub_download(
|
||||
repo_id=repo_id,
|
||||
filename=filename,
|
||||
token=hf_token,
|
||||
)
|
||||
return local_path
|
||||
|
||||
|
||||
def scan_trained_loras(outputs_dir: str = "./outputs") -> List[Tuple[str, str]]:
|
||||
"""
|
||||
Scan outputs folder for trained LoRA adapters.
|
||||
|
|
@ -679,6 +852,10 @@ class ModelConfig:
|
|||
is_cached: bool # Is this already in HF cache?
|
||||
is_vision: bool # Is this a vision model?
|
||||
is_lora: bool # Is this a lora adapter?
|
||||
is_gguf: bool = False # Is this a GGUF model?
|
||||
gguf_file: Optional[str] = None # Full path to the .gguf file (local mode)
|
||||
gguf_hf_repo: Optional[str] = None # HF repo ID for -hf mode (e.g. "unsloth/gemma-3-4b-it-GGUF")
|
||||
gguf_variant: Optional[str] = None # Quantization variant (e.g. "Q4_K_M")
|
||||
base_model: Optional[str] = None # Base model (for LoRAs)
|
||||
|
||||
@classmethod
|
||||
|
|
@ -734,37 +911,102 @@ class ModelConfig:
|
|||
cls,
|
||||
model_id: str,
|
||||
hf_token: Optional[str] = None,
|
||||
is_lora: bool = False
|
||||
is_lora: bool = False,
|
||||
gguf_variant: Optional[str] = None,
|
||||
) -> Optional['ModelConfig']:
|
||||
"""
|
||||
Create ModelConfig from a clean model identifier.
|
||||
|
||||
|
||||
For FastAPI routes where the frontend sends sanitized model paths.
|
||||
No Gradio dropdown parsing - expects clean identifiers like:
|
||||
- "unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit"
|
||||
- "./outputs/my_lora_adapter"
|
||||
- "/absolute/path/to/model"
|
||||
|
||||
|
||||
Args:
|
||||
model_id: Clean model identifier (HF repo name or local path)
|
||||
hf_token: Optional HF token for vision detection on gated models
|
||||
is_lora: Whether this is a LoRA adapter
|
||||
|
||||
gguf_variant: Optional GGUF quantization variant (e.g. "Q4_K_M").
|
||||
For remote GGUF repos, specifies which quant to load via -hf.
|
||||
If None, auto-selects using _pick_best_gguf().
|
||||
|
||||
Returns:
|
||||
ModelConfig or None if configuration cannot be created
|
||||
"""
|
||||
if not model_id or not model_id.strip():
|
||||
return None
|
||||
|
||||
|
||||
identifier = model_id.strip()
|
||||
is_local = is_local_path(identifier)
|
||||
path = normalize_path(identifier) if is_local else identifier
|
||||
|
||||
|
||||
# Add unsloth/ prefix for shorthand HF models
|
||||
if not is_local and "/" not in identifier:
|
||||
identifier = f"unsloth/{identifier}"
|
||||
path = identifier
|
||||
|
||||
|
||||
# Auto-detect GGUF models (check before LoRA/vision detection)
|
||||
if is_local:
|
||||
gguf_file = detect_gguf_model(path)
|
||||
if gguf_file:
|
||||
display_name = Path(gguf_file).stem
|
||||
logger.info(f"Detected local GGUF model: {gguf_file}")
|
||||
return cls(
|
||||
identifier=identifier,
|
||||
display_name=display_name,
|
||||
path=path,
|
||||
is_local=True,
|
||||
is_cached=True,
|
||||
is_vision=False,
|
||||
is_lora=False,
|
||||
is_gguf=True,
|
||||
gguf_file=gguf_file,
|
||||
)
|
||||
else:
|
||||
# Check if the HF repo contains GGUF files
|
||||
gguf_filename = detect_gguf_model_remote(identifier, hf_token=hf_token)
|
||||
if gguf_filename:
|
||||
# Preflight: verify llama-server binary exists BEFORE user waits
|
||||
# for a multi-GB download that llama-server handles natively
|
||||
from core.inference.llama_cpp import LlamaCppBackend
|
||||
if not LlamaCppBackend._find_llama_server_binary():
|
||||
raise RuntimeError(
|
||||
"llama-server binary not found — cannot load GGUF models. "
|
||||
"Run setup.sh to build it, or set LLAMA_SERVER_PATH."
|
||||
)
|
||||
|
||||
# Use list_gguf_variants() to detect vision & resolve variant
|
||||
variants, has_vision = list_gguf_variants(identifier, hf_token=hf_token)
|
||||
variant = gguf_variant
|
||||
if not variant:
|
||||
# Auto-select best quantization
|
||||
variant_filenames = [v.filename for v in variants]
|
||||
best = _pick_best_gguf(variant_filenames)
|
||||
if best:
|
||||
variant = _extract_quant_label(best)
|
||||
else:
|
||||
variant = "Q4_K_M" # Fallback — llama-server's own default
|
||||
|
||||
display_name = f"{identifier.split('/')[-1]} ({variant})"
|
||||
logger.info(
|
||||
f"Detected remote GGUF repo '{identifier}', "
|
||||
f"variant={variant}, vision={has_vision}"
|
||||
)
|
||||
return cls(
|
||||
identifier=identifier,
|
||||
display_name=display_name,
|
||||
path=identifier,
|
||||
is_local=False,
|
||||
is_cached=False,
|
||||
is_vision=has_vision,
|
||||
is_lora=False,
|
||||
is_gguf=True,
|
||||
gguf_file=None,
|
||||
gguf_hf_repo=identifier,
|
||||
gguf_variant=variant,
|
||||
)
|
||||
|
||||
# Auto-detect LoRA for local paths (check adapter_config.json on disk)
|
||||
if not is_lora and is_local:
|
||||
detected_base = get_base_model_from_lora(path)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ import {
|
|||
TooltipContent,
|
||||
TooltipTrigger,
|
||||
} from "@/components/ui/tooltip";
|
||||
import { listGgufVariants } from "@/features/chat/api/chat-api";
|
||||
import type { GgufVariantDetail } from "@/features/chat/types/api";
|
||||
import {
|
||||
useDebouncedValue,
|
||||
useGpuInfo,
|
||||
|
|
@ -17,7 +19,7 @@ import type { VramFitStatus } from "@/lib/vram";
|
|||
import { checkVramFit, estimateLoadingVram } from "@/lib/vram";
|
||||
import { Search01Icon } from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import { useMemo, useState, type ReactNode } from "react";
|
||||
import { useCallback, useEffect, useMemo, useState, type ReactNode } from "react";
|
||||
import type {
|
||||
LoraModelOption,
|
||||
ModelOption,
|
||||
|
|
@ -36,6 +38,15 @@ function ListLabel({ children }: { children: ReactNode }) {
|
|||
);
|
||||
}
|
||||
|
||||
/** Format bytes to a human-readable size string. */
|
||||
function formatBytes(bytes: number): string {
|
||||
if (bytes === 0) return "0 B";
|
||||
const units = ["B", "KB", "MB", "GB", "TB"];
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(1024));
|
||||
const value = bytes / 1024 ** i;
|
||||
return `${value.toFixed(value < 10 ? 1 : 0)} ${units[i]}`;
|
||||
}
|
||||
|
||||
function ModelRow({
|
||||
label,
|
||||
meta,
|
||||
|
|
@ -114,6 +125,124 @@ function ModelRow({
|
|||
return content;
|
||||
}
|
||||
|
||||
// ── GGUF Variant Expander ────────────────────────────────────
|
||||
|
||||
function GgufVariantExpander({
|
||||
repoId,
|
||||
onSelect,
|
||||
}: {
|
||||
repoId: string;
|
||||
onSelect: (id: string, meta: ModelSelectorChangeMeta) => void;
|
||||
}) {
|
||||
const [variants, setVariants] = useState<GgufVariantDetail[] | null>(null);
|
||||
const [defaultVariant, setDefaultVariant] = useState<string | null>(null);
|
||||
const [hasVision, setHasVision] = useState(false);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
let canceled = false;
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
|
||||
listGgufVariants(repoId)
|
||||
.then((res) => {
|
||||
if (canceled) return;
|
||||
setVariants(res.variants);
|
||||
setDefaultVariant(res.default_variant);
|
||||
setHasVision(res.has_vision);
|
||||
})
|
||||
.catch((err) => {
|
||||
if (canceled) return;
|
||||
setError(err instanceof Error ? err.message : "Failed to load variants");
|
||||
})
|
||||
.finally(() => {
|
||||
if (!canceled) setLoading(false);
|
||||
});
|
||||
|
||||
return () => {
|
||||
canceled = true;
|
||||
};
|
||||
}, [repoId]);
|
||||
|
||||
const handleVariantClick = useCallback(
|
||||
(quant: string) => {
|
||||
onSelect(repoId, {
|
||||
source: "hub",
|
||||
isLora: false,
|
||||
ggufVariant: quant,
|
||||
});
|
||||
},
|
||||
[repoId, onSelect],
|
||||
);
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<div className="flex items-center gap-2 px-5 py-2">
|
||||
<Spinner className="size-3 text-muted-foreground" />
|
||||
<span className="text-xs text-muted-foreground">Loading variants…</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<div className="px-5 py-2 text-xs text-destructive">{error}</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (!variants || variants.length === 0) {
|
||||
return (
|
||||
<div className="px-5 py-2 text-xs text-muted-foreground">
|
||||
No GGUF variants found.
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="pl-4 border-l-2 border-accent/50 ml-3 my-1">
|
||||
<div className="px-2 py-1 flex items-center gap-1.5">
|
||||
<span className="text-[10px] font-semibold uppercase tracking-wider text-muted-foreground">
|
||||
Quantizations
|
||||
</span>
|
||||
{hasVision && (
|
||||
<span className="text-[9px] font-medium text-blue-400">Vision</span>
|
||||
)}
|
||||
</div>
|
||||
{variants.map((v) => (
|
||||
<button
|
||||
key={v.filename}
|
||||
type="button"
|
||||
onClick={() => handleVariantClick(v.quant)}
|
||||
className={cn(
|
||||
"flex w-full items-center justify-between gap-2 rounded-md px-2.5 py-1 text-left text-sm transition-colors hover:bg-accent",
|
||||
)}
|
||||
>
|
||||
<span className="min-w-0 flex-1 truncate font-mono text-xs">
|
||||
{v.quant}
|
||||
{v.quant === defaultVariant && (
|
||||
<span className="ml-1.5 text-[9px] font-sans font-medium text-primary/70">
|
||||
recommended
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
<span className="text-[10px] text-muted-foreground shrink-0">
|
||||
{formatBytes(v.size_bytes)}
|
||||
</span>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Detect GGUF repos by naming convention ────────────────────
|
||||
|
||||
function isGgufRepo(id: string): boolean {
|
||||
return id.toUpperCase().includes("-GGUF");
|
||||
}
|
||||
|
||||
// ── Hub Model Picker ──────────────────────────────────────────
|
||||
|
||||
export function HubModelPicker({
|
||||
models,
|
||||
value,
|
||||
|
|
@ -130,6 +259,9 @@ export function HubModelPicker({
|
|||
debouncedQuery,
|
||||
);
|
||||
|
||||
// Track which GGUF repo is expanded for variant selection
|
||||
const [expandedGguf, setExpandedGguf] = useState<string | null>(null);
|
||||
|
||||
const recommendedIds = useMemo(
|
||||
() => dedupe([...models.map((model) => model.id), value ?? ""]),
|
||||
[models, value],
|
||||
|
|
@ -199,6 +331,19 @@ export function HubModelPicker({
|
|||
|
||||
const { scrollRef, sentinelRef } = useInfiniteScroll(fetchMore, results.length);
|
||||
|
||||
/** Handle clicking a model row — GGUF repos expand, others load directly. */
|
||||
const handleModelClick = useCallback(
|
||||
(id: string) => {
|
||||
if (isGgufRepo(id)) {
|
||||
// Toggle GGUF variant expander
|
||||
setExpandedGguf((prev) => (prev === id ? null : id));
|
||||
} else {
|
||||
onSelect(id, { source: "hub", isLora: false });
|
||||
}
|
||||
},
|
||||
[onSelect],
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="space-y-2">
|
||||
<div className="relative">
|
||||
|
|
@ -230,18 +375,24 @@ export function HubModelPicker({
|
|||
recommendedIds.map((id) => {
|
||||
const vram = recommendedVramMap.get(id);
|
||||
return (
|
||||
<ModelRow
|
||||
key={id}
|
||||
label={id}
|
||||
meta={vram?.detail ?? undefined}
|
||||
selected={value === id}
|
||||
onClick={() =>
|
||||
onSelect(id, { source: "hub", isLora: false })
|
||||
}
|
||||
vramStatus={vram?.status ?? null}
|
||||
vramEst={vram?.est}
|
||||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
/>
|
||||
<div key={id}>
|
||||
<ModelRow
|
||||
label={id}
|
||||
meta={
|
||||
isGgufRepo(id)
|
||||
? "GGUF"
|
||||
: vram?.detail ?? undefined
|
||||
}
|
||||
selected={value === id}
|
||||
onClick={() => handleModelClick(id)}
|
||||
vramStatus={isGgufRepo(id) ? null : vram?.status ?? null}
|
||||
vramEst={isGgufRepo(id) ? undefined : vram?.est}
|
||||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
/>
|
||||
{expandedGguf === id && (
|
||||
<GgufVariantExpander repoId={id} onSelect={onSelect} />
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})
|
||||
)}
|
||||
|
|
@ -259,18 +410,24 @@ export function HubModelPicker({
|
|||
hfIds.map((id) => {
|
||||
const vram = vramMap.get(id);
|
||||
return (
|
||||
<ModelRow
|
||||
key={id}
|
||||
label={id}
|
||||
meta={metricsById.get(id)}
|
||||
selected={value === id}
|
||||
onClick={() =>
|
||||
onSelect(id, { source: "hub", isLora: false })
|
||||
}
|
||||
vramStatus={vram?.status ?? null}
|
||||
vramEst={vram?.est}
|
||||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
/>
|
||||
<div key={id}>
|
||||
<ModelRow
|
||||
label={id}
|
||||
meta={
|
||||
isGgufRepo(id)
|
||||
? "GGUF"
|
||||
: metricsById.get(id)
|
||||
}
|
||||
selected={value === id}
|
||||
onClick={() => handleModelClick(id)}
|
||||
vramStatus={isGgufRepo(id) ? null : vram?.status ?? null}
|
||||
vramEst={isGgufRepo(id) ? undefined : vram?.est}
|
||||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
/>
|
||||
{expandedGguf === id && (
|
||||
<GgufVariantExpander repoId={id} onSelect={onSelect} />
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})
|
||||
)}
|
||||
|
|
@ -393,4 +550,3 @@ export function LoraModelPicker({
|
|||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -17,5 +17,6 @@ export interface LoraModelOption extends ModelOption {
|
|||
export interface ModelSelectorChangeMeta {
|
||||
source: "hub" | "lora" | "exported";
|
||||
isLora: boolean;
|
||||
ggufVariant?: string;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { authFetch } from "@/features/auth";
|
||||
import type {
|
||||
GgufVariantsResponse,
|
||||
InferenceStatusResponse,
|
||||
ListLorasResponse,
|
||||
ListModelsResponse,
|
||||
|
|
@ -74,6 +75,16 @@ export async function unloadModel(payload: UnloadModelRequest): Promise<void> {
|
|||
await parseJsonOrThrow<unknown>(response);
|
||||
}
|
||||
|
||||
export async function listGgufVariants(
|
||||
repoId: string,
|
||||
hfToken?: string,
|
||||
): Promise<GgufVariantsResponse> {
|
||||
const params = new URLSearchParams({ repo_id: repoId });
|
||||
if (hfToken) params.set("hf_token", hfToken);
|
||||
const response = await authFetch(`/api/models/gguf-variants?${params}`);
|
||||
return parseJsonOrThrow<GgufVariantsResponse>(response);
|
||||
}
|
||||
|
||||
function parseSseEvent(rawEvent: string): string[] {
|
||||
const dataLines: string[] = [];
|
||||
for (const line of rawEvent.split(/\r?\n/)) {
|
||||
|
|
|
|||
|
|
@ -335,7 +335,7 @@ export function ChatPage(): ReactElement {
|
|||
}, [inferenceParams.checkpoint, lorasFromStore]);
|
||||
|
||||
const handleCheckpointChange = useCallback(
|
||||
(value: string, meta?: { isLora: boolean }) => {
|
||||
(value: string, meta?: { isLora: boolean; ggufVariant?: string }) => {
|
||||
const currentCheckpoint =
|
||||
useChatRuntimeStore.getState().params.checkpoint;
|
||||
if (!value || value === currentCheckpoint) return;
|
||||
|
|
@ -367,10 +367,10 @@ export function ChatPage(): ReactElement {
|
|||
duration: 6000,
|
||||
});
|
||||
}
|
||||
|
||||
await selectModel({
|
||||
id: value,
|
||||
isLora: meta?.isLora,
|
||||
ggufVariant: meta?.ggufVariant,
|
||||
});
|
||||
})();
|
||||
},
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ const DEFAULT_MODEL_MAX_SEQ_LENGTH = 2048;
|
|||
type SelectedModelInput = {
|
||||
id: string;
|
||||
isLora?: boolean;
|
||||
ggufVariant?: string;
|
||||
loadingDescription?: string;
|
||||
};
|
||||
|
||||
|
|
@ -42,11 +43,13 @@ function stripTrailingEpoch(input: string): string {
|
|||
function describeModel(model: {
|
||||
is_lora?: boolean;
|
||||
is_vision?: boolean;
|
||||
is_gguf?: boolean;
|
||||
}): string | undefined {
|
||||
const tags: string[] = [];
|
||||
if (model.is_gguf) tags.push("GGUF");
|
||||
if (model.is_lora) tags.push("LoRA");
|
||||
if (model.is_vision) tags.push("Vision");
|
||||
if (!model.is_lora && !model.is_vision) tags.push("Base");
|
||||
if (!model.is_lora && !model.is_vision && !model.is_gguf) tags.push("Base");
|
||||
return tags.join(" · ");
|
||||
}
|
||||
|
||||
|
|
@ -55,6 +58,7 @@ function toChatModelSummary(model: {
|
|||
name?: string | null;
|
||||
is_lora?: boolean;
|
||||
is_vision?: boolean;
|
||||
is_gguf?: boolean;
|
||||
}): ChatModelSummary {
|
||||
return {
|
||||
id: model.id,
|
||||
|
|
@ -62,6 +66,7 @@ function toChatModelSummary(model: {
|
|||
description: describeModel(model),
|
||||
isLora: Boolean(model.is_lora),
|
||||
isVision: Boolean(model.is_vision),
|
||||
isGguf: Boolean(model.is_gguf),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -160,6 +165,8 @@ export function useChatModelRuntime() {
|
|||
|
||||
const explicitIsLora =
|
||||
typeof selection === "string" ? undefined : selection.isLora;
|
||||
const ggufVariant =
|
||||
typeof selection === "string" ? undefined : selection.ggufVariant;
|
||||
const extraLoadingDescription =
|
||||
typeof selection === "string" ? undefined : selection.loadingDescription;
|
||||
const model = models.find((entry) => entry.id === modelId);
|
||||
|
|
@ -193,6 +200,7 @@ export function useChatModelRuntime() {
|
|||
max_seq_length: DEFAULT_MODEL_MAX_SEQ_LENGTH,
|
||||
load_in_4bit: true,
|
||||
is_lora: isLora,
|
||||
gguf_variant: ggufVariant ?? null,
|
||||
});
|
||||
|
||||
const currentParams = useChatRuntimeStore.getState().params;
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ export interface BackendModelDetails {
|
|||
name?: string | null;
|
||||
is_vision?: boolean;
|
||||
is_lora?: boolean;
|
||||
is_gguf?: boolean;
|
||||
}
|
||||
|
||||
export interface ListModelsResponse {
|
||||
|
|
@ -29,6 +30,20 @@ export interface LoadModelRequest {
|
|||
max_seq_length: number;
|
||||
load_in_4bit: boolean;
|
||||
is_lora: boolean;
|
||||
gguf_variant?: string | null;
|
||||
}
|
||||
|
||||
export interface GgufVariantDetail {
|
||||
filename: string;
|
||||
quant: string;
|
||||
size_bytes: number;
|
||||
}
|
||||
|
||||
export interface GgufVariantsResponse {
|
||||
repo_id: string;
|
||||
variants: GgufVariantDetail[];
|
||||
has_vision: boolean;
|
||||
default_variant: string | null;
|
||||
}
|
||||
|
||||
export interface LoadModelResponse {
|
||||
|
|
@ -37,6 +52,7 @@ export interface LoadModelResponse {
|
|||
display_name: string;
|
||||
is_vision: boolean;
|
||||
is_lora: boolean;
|
||||
is_gguf?: boolean;
|
||||
inference?: {
|
||||
temperature?: number;
|
||||
top_p?: number;
|
||||
|
|
@ -52,6 +68,7 @@ export interface UnloadModelRequest {
|
|||
export interface InferenceStatusResponse {
|
||||
active_model: string | null;
|
||||
is_vision: boolean;
|
||||
is_gguf?: boolean;
|
||||
loading: string[];
|
||||
loaded: string[];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ export interface ChatModelSummary {
|
|||
description?: string;
|
||||
isVision: boolean;
|
||||
isLora: boolean;
|
||||
isGguf?: boolean;
|
||||
}
|
||||
|
||||
export interface ChatLoraSummary {
|
||||
|
|
|
|||
|
|
@ -172,14 +172,25 @@ export function ModelSection() {
|
|||
return ids;
|
||||
}, [hfResults, selectedModel]);
|
||||
|
||||
// Filter out GGUF models — they can't be used for training
|
||||
const trainableLocalModels = useMemo(
|
||||
() =>
|
||||
localModels.filter((m) => {
|
||||
if (m.path.endsWith(".gguf")) return false;
|
||||
if (m.id.toLowerCase().includes("-gguf")) return false;
|
||||
return true;
|
||||
}),
|
||||
[localModels],
|
||||
);
|
||||
|
||||
const localMetaById = useMemo(() => {
|
||||
const map = new Map<string, LocalModelInfo>();
|
||||
for (const model of localModels) map.set(model.id, model);
|
||||
for (const model of trainableLocalModels) map.set(model.id, model);
|
||||
return map;
|
||||
}, [localModels]);
|
||||
}, [trainableLocalModels]);
|
||||
|
||||
const localResultIds = useMemo(() => {
|
||||
const ids = localModels.map((model) => model.id);
|
||||
const ids = trainableLocalModels.map((model) => model.id);
|
||||
const manual = localModelInput.trim();
|
||||
if (manual && !ids.includes(manual)) {
|
||||
ids.unshift(manual);
|
||||
|
|
@ -346,8 +357,8 @@ export function ModelSection() {
|
|||
<p className="text-[10px] text-red-500">{localModelsError}</p>
|
||||
) : (
|
||||
<p className="text-[10px] text-muted-foreground">
|
||||
{localModels.length > 0
|
||||
? `${localModels.length} local/cached models found`
|
||||
{trainableLocalModels.length > 0
|
||||
? `${trainableLocalModels.length} local/cached models found`
|
||||
: "No local models found. Enter path manually."}
|
||||
</p>
|
||||
)}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue