Use llama-server -hf mode, add GGUF variant selector, fix vision detection
Replace Python-side GGUF download with llama-server's native -hf flag for HuggingFace repos. Add frontend variant picker so users can choose quantization (Q4_K_M, Q8_0, BF16, etc.) with file sizes. Fix vision detection via mmproj files instead of hardcoding is_vision=False.
This commit is contained in:
Roland Tannous
parent
08aeeaee4b
commit
ef1cd3ac98
13 changed files with 489 additions and 82 deletions
|
|
@ -36,6 +36,9 @@ class LlamaCppBackend:
|
|||
self._port: Optional[int] = None
|
||||
self._model_identifier: Optional[str] = None
|
||||
self._gguf_path: Optional[str] = None
|
||||
self._hf_repo: Optional[str] = None
|
||||
self._hf_variant: Optional[str] = None
|
||||
self._is_vision: bool = False
|
||||
self._healthy = False
|
||||
self._lock = threading.Lock()
|
||||
self._chat_template: Optional[str] = None
|
||||
|
|
@ -56,6 +59,10 @@ class LlamaCppBackend:
|
|||
def model_identifier(self) -> Optional[str]:
|
||||
return self._model_identifier
|
||||
|
||||
@property
|
||||
def is_vision(self) -> bool:
|
||||
return self._is_vision
|
||||
|
||||
# ── Binary discovery ──────────────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -109,27 +116,33 @@ class LlamaCppBackend:
|
|||
|
||||
def load_model(
|
||||
self,
|
||||
gguf_path: str,
|
||||
*,
|
||||
# Local mode: pass a path to a .gguf file
|
||||
gguf_path: Optional[str] = None,
|
||||
# HF mode: let llama-server download via -hf "repo:quant"
|
||||
hf_repo: Optional[str] = None,
|
||||
hf_variant: Optional[str] = None,
|
||||
hf_token: Optional[str] = None,
|
||||
# Common
|
||||
model_identifier: str,
|
||||
is_vision: bool = False,
|
||||
n_ctx: int = 4096,
|
||||
n_gpu_layers: int = -1,
|
||||
n_threads: Optional[int] = None,
|
||||
) -> bool:
|
||||
"""
|
||||
Start llama-server with the given GGUF file.
|
||||
Start llama-server with a GGUF model.
|
||||
|
||||
Args:
|
||||
gguf_path: Path to the .gguf file
|
||||
model_identifier: Display identifier for the model
|
||||
n_ctx: Context window size
|
||||
n_gpu_layers: Number of layers to offload to GPU (-1 = all)
|
||||
n_threads: Number of CPU threads (None = auto)
|
||||
Two modes:
|
||||
- Local: ``gguf_path="/path/to/model.gguf"`` → uses ``-m``
|
||||
- HF: ``hf_repo="unsloth/gemma-3-4b-it-GGUF", hf_variant="Q4_K_M"`` → uses ``-hf``
|
||||
|
||||
Returns:
|
||||
True if server started and health check passed.
|
||||
In HF mode, llama-server handles downloading, caching, and
|
||||
auto-loading mmproj files for vision models.
|
||||
|
||||
Returns True if server started and health check passed.
|
||||
"""
|
||||
with self._lock:
|
||||
# Kill existing process if any
|
||||
self._kill_process()
|
||||
|
||||
binary = self._find_llama_server_binary()
|
||||
|
|
@ -140,17 +153,33 @@ class LlamaCppBackend:
|
|||
"or set LLAMA_SERVER_PATH environment variable."
|
||||
)
|
||||
|
||||
if not Path(gguf_path).is_file():
|
||||
raise FileNotFoundError(f"GGUF file not found: {gguf_path}")
|
||||
|
||||
self._port = self._find_free_port()
|
||||
cmd = [
|
||||
binary,
|
||||
"-m", gguf_path,
|
||||
"--port", str(self._port),
|
||||
"-c", str(n_ctx),
|
||||
"-ngl", str(n_gpu_layers),
|
||||
]
|
||||
|
||||
# Build command based on mode
|
||||
if hf_repo:
|
||||
hf_spec = f"{hf_repo}:{hf_variant}" if hf_variant else hf_repo
|
||||
cmd = [
|
||||
binary,
|
||||
"-hf", hf_spec,
|
||||
"--port", str(self._port),
|
||||
"-c", str(n_ctx),
|
||||
"-ngl", str(n_gpu_layers),
|
||||
]
|
||||
if hf_token:
|
||||
cmd.extend(["--hf-token", hf_token])
|
||||
elif gguf_path:
|
||||
if not Path(gguf_path).is_file():
|
||||
raise FileNotFoundError(f"GGUF file not found: {gguf_path}")
|
||||
cmd = [
|
||||
binary,
|
||||
"-m", gguf_path,
|
||||
"--port", str(self._port),
|
||||
"-c", str(n_ctx),
|
||||
"-ngl", str(n_gpu_layers),
|
||||
]
|
||||
else:
|
||||
raise ValueError("Either gguf_path or hf_repo must be provided")
|
||||
|
||||
if n_threads is not None:
|
||||
cmd.extend(["--threads", str(n_threads)])
|
||||
|
||||
|
|
@ -173,10 +202,14 @@ class LlamaCppBackend:
|
|||
)
|
||||
|
||||
self._gguf_path = gguf_path
|
||||
self._hf_repo = hf_repo
|
||||
self._hf_variant = hf_variant
|
||||
self._is_vision = is_vision
|
||||
self._model_identifier = model_identifier
|
||||
|
||||
# Wait for health
|
||||
if not self._wait_for_health(timeout=120.0):
|
||||
# HF mode: llama-server downloads before becoming healthy — need longer timeout
|
||||
timeout = 600.0 if hf_repo else 120.0
|
||||
if not self._wait_for_health(timeout=timeout):
|
||||
self._kill_process()
|
||||
raise RuntimeError(
|
||||
"llama-server failed to start. "
|
||||
|
|
@ -185,8 +218,12 @@ class LlamaCppBackend:
|
|||
|
||||
self._healthy = True
|
||||
|
||||
# Try to read chat template from GGUF metadata
|
||||
self._chat_template = self._read_gguf_chat_template(gguf_path)
|
||||
# Read chat template from local GGUF metadata (skip in HF mode —
|
||||
# llama-server handles template application internally)
|
||||
if gguf_path:
|
||||
self._chat_template = self._read_gguf_chat_template(gguf_path)
|
||||
else:
|
||||
self._chat_template = None
|
||||
|
||||
logger.info(
|
||||
f"llama-server ready on port {self._port} "
|
||||
|
|
@ -201,6 +238,9 @@ class LlamaCppBackend:
|
|||
logger.info(f"Unloaded GGUF model: {self._model_identifier}")
|
||||
self._model_identifier = None
|
||||
self._gguf_path = None
|
||||
self._hf_repo = None
|
||||
self._hf_variant = None
|
||||
self._is_vision = False
|
||||
self._port = None
|
||||
self._healthy = False
|
||||
self._chat_template = None
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ class LoadRequest(BaseModel):
|
|||
max_seq_length: int = Field(2048, ge=128, le=32768, description="Maximum sequence length")
|
||||
load_in_4bit: bool = Field(True, description="Load model in 4-bit quantization")
|
||||
is_lora: bool = Field(False, description="Whether this is a LoRA adapter")
|
||||
gguf_variant: Optional[str] = Field(None, description="GGUF quantization variant (e.g. 'Q4_K_M')")
|
||||
|
||||
|
||||
class UnloadRequest(BaseModel):
|
||||
|
|
|
|||
|
|
@ -76,6 +76,21 @@ class ModelListResponse(BaseModel):
|
|||
default_models: List[str] = Field(default_factory=list, description="List of default model IDs")
|
||||
|
||||
|
||||
class GgufVariantDetail(BaseModel):
|
||||
"""A single GGUF quantization variant in a HuggingFace repo."""
|
||||
filename: str = Field(..., description="GGUF filename (e.g., 'gemma-3-4b-it-Q4_K_M.gguf')")
|
||||
quant: str = Field(..., description="Quantization label (e.g., 'Q4_K_M')")
|
||||
size_bytes: int = Field(0, description="File size in bytes")
|
||||
|
||||
|
||||
class GgufVariantsResponse(BaseModel):
|
||||
"""Response for listing GGUF quantization variants in a HuggingFace repo."""
|
||||
repo_id: str = Field(..., description="HuggingFace repo ID")
|
||||
variants: List[GgufVariantDetail] = Field(default_factory=list, description="Available GGUF variants")
|
||||
has_vision: bool = Field(False, description="Whether the model has vision support (mmproj files)")
|
||||
default_variant: Optional[str] = Field(None, description="Recommended default quantization variant")
|
||||
|
||||
|
||||
class LocalModelInfo(BaseModel):
|
||||
"""Discovered local model candidate."""
|
||||
id: str = Field(..., description="Identifier to use for loading/training")
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@ async def load_model(request: LoadRequest):
|
|||
config = ModelConfig.from_identifier(
|
||||
model_id=request.model_path,
|
||||
hf_token=request.hf_token,
|
||||
gguf_variant=request.gguf_variant,
|
||||
)
|
||||
|
||||
if not config:
|
||||
|
|
@ -105,11 +106,25 @@ async def load_model(request: LoadRequest):
|
|||
logger.info(f"Unloading Unsloth model '{unsloth_backend.active_model_name}' before loading GGUF")
|
||||
unsloth_backend.unload_model(unsloth_backend.active_model_name)
|
||||
|
||||
success = llama_backend.load_model(
|
||||
gguf_path=config.gguf_file,
|
||||
model_identifier=config.identifier,
|
||||
n_ctx=request.max_seq_length,
|
||||
)
|
||||
# Route to HF mode or local mode based on config
|
||||
if config.gguf_hf_repo:
|
||||
# HF mode: llama-server downloads via -hf "repo:quant"
|
||||
success = llama_backend.load_model(
|
||||
hf_repo=config.gguf_hf_repo,
|
||||
hf_variant=config.gguf_variant,
|
||||
hf_token=request.hf_token,
|
||||
model_identifier=config.identifier,
|
||||
is_vision=config.is_vision,
|
||||
n_ctx=request.max_seq_length,
|
||||
)
|
||||
else:
|
||||
# Local mode: llama-server loads via -m <path>
|
||||
success = llama_backend.load_model(
|
||||
gguf_path=config.gguf_file,
|
||||
model_identifier=config.identifier,
|
||||
is_vision=config.is_vision,
|
||||
n_ctx=request.max_seq_length,
|
||||
)
|
||||
|
||||
if not success:
|
||||
raise HTTPException(
|
||||
|
|
@ -125,7 +140,7 @@ async def load_model(request: LoadRequest):
|
|||
status="loaded",
|
||||
model=config.identifier,
|
||||
display_name=config.display_name,
|
||||
is_vision=False,
|
||||
is_vision=config.is_vision,
|
||||
is_lora=False,
|
||||
is_gguf=True,
|
||||
inference=inference_config,
|
||||
|
|
@ -292,7 +307,7 @@ async def get_status():
|
|||
if llama_backend.is_loaded:
|
||||
return InferenceStatusResponse(
|
||||
active_model=llama_backend.model_identifier,
|
||||
is_vision=False,
|
||||
is_vision=llama_backend.is_vision,
|
||||
is_gguf=True,
|
||||
loading=[],
|
||||
loaded=[llama_backend.model_identifier],
|
||||
|
|
@ -425,12 +440,12 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque
|
|||
|
||||
# ── GGUF path: format prompt → proxy to llama-server ──────
|
||||
if using_gguf:
|
||||
# GGUF models don't support vision
|
||||
# Reject images if this GGUF model doesn't support vision
|
||||
image_b64 = extracted_image_b64 or payload.image_base64
|
||||
if image_b64:
|
||||
if image_b64 and not llama_backend.is_vision:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Image provided but GGUF models do not support vision.",
|
||||
detail="Image provided but current GGUF model does not support vision.",
|
||||
)
|
||||
|
||||
prompt = llama_backend.format_prompt(chat_messages, system_prompt)
|
||||
|
|
|
|||
|
|
@ -22,8 +22,10 @@ try:
|
|||
get_base_model_from_lora,
|
||||
is_vision_model,
|
||||
scan_checkpoints,
|
||||
list_gguf_variants,
|
||||
ModelConfig,
|
||||
)
|
||||
from utils.models.model_config import _pick_best_gguf, _extract_quant_label
|
||||
from core.inference import get_inference_backend
|
||||
except ImportError:
|
||||
# Fallback: try to import from parent directory
|
||||
|
|
@ -36,8 +38,10 @@ except ImportError:
|
|||
get_base_model_from_lora,
|
||||
is_vision_model,
|
||||
scan_checkpoints,
|
||||
list_gguf_variants,
|
||||
ModelConfig,
|
||||
)
|
||||
from utils.models.model_config import _pick_best_gguf, _extract_quant_label
|
||||
from core.inference import get_inference_backend
|
||||
|
||||
from models import (
|
||||
|
|
@ -51,6 +55,7 @@ from models import (
|
|||
LoRAInfo,
|
||||
ModelListResponse,
|
||||
)
|
||||
from models.models import GgufVariantDetail, GgufVariantsResponse
|
||||
from models.responses import LoRABaseModelResponse, VisionCheckResponse
|
||||
|
||||
router = APIRouter()
|
||||
|
|
@ -405,6 +410,49 @@ async def check_vision_model(
|
|||
detail=f"Failed to check vision model: {str(e)}"
|
||||
)
|
||||
|
||||
@router.get("/gguf-variants", response_model=GgufVariantsResponse)
|
||||
async def get_gguf_variants(
|
||||
repo_id: str = Query(..., description="HuggingFace repo ID (e.g. 'unsloth/gemma-3-4b-it-GGUF')"),
|
||||
hf_token: Optional[str] = Query(None, description="HuggingFace token for private repos"),
|
||||
current_subject: str = Depends(get_current_subject),
|
||||
):
|
||||
"""
|
||||
List available GGUF quantization variants for a HuggingFace repo.
|
||||
|
||||
Returns all available quantization variants (Q4_K_M, Q8_0, BF16, etc.)
|
||||
with file sizes, whether the model supports vision, and the recommended
|
||||
default variant.
|
||||
"""
|
||||
try:
|
||||
variants, has_vision = list_gguf_variants(repo_id, hf_token=hf_token)
|
||||
|
||||
# Determine default variant
|
||||
filenames = [v.filename for v in variants]
|
||||
best = _pick_best_gguf(filenames)
|
||||
default_variant = _extract_quant_label(best) if best else None
|
||||
|
||||
return GgufVariantsResponse(
|
||||
repo_id=repo_id,
|
||||
variants=[
|
||||
GgufVariantDetail(
|
||||
filename=v.filename,
|
||||
quant=v.quant,
|
||||
size_bytes=v.size_bytes,
|
||||
)
|
||||
for v in variants
|
||||
],
|
||||
has_vision=has_vision,
|
||||
default_variant=default_variant,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error listing GGUF variants for '{repo_id}': {e}", exc_info=True)
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Failed to list GGUF variants: {str(e)}",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/checkpoints", response_model=CheckpointListResponse)
|
||||
async def list_checkpoints(
|
||||
outputs_dir: str = Query(
|
||||
|
|
|
|||
|
|
@ -3,11 +3,13 @@ Model and LoRA configuration handling
|
|||
"""
|
||||
from .model_config import (
|
||||
ModelConfig,
|
||||
GgufVariantInfo,
|
||||
is_vision_model,
|
||||
scan_trained_loras,
|
||||
load_model_defaults,
|
||||
get_base_model_from_lora,
|
||||
load_model_config,
|
||||
list_gguf_variants,
|
||||
MODEL_NAME_MAPPING,
|
||||
UI_STATUS_INDICATORS,
|
||||
)
|
||||
|
|
@ -15,11 +17,13 @@ from .checkpoints import scan_checkpoints
|
|||
|
||||
__all__ = [
|
||||
'ModelConfig',
|
||||
'GgufVariantInfo',
|
||||
'is_vision_model',
|
||||
'scan_trained_loras',
|
||||
'load_model_defaults',
|
||||
'get_base_model_from_lora',
|
||||
'load_model_config',
|
||||
'list_gguf_variants',
|
||||
'MODEL_NAME_MAPPING',
|
||||
'UI_STATUS_INDICATORS',
|
||||
'scan_checkpoints',
|
||||
|
|
|
|||
|
|
@ -478,6 +478,83 @@ def _pick_best_gguf(filenames: list[str]) -> Optional[str]:
|
|||
return gguf_files[0]
|
||||
|
||||
|
||||
@dataclass
|
||||
class GgufVariantInfo:
|
||||
"""A single GGUF quantization variant from a HuggingFace repo."""
|
||||
filename: str # e.g., "gemma-3-4b-it-Q4_K_M.gguf"
|
||||
quant: str # e.g., "Q4_K_M" (extracted from filename)
|
||||
size_bytes: int # file size
|
||||
|
||||
|
||||
def _extract_quant_label(filename: str) -> str:
|
||||
"""
|
||||
Extract quantization label like Q4_K_M, IQ4_XS, BF16 from a GGUF filename.
|
||||
|
||||
Examples:
|
||||
"gemma-3-4b-it-Q4_K_M.gguf" → "Q4_K_M"
|
||||
"model-IQ4_NL.gguf" → "IQ4_NL"
|
||||
"model-BF16.gguf" → "BF16"
|
||||
"model-UD-IQ1_S.gguf" → "UD-IQ1_S"
|
||||
"""
|
||||
import re
|
||||
stem = filename.rsplit(".", 1)[0] # Remove .gguf
|
||||
# Match known quantization patterns (UD- prefix, IQ, Q, BF/F variants)
|
||||
match = re.search(
|
||||
r'(UD-)?' # Optional UD- prefix (Ultra Discrete)
|
||||
r'(IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?' # IQ variants: IQ4_XS, IQ4_NL, IQ1_S
|
||||
r'|Q[0-9]+_K_[A-Z]+' # K-quant: Q4_K_M, Q3_K_S
|
||||
r'|Q[0-9]+_[0-9]+' # Standard: Q8_0, Q5_1
|
||||
r'|Q[0-9]+_K' # Short K-quant: Q6_K
|
||||
r'|BF16|F16|F32)', # Full precision
|
||||
stem, re.IGNORECASE,
|
||||
)
|
||||
if match:
|
||||
prefix = match.group(1) or ""
|
||||
return f"{prefix}{match.group(2)}"
|
||||
# Fallback: last segment after hyphen
|
||||
return stem.split("-")[-1]
|
||||
|
||||
|
||||
def list_gguf_variants(
|
||||
repo_id: str,
|
||||
hf_token: Optional[str] = None,
|
||||
) -> tuple[list[GgufVariantInfo], bool]:
|
||||
"""
|
||||
List all GGUF quantization variants in a HuggingFace repo.
|
||||
|
||||
Separates main model files from mmproj (vision projection) files.
|
||||
The presence of mmproj files indicates a vision-capable model.
|
||||
|
||||
Returns:
|
||||
(variants, has_vision): list of non-mmproj GGUF variants + vision flag.
|
||||
"""
|
||||
from huggingface_hub import model_info as hf_model_info
|
||||
|
||||
info = hf_model_info(repo_id, token=hf_token)
|
||||
variants: list[GgufVariantInfo] = []
|
||||
has_vision = False
|
||||
|
||||
for sibling in info.siblings:
|
||||
fname = sibling.rfilename
|
||||
if not fname.endswith(".gguf"):
|
||||
continue
|
||||
size = sibling.size or 0
|
||||
|
||||
# mmproj files are vision projection models, not main model files
|
||||
if "mmproj" in fname.lower():
|
||||
has_vision = True
|
||||
continue
|
||||
|
||||
quant = _extract_quant_label(fname)
|
||||
variants.append(GgufVariantInfo(
|
||||
filename=fname,
|
||||
quant=quant,
|
||||
size_bytes=size,
|
||||
))
|
||||
|
||||
return variants, has_vision
|
||||
|
||||
|
||||
def detect_gguf_model_remote(
|
||||
repo_id: str,
|
||||
hf_token: Optional[str] = None,
|
||||
|
|
@ -692,7 +769,9 @@ class ModelConfig:
|
|||
is_vision: bool # Is this a vision model?
|
||||
is_lora: bool # Is this a lora adapter?
|
||||
is_gguf: bool = False # Is this a GGUF model?
|
||||
gguf_file: Optional[str] = None # Full path to the .gguf file
|
||||
gguf_file: Optional[str] = None # Full path to the .gguf file (local mode)
|
||||
gguf_hf_repo: Optional[str] = None # HF repo ID for -hf mode (e.g. "unsloth/gemma-3-4b-it-GGUF")
|
||||
gguf_variant: Optional[str] = None # Quantization variant (e.g. "Q4_K_M")
|
||||
base_model: Optional[str] = None # Base model (for LoRAs)
|
||||
|
||||
@classmethod
|
||||
|
|
@ -748,28 +827,32 @@ class ModelConfig:
|
|||
cls,
|
||||
model_id: str,
|
||||
hf_token: Optional[str] = None,
|
||||
is_lora: bool = False
|
||||
is_lora: bool = False,
|
||||
gguf_variant: Optional[str] = None,
|
||||
) -> Optional['ModelConfig']:
|
||||
"""
|
||||
Create ModelConfig from a clean model identifier.
|
||||
|
||||
|
||||
For FastAPI routes where the frontend sends sanitized model paths.
|
||||
No Gradio dropdown parsing - expects clean identifiers like:
|
||||
- "unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit"
|
||||
- "./outputs/my_lora_adapter"
|
||||
- "/absolute/path/to/model"
|
||||
|
||||
|
||||
Args:
|
||||
model_id: Clean model identifier (HF repo name or local path)
|
||||
hf_token: Optional HF token for vision detection on gated models
|
||||
is_lora: Whether this is a LoRA adapter
|
||||
|
||||
gguf_variant: Optional GGUF quantization variant (e.g. "Q4_K_M").
|
||||
For remote GGUF repos, specifies which quant to load via -hf.
|
||||
If None, auto-selects using _pick_best_gguf().
|
||||
|
||||
Returns:
|
||||
ModelConfig or None if configuration cannot be created
|
||||
"""
|
||||
if not model_id or not model_id.strip():
|
||||
return None
|
||||
|
||||
|
||||
identifier = model_id.strip()
|
||||
is_local = is_local_path(identifier)
|
||||
path = normalize_path(identifier) if is_local else identifier
|
||||
|
|
@ -800,8 +883,8 @@ class ModelConfig:
|
|||
# Check if the HF repo contains GGUF files
|
||||
gguf_filename = detect_gguf_model_remote(identifier, hf_token=hf_token)
|
||||
if gguf_filename:
|
||||
# Preflight: verify llama-server binary exists before downloading
|
||||
# a potentially multi-GB GGUF file
|
||||
# Preflight: verify llama-server binary exists BEFORE user waits
|
||||
# for a multi-GB download that llama-server handles natively
|
||||
from core.inference.llama_cpp import LlamaCppBackend
|
||||
if not LlamaCppBackend._find_llama_server_binary():
|
||||
raise RuntimeError(
|
||||
|
|
@ -809,24 +892,35 @@ class ModelConfig:
|
|||
"Run setup.sh to build it, or set LLAMA_SERVER_PATH."
|
||||
)
|
||||
|
||||
logger.info(f"Detected remote GGUF repo '{identifier}', file: {gguf_filename}")
|
||||
logger.info(f"Downloading GGUF file '{gguf_filename}' from '{identifier}'...")
|
||||
local_gguf_path = download_gguf_file(
|
||||
repo_id=identifier,
|
||||
filename=gguf_filename,
|
||||
hf_token=hf_token,
|
||||
# Use list_gguf_variants() to detect vision & resolve variant
|
||||
variants, has_vision = list_gguf_variants(identifier, hf_token=hf_token)
|
||||
variant = gguf_variant
|
||||
if not variant:
|
||||
# Auto-select best quantization
|
||||
variant_filenames = [v.filename for v in variants]
|
||||
best = _pick_best_gguf(variant_filenames)
|
||||
if best:
|
||||
variant = _extract_quant_label(best)
|
||||
else:
|
||||
variant = "Q4_K_M" # Fallback — llama-server's own default
|
||||
|
||||
display_name = f"{identifier.split('/')[-1]} ({variant})"
|
||||
logger.info(
|
||||
f"Detected remote GGUF repo '{identifier}', "
|
||||
f"variant={variant}, vision={has_vision}"
|
||||
)
|
||||
display_name = Path(gguf_filename).stem
|
||||
return cls(
|
||||
identifier=identifier,
|
||||
display_name=display_name,
|
||||
path=local_gguf_path,
|
||||
path=identifier,
|
||||
is_local=False,
|
||||
is_cached=True,
|
||||
is_vision=False,
|
||||
is_cached=False,
|
||||
is_vision=has_vision,
|
||||
is_lora=False,
|
||||
is_gguf=True,
|
||||
gguf_file=local_gguf_path,
|
||||
gguf_file=None,
|
||||
gguf_hf_repo=identifier,
|
||||
gguf_variant=variant,
|
||||
)
|
||||
|
||||
# Auto-detect LoRA for local paths (check adapter_config.json on disk)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ import {
|
|||
TooltipContent,
|
||||
TooltipTrigger,
|
||||
} from "@/components/ui/tooltip";
|
||||
import { listGgufVariants } from "@/features/chat/api/chat-api";
|
||||
import type { GgufVariantDetail } from "@/features/chat/types/api";
|
||||
import {
|
||||
useDebouncedValue,
|
||||
useGpuInfo,
|
||||
|
|
@ -17,7 +19,7 @@ import type { VramFitStatus } from "@/lib/vram";
|
|||
import { checkVramFit, estimateLoadingVram } from "@/lib/vram";
|
||||
import { Search01Icon } from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import { useMemo, useState, type ReactNode } from "react";
|
||||
import { useCallback, useEffect, useMemo, useState, type ReactNode } from "react";
|
||||
import type {
|
||||
LoraModelOption,
|
||||
ModelOption,
|
||||
|
|
@ -36,6 +38,15 @@ function ListLabel({ children }: { children: ReactNode }) {
|
|||
);
|
||||
}
|
||||
|
||||
/** Format bytes to a human-readable size string. */
|
||||
function formatBytes(bytes: number): string {
|
||||
if (bytes === 0) return "0 B";
|
||||
const units = ["B", "KB", "MB", "GB", "TB"];
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(1024));
|
||||
const value = bytes / 1024 ** i;
|
||||
return `${value.toFixed(value < 10 ? 1 : 0)} ${units[i]}`;
|
||||
}
|
||||
|
||||
function ModelRow({
|
||||
label,
|
||||
meta,
|
||||
|
|
@ -114,6 +125,124 @@ function ModelRow({
|
|||
return content;
|
||||
}
|
||||
|
||||
// ── GGUF Variant Expander ────────────────────────────────────
|
||||
|
||||
function GgufVariantExpander({
|
||||
repoId,
|
||||
onSelect,
|
||||
}: {
|
||||
repoId: string;
|
||||
onSelect: (id: string, meta: ModelSelectorChangeMeta) => void;
|
||||
}) {
|
||||
const [variants, setVariants] = useState<GgufVariantDetail[] | null>(null);
|
||||
const [defaultVariant, setDefaultVariant] = useState<string | null>(null);
|
||||
const [hasVision, setHasVision] = useState(false);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
let canceled = false;
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
|
||||
listGgufVariants(repoId)
|
||||
.then((res) => {
|
||||
if (canceled) return;
|
||||
setVariants(res.variants);
|
||||
setDefaultVariant(res.default_variant);
|
||||
setHasVision(res.has_vision);
|
||||
})
|
||||
.catch((err) => {
|
||||
if (canceled) return;
|
||||
setError(err instanceof Error ? err.message : "Failed to load variants");
|
||||
})
|
||||
.finally(() => {
|
||||
if (!canceled) setLoading(false);
|
||||
});
|
||||
|
||||
return () => {
|
||||
canceled = true;
|
||||
};
|
||||
}, [repoId]);
|
||||
|
||||
const handleVariantClick = useCallback(
|
||||
(quant: string) => {
|
||||
onSelect(repoId, {
|
||||
source: "hub",
|
||||
isLora: false,
|
||||
ggufVariant: quant,
|
||||
});
|
||||
},
|
||||
[repoId, onSelect],
|
||||
);
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<div className="flex items-center gap-2 px-5 py-2">
|
||||
<Spinner className="size-3 text-muted-foreground" />
|
||||
<span className="text-xs text-muted-foreground">Loading variants…</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<div className="px-5 py-2 text-xs text-destructive">{error}</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (!variants || variants.length === 0) {
|
||||
return (
|
||||
<div className="px-5 py-2 text-xs text-muted-foreground">
|
||||
No GGUF variants found.
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="pl-4 border-l-2 border-accent/50 ml-3 my-1">
|
||||
<div className="px-2 py-1 flex items-center gap-1.5">
|
||||
<span className="text-[10px] font-semibold uppercase tracking-wider text-muted-foreground">
|
||||
Quantizations
|
||||
</span>
|
||||
{hasVision && (
|
||||
<span className="text-[9px] font-medium text-blue-400">Vision</span>
|
||||
)}
|
||||
</div>
|
||||
{variants.map((v) => (
|
||||
<button
|
||||
key={v.filename}
|
||||
type="button"
|
||||
onClick={() => handleVariantClick(v.quant)}
|
||||
className={cn(
|
||||
"flex w-full items-center justify-between gap-2 rounded-md px-2.5 py-1 text-left text-sm transition-colors hover:bg-accent",
|
||||
)}
|
||||
>
|
||||
<span className="min-w-0 flex-1 truncate font-mono text-xs">
|
||||
{v.quant}
|
||||
{v.quant === defaultVariant && (
|
||||
<span className="ml-1.5 text-[9px] font-sans font-medium text-primary/70">
|
||||
recommended
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
<span className="text-[10px] text-muted-foreground shrink-0">
|
||||
{formatBytes(v.size_bytes)}
|
||||
</span>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Detect GGUF repos by naming convention ────────────────────
|
||||
|
||||
function isGgufRepo(id: string): boolean {
|
||||
return id.toUpperCase().includes("-GGUF");
|
||||
}
|
||||
|
||||
// ── Hub Model Picker ──────────────────────────────────────────
|
||||
|
||||
export function HubModelPicker({
|
||||
models,
|
||||
value,
|
||||
|
|
@ -130,6 +259,9 @@ export function HubModelPicker({
|
|||
debouncedQuery,
|
||||
);
|
||||
|
||||
// Track which GGUF repo is expanded for variant selection
|
||||
const [expandedGguf, setExpandedGguf] = useState<string | null>(null);
|
||||
|
||||
const recommendedIds = useMemo(
|
||||
() => dedupe([...models.map((model) => model.id), value ?? ""]),
|
||||
[models, value],
|
||||
|
|
@ -199,6 +331,19 @@ export function HubModelPicker({
|
|||
|
||||
const { scrollRef, sentinelRef } = useInfiniteScroll(fetchMore, results.length);
|
||||
|
||||
/** Handle clicking a model row — GGUF repos expand, others load directly. */
|
||||
const handleModelClick = useCallback(
|
||||
(id: string) => {
|
||||
if (isGgufRepo(id)) {
|
||||
// Toggle GGUF variant expander
|
||||
setExpandedGguf((prev) => (prev === id ? null : id));
|
||||
} else {
|
||||
onSelect(id, { source: "hub", isLora: false });
|
||||
}
|
||||
},
|
||||
[onSelect],
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="space-y-2">
|
||||
<div className="relative">
|
||||
|
|
@ -230,18 +375,24 @@ export function HubModelPicker({
|
|||
recommendedIds.map((id) => {
|
||||
const vram = recommendedVramMap.get(id);
|
||||
return (
|
||||
<ModelRow
|
||||
key={id}
|
||||
label={id}
|
||||
meta={vram?.detail ?? undefined}
|
||||
selected={value === id}
|
||||
onClick={() =>
|
||||
onSelect(id, { source: "hub", isLora: false })
|
||||
}
|
||||
vramStatus={vram?.status ?? null}
|
||||
vramEst={vram?.est}
|
||||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
/>
|
||||
<div key={id}>
|
||||
<ModelRow
|
||||
label={id}
|
||||
meta={
|
||||
isGgufRepo(id)
|
||||
? "GGUF"
|
||||
: vram?.detail ?? undefined
|
||||
}
|
||||
selected={value === id}
|
||||
onClick={() => handleModelClick(id)}
|
||||
vramStatus={isGgufRepo(id) ? null : vram?.status ?? null}
|
||||
vramEst={isGgufRepo(id) ? undefined : vram?.est}
|
||||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
/>
|
||||
{expandedGguf === id && (
|
||||
<GgufVariantExpander repoId={id} onSelect={onSelect} />
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})
|
||||
)}
|
||||
|
|
@ -259,18 +410,24 @@ export function HubModelPicker({
|
|||
hfIds.map((id) => {
|
||||
const vram = vramMap.get(id);
|
||||
return (
|
||||
<ModelRow
|
||||
key={id}
|
||||
label={id}
|
||||
meta={metricsById.get(id)}
|
||||
selected={value === id}
|
||||
onClick={() =>
|
||||
onSelect(id, { source: "hub", isLora: false })
|
||||
}
|
||||
vramStatus={vram?.status ?? null}
|
||||
vramEst={vram?.est}
|
||||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
/>
|
||||
<div key={id}>
|
||||
<ModelRow
|
||||
label={id}
|
||||
meta={
|
||||
isGgufRepo(id)
|
||||
? "GGUF"
|
||||
: metricsById.get(id)
|
||||
}
|
||||
selected={value === id}
|
||||
onClick={() => handleModelClick(id)}
|
||||
vramStatus={isGgufRepo(id) ? null : vram?.status ?? null}
|
||||
vramEst={isGgufRepo(id) ? undefined : vram?.est}
|
||||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
/>
|
||||
{expandedGguf === id && (
|
||||
<GgufVariantExpander repoId={id} onSelect={onSelect} />
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})
|
||||
)}
|
||||
|
|
@ -382,4 +539,3 @@ export function LoraModelPicker({
|
|||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -15,5 +15,6 @@ export interface LoraModelOption extends ModelOption {
|
|||
export interface ModelSelectorChangeMeta {
|
||||
source: "hub" | "lora";
|
||||
isLora: boolean;
|
||||
ggufVariant?: string;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { authFetch } from "@/features/auth";
|
||||
import type {
|
||||
GgufVariantsResponse,
|
||||
InferenceStatusResponse,
|
||||
ListLorasResponse,
|
||||
ListModelsResponse,
|
||||
|
|
@ -74,6 +75,16 @@ export async function unloadModel(payload: UnloadModelRequest): Promise<void> {
|
|||
await parseJsonOrThrow<unknown>(response);
|
||||
}
|
||||
|
||||
export async function listGgufVariants(
|
||||
repoId: string,
|
||||
hfToken?: string,
|
||||
): Promise<GgufVariantsResponse> {
|
||||
const params = new URLSearchParams({ repo_id: repoId });
|
||||
if (hfToken) params.set("hf_token", hfToken);
|
||||
const response = await authFetch(`/api/models/gguf-variants?${params}`);
|
||||
return parseJsonOrThrow<GgufVariantsResponse>(response);
|
||||
}
|
||||
|
||||
function parseSseEvent(rawEvent: string): string[] {
|
||||
const dataLines: string[] = [];
|
||||
for (const line of rawEvent.split(/\r?\n/)) {
|
||||
|
|
|
|||
|
|
@ -300,7 +300,7 @@ export function ChatPage(): ReactElement {
|
|||
}, [inferenceParams.checkpoint, lorasFromStore]);
|
||||
|
||||
const handleCheckpointChange = useCallback(
|
||||
(value: string, meta?: { isLora: boolean }) => {
|
||||
(value: string, meta?: { isLora: boolean; ggufVariant?: string }) => {
|
||||
const currentCheckpoint =
|
||||
useChatRuntimeStore.getState().params.checkpoint;
|
||||
if (!value || value === currentCheckpoint) return;
|
||||
|
|
@ -309,7 +309,11 @@ export function ChatPage(): ReactElement {
|
|||
if (currentCheckpoint) {
|
||||
await ejectModel();
|
||||
}
|
||||
await selectModel({ id: value, isLora: meta?.isLora });
|
||||
await selectModel({
|
||||
id: value,
|
||||
isLora: meta?.isLora,
|
||||
ggufVariant: meta?.ggufVariant,
|
||||
});
|
||||
})();
|
||||
},
|
||||
[selectModel, ejectModel],
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ const DEFAULT_MODEL_MAX_SEQ_LENGTH = 2048;
|
|||
type SelectedModelInput = {
|
||||
id: string;
|
||||
isLora?: boolean;
|
||||
ggufVariant?: string;
|
||||
};
|
||||
|
||||
const LORA_SUFFIX_RE = /_(\d{9,})$/;
|
||||
|
|
@ -159,6 +160,8 @@ export function useChatModelRuntime() {
|
|||
|
||||
const explicitIsLora =
|
||||
typeof selection === "string" ? undefined : selection.isLora;
|
||||
const ggufVariant =
|
||||
typeof selection === "string" ? undefined : selection.ggufVariant;
|
||||
const model = models.find((entry) => entry.id === modelId);
|
||||
const lora = loras.find((entry) => entry.id === modelId);
|
||||
const isLora =
|
||||
|
|
@ -181,6 +184,7 @@ export function useChatModelRuntime() {
|
|||
max_seq_length: DEFAULT_MODEL_MAX_SEQ_LENGTH,
|
||||
load_in_4bit: true,
|
||||
is_lora: isLora,
|
||||
gguf_variant: ggufVariant ?? null,
|
||||
});
|
||||
|
||||
const currentParams = useChatRuntimeStore.getState().params;
|
||||
|
|
|
|||
|
|
@ -28,6 +28,20 @@ export interface LoadModelRequest {
|
|||
max_seq_length: number;
|
||||
load_in_4bit: boolean;
|
||||
is_lora: boolean;
|
||||
gguf_variant?: string | null;
|
||||
}
|
||||
|
||||
export interface GgufVariantDetail {
|
||||
filename: string;
|
||||
quant: string;
|
||||
size_bytes: number;
|
||||
}
|
||||
|
||||
export interface GgufVariantsResponse {
|
||||
repo_id: string;
|
||||
variants: GgufVariantDetail[];
|
||||
has_vision: boolean;
|
||||
default_variant: string | null;
|
||||
}
|
||||
|
||||
export interface LoadModelResponse {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue