diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 1489365686..b5c7d3c50d 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -36,6 +36,9 @@ class LlamaCppBackend: self._port: Optional[int] = None self._model_identifier: Optional[str] = None self._gguf_path: Optional[str] = None + self._hf_repo: Optional[str] = None + self._hf_variant: Optional[str] = None + self._is_vision: bool = False self._healthy = False self._lock = threading.Lock() self._chat_template: Optional[str] = None @@ -56,6 +59,10 @@ class LlamaCppBackend: def model_identifier(self) -> Optional[str]: return self._model_identifier + @property + def is_vision(self) -> bool: + return self._is_vision + # ── Binary discovery ────────────────────────────────────────── @staticmethod @@ -109,27 +116,33 @@ class LlamaCppBackend: def load_model( self, - gguf_path: str, + *, + # Local mode: pass a path to a .gguf file + gguf_path: Optional[str] = None, + # HF mode: let llama-server download via -hf "repo:quant" + hf_repo: Optional[str] = None, + hf_variant: Optional[str] = None, + hf_token: Optional[str] = None, + # Common model_identifier: str, + is_vision: bool = False, n_ctx: int = 4096, n_gpu_layers: int = -1, n_threads: Optional[int] = None, ) -> bool: """ - Start llama-server with the given GGUF file. + Start llama-server with a GGUF model. - Args: - gguf_path: Path to the .gguf file - model_identifier: Display identifier for the model - n_ctx: Context window size - n_gpu_layers: Number of layers to offload to GPU (-1 = all) - n_threads: Number of CPU threads (None = auto) + Two modes: + - Local: ``gguf_path="/path/to/model.gguf"`` → uses ``-m`` + - HF: ``hf_repo="unsloth/gemma-3-4b-it-GGUF", hf_variant="Q4_K_M"`` → uses ``-hf`` - Returns: - True if server started and health check passed. + In HF mode, llama-server handles downloading, caching, and + auto-loading mmproj files for vision models. + + Returns True if server started and health check passed. """ with self._lock: - # Kill existing process if any self._kill_process() binary = self._find_llama_server_binary() @@ -140,17 +153,33 @@ class LlamaCppBackend: "or set LLAMA_SERVER_PATH environment variable." ) - if not Path(gguf_path).is_file(): - raise FileNotFoundError(f"GGUF file not found: {gguf_path}") - self._port = self._find_free_port() - cmd = [ - binary, - "-m", gguf_path, - "--port", str(self._port), - "-c", str(n_ctx), - "-ngl", str(n_gpu_layers), - ] + + # Build command based on mode + if hf_repo: + hf_spec = f"{hf_repo}:{hf_variant}" if hf_variant else hf_repo + cmd = [ + binary, + "-hf", hf_spec, + "--port", str(self._port), + "-c", str(n_ctx), + "-ngl", str(n_gpu_layers), + ] + if hf_token: + cmd.extend(["--hf-token", hf_token]) + elif gguf_path: + if not Path(gguf_path).is_file(): + raise FileNotFoundError(f"GGUF file not found: {gguf_path}") + cmd = [ + binary, + "-m", gguf_path, + "--port", str(self._port), + "-c", str(n_ctx), + "-ngl", str(n_gpu_layers), + ] + else: + raise ValueError("Either gguf_path or hf_repo must be provided") + if n_threads is not None: cmd.extend(["--threads", str(n_threads)]) @@ -173,10 +202,14 @@ class LlamaCppBackend: ) self._gguf_path = gguf_path + self._hf_repo = hf_repo + self._hf_variant = hf_variant + self._is_vision = is_vision self._model_identifier = model_identifier - # Wait for health - if not self._wait_for_health(timeout=120.0): + # HF mode: llama-server downloads before becoming healthy — need longer timeout + timeout = 600.0 if hf_repo else 120.0 + if not self._wait_for_health(timeout=timeout): self._kill_process() raise RuntimeError( "llama-server failed to start. " @@ -185,8 +218,12 @@ class LlamaCppBackend: self._healthy = True - # Try to read chat template from GGUF metadata - self._chat_template = self._read_gguf_chat_template(gguf_path) + # Read chat template from local GGUF metadata (skip in HF mode — + # llama-server handles template application internally) + if gguf_path: + self._chat_template = self._read_gguf_chat_template(gguf_path) + else: + self._chat_template = None logger.info( f"llama-server ready on port {self._port} " @@ -201,6 +238,9 @@ class LlamaCppBackend: logger.info(f"Unloaded GGUF model: {self._model_identifier}") self._model_identifier = None self._gguf_path = None + self._hf_repo = None + self._hf_variant = None + self._is_vision = False self._port = None self._healthy = False self._chat_template = None diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index c4a062ae3a..fc3f788ac6 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -17,6 +17,7 @@ class LoadRequest(BaseModel): max_seq_length: int = Field(2048, ge=128, le=32768, description="Maximum sequence length") load_in_4bit: bool = Field(True, description="Load model in 4-bit quantization") is_lora: bool = Field(False, description="Whether this is a LoRA adapter") + gguf_variant: Optional[str] = Field(None, description="GGUF quantization variant (e.g. 'Q4_K_M')") class UnloadRequest(BaseModel): diff --git a/studio/backend/models/models.py b/studio/backend/models/models.py index b3f9b50ed2..bd035fcbee 100644 --- a/studio/backend/models/models.py +++ b/studio/backend/models/models.py @@ -76,6 +76,21 @@ class ModelListResponse(BaseModel): default_models: List[str] = Field(default_factory=list, description="List of default model IDs") +class GgufVariantDetail(BaseModel): + """A single GGUF quantization variant in a HuggingFace repo.""" + filename: str = Field(..., description="GGUF filename (e.g., 'gemma-3-4b-it-Q4_K_M.gguf')") + quant: str = Field(..., description="Quantization label (e.g., 'Q4_K_M')") + size_bytes: int = Field(0, description="File size in bytes") + + +class GgufVariantsResponse(BaseModel): + """Response for listing GGUF quantization variants in a HuggingFace repo.""" + repo_id: str = Field(..., description="HuggingFace repo ID") + variants: List[GgufVariantDetail] = Field(default_factory=list, description="Available GGUF variants") + has_vision: bool = Field(False, description="Whether the model has vision support (mmproj files)") + default_variant: Optional[str] = Field(None, description="Recommended default quantization variant") + + class LocalModelInfo(BaseModel): """Discovered local model candidate.""" id: str = Field(..., description="Identifier to use for loading/training") diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 156578b960..b7a7a33b43 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -87,6 +87,7 @@ async def load_model(request: LoadRequest): config = ModelConfig.from_identifier( model_id=request.model_path, hf_token=request.hf_token, + gguf_variant=request.gguf_variant, ) if not config: @@ -105,11 +106,25 @@ async def load_model(request: LoadRequest): logger.info(f"Unloading Unsloth model '{unsloth_backend.active_model_name}' before loading GGUF") unsloth_backend.unload_model(unsloth_backend.active_model_name) - success = llama_backend.load_model( - gguf_path=config.gguf_file, - model_identifier=config.identifier, - n_ctx=request.max_seq_length, - ) + # Route to HF mode or local mode based on config + if config.gguf_hf_repo: + # HF mode: llama-server downloads via -hf "repo:quant" + success = llama_backend.load_model( + hf_repo=config.gguf_hf_repo, + hf_variant=config.gguf_variant, + hf_token=request.hf_token, + model_identifier=config.identifier, + is_vision=config.is_vision, + n_ctx=request.max_seq_length, + ) + else: + # Local mode: llama-server loads via -m + success = llama_backend.load_model( + gguf_path=config.gguf_file, + model_identifier=config.identifier, + is_vision=config.is_vision, + n_ctx=request.max_seq_length, + ) if not success: raise HTTPException( @@ -125,7 +140,7 @@ async def load_model(request: LoadRequest): status="loaded", model=config.identifier, display_name=config.display_name, - is_vision=False, + is_vision=config.is_vision, is_lora=False, is_gguf=True, inference=inference_config, @@ -292,7 +307,7 @@ async def get_status(): if llama_backend.is_loaded: return InferenceStatusResponse( active_model=llama_backend.model_identifier, - is_vision=False, + is_vision=llama_backend.is_vision, is_gguf=True, loading=[], loaded=[llama_backend.model_identifier], @@ -425,12 +440,12 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque # ── GGUF path: format prompt → proxy to llama-server ────── if using_gguf: - # GGUF models don't support vision + # Reject images if this GGUF model doesn't support vision image_b64 = extracted_image_b64 or payload.image_base64 - if image_b64: + if image_b64 and not llama_backend.is_vision: raise HTTPException( status_code=400, - detail="Image provided but GGUF models do not support vision.", + detail="Image provided but current GGUF model does not support vision.", ) prompt = llama_backend.format_prompt(chat_messages, system_prompt) diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py index 881f399c3d..31cc36cc45 100644 --- a/studio/backend/routes/models.py +++ b/studio/backend/routes/models.py @@ -22,8 +22,10 @@ try: get_base_model_from_lora, is_vision_model, scan_checkpoints, + list_gguf_variants, ModelConfig, ) + from utils.models.model_config import _pick_best_gguf, _extract_quant_label from core.inference import get_inference_backend except ImportError: # Fallback: try to import from parent directory @@ -36,8 +38,10 @@ except ImportError: get_base_model_from_lora, is_vision_model, scan_checkpoints, + list_gguf_variants, ModelConfig, ) + from utils.models.model_config import _pick_best_gguf, _extract_quant_label from core.inference import get_inference_backend from models import ( @@ -51,6 +55,7 @@ from models import ( LoRAInfo, ModelListResponse, ) +from models.models import GgufVariantDetail, GgufVariantsResponse from models.responses import LoRABaseModelResponse, VisionCheckResponse router = APIRouter() @@ -405,6 +410,49 @@ async def check_vision_model( detail=f"Failed to check vision model: {str(e)}" ) +@router.get("/gguf-variants", response_model=GgufVariantsResponse) +async def get_gguf_variants( + repo_id: str = Query(..., description="HuggingFace repo ID (e.g. 'unsloth/gemma-3-4b-it-GGUF')"), + hf_token: Optional[str] = Query(None, description="HuggingFace token for private repos"), + current_subject: str = Depends(get_current_subject), +): + """ + List available GGUF quantization variants for a HuggingFace repo. + + Returns all available quantization variants (Q4_K_M, Q8_0, BF16, etc.) + with file sizes, whether the model supports vision, and the recommended + default variant. + """ + try: + variants, has_vision = list_gguf_variants(repo_id, hf_token=hf_token) + + # Determine default variant + filenames = [v.filename for v in variants] + best = _pick_best_gguf(filenames) + default_variant = _extract_quant_label(best) if best else None + + return GgufVariantsResponse( + repo_id=repo_id, + variants=[ + GgufVariantDetail( + filename=v.filename, + quant=v.quant, + size_bytes=v.size_bytes, + ) + for v in variants + ], + has_vision=has_vision, + default_variant=default_variant, + ) + + except Exception as e: + logger.error(f"Error listing GGUF variants for '{repo_id}': {e}", exc_info=True) + raise HTTPException( + status_code=500, + detail=f"Failed to list GGUF variants: {str(e)}", + ) + + @router.get("/checkpoints", response_model=CheckpointListResponse) async def list_checkpoints( outputs_dir: str = Query( diff --git a/studio/backend/utils/models/__init__.py b/studio/backend/utils/models/__init__.py index 505fd35edd..4006e63908 100644 --- a/studio/backend/utils/models/__init__.py +++ b/studio/backend/utils/models/__init__.py @@ -3,11 +3,13 @@ Model and LoRA configuration handling """ from .model_config import ( ModelConfig, + GgufVariantInfo, is_vision_model, scan_trained_loras, load_model_defaults, get_base_model_from_lora, load_model_config, + list_gguf_variants, MODEL_NAME_MAPPING, UI_STATUS_INDICATORS, ) @@ -15,11 +17,13 @@ from .checkpoints import scan_checkpoints __all__ = [ 'ModelConfig', + 'GgufVariantInfo', 'is_vision_model', 'scan_trained_loras', 'load_model_defaults', 'get_base_model_from_lora', 'load_model_config', + 'list_gguf_variants', 'MODEL_NAME_MAPPING', 'UI_STATUS_INDICATORS', 'scan_checkpoints', diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 601ea75770..0c29d1c869 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -478,6 +478,83 @@ def _pick_best_gguf(filenames: list[str]) -> Optional[str]: return gguf_files[0] +@dataclass +class GgufVariantInfo: + """A single GGUF quantization variant from a HuggingFace repo.""" + filename: str # e.g., "gemma-3-4b-it-Q4_K_M.gguf" + quant: str # e.g., "Q4_K_M" (extracted from filename) + size_bytes: int # file size + + +def _extract_quant_label(filename: str) -> str: + """ + Extract quantization label like Q4_K_M, IQ4_XS, BF16 from a GGUF filename. + + Examples: + "gemma-3-4b-it-Q4_K_M.gguf" → "Q4_K_M" + "model-IQ4_NL.gguf" → "IQ4_NL" + "model-BF16.gguf" → "BF16" + "model-UD-IQ1_S.gguf" → "UD-IQ1_S" + """ + import re + stem = filename.rsplit(".", 1)[0] # Remove .gguf + # Match known quantization patterns (UD- prefix, IQ, Q, BF/F variants) + match = re.search( + r'(UD-)?' # Optional UD- prefix (Ultra Discrete) + r'(IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?' # IQ variants: IQ4_XS, IQ4_NL, IQ1_S + r'|Q[0-9]+_K_[A-Z]+' # K-quant: Q4_K_M, Q3_K_S + r'|Q[0-9]+_[0-9]+' # Standard: Q8_0, Q5_1 + r'|Q[0-9]+_K' # Short K-quant: Q6_K + r'|BF16|F16|F32)', # Full precision + stem, re.IGNORECASE, + ) + if match: + prefix = match.group(1) or "" + return f"{prefix}{match.group(2)}" + # Fallback: last segment after hyphen + return stem.split("-")[-1] + + +def list_gguf_variants( + repo_id: str, + hf_token: Optional[str] = None, +) -> tuple[list[GgufVariantInfo], bool]: + """ + List all GGUF quantization variants in a HuggingFace repo. + + Separates main model files from mmproj (vision projection) files. + The presence of mmproj files indicates a vision-capable model. + + Returns: + (variants, has_vision): list of non-mmproj GGUF variants + vision flag. + """ + from huggingface_hub import model_info as hf_model_info + + info = hf_model_info(repo_id, token=hf_token) + variants: list[GgufVariantInfo] = [] + has_vision = False + + for sibling in info.siblings: + fname = sibling.rfilename + if not fname.endswith(".gguf"): + continue + size = sibling.size or 0 + + # mmproj files are vision projection models, not main model files + if "mmproj" in fname.lower(): + has_vision = True + continue + + quant = _extract_quant_label(fname) + variants.append(GgufVariantInfo( + filename=fname, + quant=quant, + size_bytes=size, + )) + + return variants, has_vision + + def detect_gguf_model_remote( repo_id: str, hf_token: Optional[str] = None, @@ -692,7 +769,9 @@ class ModelConfig: is_vision: bool # Is this a vision model? is_lora: bool # Is this a lora adapter? is_gguf: bool = False # Is this a GGUF model? - gguf_file: Optional[str] = None # Full path to the .gguf file + gguf_file: Optional[str] = None # Full path to the .gguf file (local mode) + gguf_hf_repo: Optional[str] = None # HF repo ID for -hf mode (e.g. "unsloth/gemma-3-4b-it-GGUF") + gguf_variant: Optional[str] = None # Quantization variant (e.g. "Q4_K_M") base_model: Optional[str] = None # Base model (for LoRAs) @classmethod @@ -748,28 +827,32 @@ class ModelConfig: cls, model_id: str, hf_token: Optional[str] = None, - is_lora: bool = False + is_lora: bool = False, + gguf_variant: Optional[str] = None, ) -> Optional['ModelConfig']: """ Create ModelConfig from a clean model identifier. - + For FastAPI routes where the frontend sends sanitized model paths. No Gradio dropdown parsing - expects clean identifiers like: - "unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit" - "./outputs/my_lora_adapter" - "/absolute/path/to/model" - + Args: model_id: Clean model identifier (HF repo name or local path) hf_token: Optional HF token for vision detection on gated models is_lora: Whether this is a LoRA adapter - + gguf_variant: Optional GGUF quantization variant (e.g. "Q4_K_M"). + For remote GGUF repos, specifies which quant to load via -hf. + If None, auto-selects using _pick_best_gguf(). + Returns: ModelConfig or None if configuration cannot be created """ if not model_id or not model_id.strip(): return None - + identifier = model_id.strip() is_local = is_local_path(identifier) path = normalize_path(identifier) if is_local else identifier @@ -800,8 +883,8 @@ class ModelConfig: # Check if the HF repo contains GGUF files gguf_filename = detect_gguf_model_remote(identifier, hf_token=hf_token) if gguf_filename: - # Preflight: verify llama-server binary exists before downloading - # a potentially multi-GB GGUF file + # Preflight: verify llama-server binary exists BEFORE user waits + # for a multi-GB download that llama-server handles natively from core.inference.llama_cpp import LlamaCppBackend if not LlamaCppBackend._find_llama_server_binary(): raise RuntimeError( @@ -809,24 +892,35 @@ class ModelConfig: "Run setup.sh to build it, or set LLAMA_SERVER_PATH." ) - logger.info(f"Detected remote GGUF repo '{identifier}', file: {gguf_filename}") - logger.info(f"Downloading GGUF file '{gguf_filename}' from '{identifier}'...") - local_gguf_path = download_gguf_file( - repo_id=identifier, - filename=gguf_filename, - hf_token=hf_token, + # Use list_gguf_variants() to detect vision & resolve variant + variants, has_vision = list_gguf_variants(identifier, hf_token=hf_token) + variant = gguf_variant + if not variant: + # Auto-select best quantization + variant_filenames = [v.filename for v in variants] + best = _pick_best_gguf(variant_filenames) + if best: + variant = _extract_quant_label(best) + else: + variant = "Q4_K_M" # Fallback — llama-server's own default + + display_name = f"{identifier.split('/')[-1]} ({variant})" + logger.info( + f"Detected remote GGUF repo '{identifier}', " + f"variant={variant}, vision={has_vision}" ) - display_name = Path(gguf_filename).stem return cls( identifier=identifier, display_name=display_name, - path=local_gguf_path, + path=identifier, is_local=False, - is_cached=True, - is_vision=False, + is_cached=False, + is_vision=has_vision, is_lora=False, is_gguf=True, - gguf_file=local_gguf_path, + gguf_file=None, + gguf_hf_repo=identifier, + gguf_variant=variant, ) # Auto-detect LoRA for local paths (check adapter_config.json on disk) diff --git a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx index 80e9dd0b6b..c61f33785b 100644 --- a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx +++ b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx @@ -5,6 +5,8 @@ import { TooltipContent, TooltipTrigger, } from "@/components/ui/tooltip"; +import { listGgufVariants } from "@/features/chat/api/chat-api"; +import type { GgufVariantDetail } from "@/features/chat/types/api"; import { useDebouncedValue, useGpuInfo, @@ -17,7 +19,7 @@ import type { VramFitStatus } from "@/lib/vram"; import { checkVramFit, estimateLoadingVram } from "@/lib/vram"; import { Search01Icon } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; -import { useMemo, useState, type ReactNode } from "react"; +import { useCallback, useEffect, useMemo, useState, type ReactNode } from "react"; import type { LoraModelOption, ModelOption, @@ -36,6 +38,15 @@ function ListLabel({ children }: { children: ReactNode }) { ); } +/** Format bytes to a human-readable size string. */ +function formatBytes(bytes: number): string { + if (bytes === 0) return "0 B"; + const units = ["B", "KB", "MB", "GB", "TB"]; + const i = Math.floor(Math.log(bytes) / Math.log(1024)); + const value = bytes / 1024 ** i; + return `${value.toFixed(value < 10 ? 1 : 0)} ${units[i]}`; +} + function ModelRow({ label, meta, @@ -114,6 +125,124 @@ function ModelRow({ return content; } +// ── GGUF Variant Expander ──────────────────────────────────── + +function GgufVariantExpander({ + repoId, + onSelect, +}: { + repoId: string; + onSelect: (id: string, meta: ModelSelectorChangeMeta) => void; +}) { + const [variants, setVariants] = useState(null); + const [defaultVariant, setDefaultVariant] = useState(null); + const [hasVision, setHasVision] = useState(false); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + let canceled = false; + setLoading(true); + setError(null); + + listGgufVariants(repoId) + .then((res) => { + if (canceled) return; + setVariants(res.variants); + setDefaultVariant(res.default_variant); + setHasVision(res.has_vision); + }) + .catch((err) => { + if (canceled) return; + setError(err instanceof Error ? err.message : "Failed to load variants"); + }) + .finally(() => { + if (!canceled) setLoading(false); + }); + + return () => { + canceled = true; + }; + }, [repoId]); + + const handleVariantClick = useCallback( + (quant: string) => { + onSelect(repoId, { + source: "hub", + isLora: false, + ggufVariant: quant, + }); + }, + [repoId, onSelect], + ); + + if (loading) { + return ( +
+ + Loading variants… +
+ ); + } + + if (error) { + return ( +
{error}
+ ); + } + + if (!variants || variants.length === 0) { + return ( +
+ No GGUF variants found. +
+ ); + } + + return ( +
+
+ + Quantizations + + {hasVision && ( + Vision + )} +
+ {variants.map((v) => ( + + ))} +
+ ); +} + +// ── Detect GGUF repos by naming convention ──────────────────── + +function isGgufRepo(id: string): boolean { + return id.toUpperCase().includes("-GGUF"); +} + +// ── Hub Model Picker ────────────────────────────────────────── + export function HubModelPicker({ models, value, @@ -130,6 +259,9 @@ export function HubModelPicker({ debouncedQuery, ); + // Track which GGUF repo is expanded for variant selection + const [expandedGguf, setExpandedGguf] = useState(null); + const recommendedIds = useMemo( () => dedupe([...models.map((model) => model.id), value ?? ""]), [models, value], @@ -199,6 +331,19 @@ export function HubModelPicker({ const { scrollRef, sentinelRef } = useInfiniteScroll(fetchMore, results.length); + /** Handle clicking a model row — GGUF repos expand, others load directly. */ + const handleModelClick = useCallback( + (id: string) => { + if (isGgufRepo(id)) { + // Toggle GGUF variant expander + setExpandedGguf((prev) => (prev === id ? null : id)); + } else { + onSelect(id, { source: "hub", isLora: false }); + } + }, + [onSelect], + ); + return (
@@ -230,18 +375,24 @@ export function HubModelPicker({ recommendedIds.map((id) => { const vram = recommendedVramMap.get(id); return ( - - onSelect(id, { source: "hub", isLora: false }) - } - vramStatus={vram?.status ?? null} - vramEst={vram?.est} - gpuGb={gpu.available ? gpu.memoryTotalGb : undefined} - /> +
+ handleModelClick(id)} + vramStatus={isGgufRepo(id) ? null : vram?.status ?? null} + vramEst={isGgufRepo(id) ? undefined : vram?.est} + gpuGb={gpu.available ? gpu.memoryTotalGb : undefined} + /> + {expandedGguf === id && ( + + )} +
); }) )} @@ -259,18 +410,24 @@ export function HubModelPicker({ hfIds.map((id) => { const vram = vramMap.get(id); return ( - - onSelect(id, { source: "hub", isLora: false }) - } - vramStatus={vram?.status ?? null} - vramEst={vram?.est} - gpuGb={gpu.available ? gpu.memoryTotalGb : undefined} - /> +
+ handleModelClick(id)} + vramStatus={isGgufRepo(id) ? null : vram?.status ?? null} + vramEst={isGgufRepo(id) ? undefined : vram?.est} + gpuGb={gpu.available ? gpu.memoryTotalGb : undefined} + /> + {expandedGguf === id && ( + + )} +
); }) )} @@ -382,4 +539,3 @@ export function LoraModelPicker({
); } - diff --git a/studio/frontend/src/components/assistant-ui/model-selector/types.ts b/studio/frontend/src/components/assistant-ui/model-selector/types.ts index dcf110bfb7..a94d3dd931 100644 --- a/studio/frontend/src/components/assistant-ui/model-selector/types.ts +++ b/studio/frontend/src/components/assistant-ui/model-selector/types.ts @@ -15,5 +15,6 @@ export interface LoraModelOption extends ModelOption { export interface ModelSelectorChangeMeta { source: "hub" | "lora"; isLora: boolean; + ggufVariant?: string; } diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts index 72baf9a6f6..5d5a9551ef 100644 --- a/studio/frontend/src/features/chat/api/chat-api.ts +++ b/studio/frontend/src/features/chat/api/chat-api.ts @@ -1,5 +1,6 @@ import { authFetch } from "@/features/auth"; import type { + GgufVariantsResponse, InferenceStatusResponse, ListLorasResponse, ListModelsResponse, @@ -74,6 +75,16 @@ export async function unloadModel(payload: UnloadModelRequest): Promise { await parseJsonOrThrow(response); } +export async function listGgufVariants( + repoId: string, + hfToken?: string, +): Promise { + const params = new URLSearchParams({ repo_id: repoId }); + if (hfToken) params.set("hf_token", hfToken); + const response = await authFetch(`/api/models/gguf-variants?${params}`); + return parseJsonOrThrow(response); +} + function parseSseEvent(rawEvent: string): string[] { const dataLines: string[] = []; for (const line of rawEvent.split(/\r?\n/)) { diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx index c363704d61..7c57bdf0a4 100644 --- a/studio/frontend/src/features/chat/chat-page.tsx +++ b/studio/frontend/src/features/chat/chat-page.tsx @@ -300,7 +300,7 @@ export function ChatPage(): ReactElement { }, [inferenceParams.checkpoint, lorasFromStore]); const handleCheckpointChange = useCallback( - (value: string, meta?: { isLora: boolean }) => { + (value: string, meta?: { isLora: boolean; ggufVariant?: string }) => { const currentCheckpoint = useChatRuntimeStore.getState().params.checkpoint; if (!value || value === currentCheckpoint) return; @@ -309,7 +309,11 @@ export function ChatPage(): ReactElement { if (currentCheckpoint) { await ejectModel(); } - await selectModel({ id: value, isLora: meta?.isLora }); + await selectModel({ + id: value, + isLora: meta?.isLora, + ggufVariant: meta?.ggufVariant, + }); })(); }, [selectModel, ejectModel], diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts index 030fe55bff..24a6b3c01f 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts @@ -20,6 +20,7 @@ const DEFAULT_MODEL_MAX_SEQ_LENGTH = 2048; type SelectedModelInput = { id: string; isLora?: boolean; + ggufVariant?: string; }; const LORA_SUFFIX_RE = /_(\d{9,})$/; @@ -159,6 +160,8 @@ export function useChatModelRuntime() { const explicitIsLora = typeof selection === "string" ? undefined : selection.isLora; + const ggufVariant = + typeof selection === "string" ? undefined : selection.ggufVariant; const model = models.find((entry) => entry.id === modelId); const lora = loras.find((entry) => entry.id === modelId); const isLora = @@ -181,6 +184,7 @@ export function useChatModelRuntime() { max_seq_length: DEFAULT_MODEL_MAX_SEQ_LENGTH, load_in_4bit: true, is_lora: isLora, + gguf_variant: ggufVariant ?? null, }); const currentParams = useChatRuntimeStore.getState().params; diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index bc9268e2c2..f0a10a6cae 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -28,6 +28,20 @@ export interface LoadModelRequest { max_seq_length: number; load_in_4bit: boolean; is_lora: boolean; + gguf_variant?: string | null; +} + +export interface GgufVariantDetail { + filename: string; + quant: string; + size_bytes: number; +} + +export interface GgufVariantsResponse { + repo_id: string; + variants: GgufVariantDetail[]; + has_vision: boolean; + default_variant: string | null; } export interface LoadModelResponse {