diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 30fe660404..68bf871590 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -62,6 +62,10 @@ class LlamaCppBackend: def is_vision(self) -> bool: return self._is_vision + @property + def hf_variant(self) -> Optional[str]: + return self._hf_variant + # ── Binary discovery ────────────────────────────────────────── @staticmethod diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index fc3f788ac6..3a908dcfc3 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -59,6 +59,7 @@ class InferenceStatusResponse(BaseModel): active_model: Optional[str] = Field(None, description="Currently active model identifier") is_vision: bool = Field(False, description="Whether the active model is a vision model") is_gguf: bool = Field(False, description="Whether the active model is a GGUF model (llama.cpp)") + gguf_variant: Optional[str] = Field(None, description="GGUF quantization variant (e.g. Q4_K_M)") loading: List[str] = Field(default_factory=list, description="Models currently being loaded") loaded: List[str] = Field(default_factory=list, description="Models currently loaded") diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index bc450add0d..c685a3ab10 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -309,6 +309,7 @@ async def get_status(): active_model=llama_backend.model_identifier, is_vision=llama_backend.is_vision, is_gguf=True, + gguf_variant=llama_backend.hf_variant, loading=[], loaded=[llama_backend.model_identifier], ) diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 88cc997d81..b84a14d226 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -491,21 +491,28 @@ def _extract_quant_label(filename: str) -> str: Extract quantization label like Q4_K_M, IQ4_XS, BF16 from a GGUF filename. Examples: - "gemma-3-4b-it-Q4_K_M.gguf" → "Q4_K_M" - "model-IQ4_NL.gguf" → "IQ4_NL" - "model-BF16.gguf" → "BF16" - "model-UD-IQ1_S.gguf" → "UD-IQ1_S" + "gemma-3-4b-it-Q4_K_M.gguf" → "Q4_K_M" + "model-IQ4_NL.gguf" → "IQ4_NL" + "model-BF16.gguf" → "BF16" + "model-UD-IQ1_S.gguf" → "UD-IQ1_S" + "model-UD-TQ1_0.gguf" → "UD-TQ1_0" + "MXFP4_MOE/model-MXFP4_MOE-0001.gguf"→ "MXFP4_MOE" """ import re - stem = filename.rsplit(".", 1)[0] # Remove .gguf - # Match known quantization patterns (UD- prefix, IQ, Q, BF/F variants) + # Use only the basename (rfilename may include directory) + basename = filename.rsplit("/", 1)[-1] + # Strip .gguf and any shard suffix (-00001-of-00010) + stem = re.sub(r'-\d{3,}-of-\d{3,}', '', basename.rsplit(".", 1)[0]) + # Match known quantization patterns match = re.search( r'(UD-)?' # Optional UD- prefix (Ultra Discrete) - r'(IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?' # IQ variants: IQ4_XS, IQ4_NL, IQ1_S - r'|Q[0-9]+_K_[A-Z]+' # K-quant: Q4_K_M, Q3_K_S - r'|Q[0-9]+_[0-9]+' # Standard: Q8_0, Q5_1 - r'|Q[0-9]+_K' # Short K-quant: Q6_K - r'|BF16|F16|F32)', # Full precision + r'(MXFP[0-9]+(?:_[A-Z0-9]+)*' # MXFP variants: MXFP4, MXFP4_MOE + r'|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?' # IQ variants: IQ4_XS, IQ4_NL, IQ1_S + r'|TQ[0-9]+_[0-9]+' # Ternary quant: TQ1_0, TQ2_0 + r'|Q[0-9]+_K_[A-Z]+' # K-quant: Q4_K_M, Q3_K_S + r'|Q[0-9]+_[0-9]+' # Standard: Q8_0, Q5_1 + r'|Q[0-9]+_K' # Short K-quant: Q6_K + r'|BF16|F16|F32)', # Full precision stem, re.IGNORECASE, ) if match: @@ -534,6 +541,9 @@ def list_gguf_variants( variants: list[GgufVariantInfo] = [] has_vision = False + quant_totals: dict[str, int] = {} # quant -> total bytes + quant_first_file: dict[str, str] = {} # quant -> first filename (for display) + for sibling in info.siblings: fname = sibling.rfilename if not fname.endswith(".gguf"): @@ -546,10 +556,15 @@ def list_gguf_variants( continue quant = _extract_quant_label(fname) + quant_totals[quant] = quant_totals.get(quant, 0) + size + if quant not in quant_first_file: + quant_first_file[quant] = fname + + for quant, total_size in quant_totals.items(): variants.append(GgufVariantInfo( - filename=fname, + filename=quant_first_file[quant], quant=quant, - size_bytes=size, + size_bytes=total_size, )) return variants, has_vision diff --git a/studio/frontend/src/components/assistant-ui/model-selector.tsx b/studio/frontend/src/components/assistant-ui/model-selector.tsx index d470034832..c6685f0214 100644 --- a/studio/frontend/src/components/assistant-ui/model-selector.tsx +++ b/studio/frontend/src/components/assistant-ui/model-selector.tsx @@ -27,6 +27,7 @@ interface ModelSelectorProps { loraModels?: LoraModelOption[]; value?: string; defaultValue?: string; + activeGgufVariant?: string | null; onValueChange?: (value: string, meta: ModelSelectorChangeMeta) => void; onEject?: () => void; variant?: "outline" | "ghost" | "muted"; @@ -158,6 +159,7 @@ export function ModelSelector({ loraModels = [], value, defaultValue, + activeGgufVariant, onValueChange, onEject, variant = "outline", @@ -202,9 +204,15 @@ export function ModelSelector({ return all; }, [loraModels, models]); - const currentModel = selected - ? optionById.get(selected) ?? { id: selected, name: selected } - : undefined; + const currentModel = useMemo(() => { + if (!selected) return undefined; + const found = optionById.get(selected); + if (activeGgufVariant) { + const desc = `GGUF · ${activeGgufVariant}`; + return found ? { ...found, description: desc } : { id: selected, name: selected, description: desc }; + } + return found ?? { id: selected, name: selected }; + }, [selected, optionById, activeGgufVariant]); function handleSelect(id: string, meta: ModelSelectorChangeMeta) { if (onValueChange) { diff --git a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx index 26f0e6e028..ca1e6c73d0 100644 --- a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx +++ b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx @@ -130,9 +130,11 @@ function ModelRow({ function GgufVariantExpander({ repoId, onSelect, + gpuGb, }: { repoId: string; onSelect: (id: string, meta: ModelSelectorChangeMeta) => void; + gpuGb?: number; }) { const [variants, setVariants] = useState(null); const [defaultVariant, setDefaultVariant] = useState(null); @@ -209,28 +211,45 @@ function GgufVariantExpander({ Vision )} - {variants.map((v) => ( - - ))} + > + + {v.quant} + {v.quant === defaultVariant && ( + + recommended + + )} + + + {fitStatus === "exceeds" && ( + OOM + )} + {fitStatus === "tight" && ( + TIGHT + )} + {fitStatus === "fits" && ( + FIT + )} + + {formatBytes(v.size_bytes)} + + + + ); + })} ); } @@ -390,7 +409,7 @@ export function HubModelPicker({ gpuGb={gpu.available ? gpu.memoryTotalGb : undefined} /> {expandedGguf === id && ( - + )} ); @@ -425,7 +444,7 @@ export function HubModelPicker({ gpuGb={gpu.available ? gpu.memoryTotalGb : undefined} /> {expandedGguf === id && ( - + )} ); diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx index 7e986721ab..14a457ad76 100644 --- a/studio/frontend/src/features/chat/chat-page.tsx +++ b/studio/frontend/src/features/chat/chat-page.tsx @@ -314,6 +314,7 @@ export function ChatPage(): ReactElement { ); const inferenceParams = useChatRuntimeStore((state) => state.params); const setInferenceParams = useChatRuntimeStore((state) => state.setParams); + const activeGgufVariant = useChatRuntimeStore((state) => state.activeGgufVariant); const autoTitle = useChatRuntimeStore((state) => state.autoTitle); const setAutoTitle = useChatRuntimeStore((state) => state.setAutoTitle); const modelsFromStore = useChatRuntimeStore((state) => state.models); @@ -336,9 +337,10 @@ export function ChatPage(): ReactElement { const handleCheckpointChange = useCallback( (value: string, meta?: { isLora: boolean; ggufVariant?: string }) => { - const currentCheckpoint = - useChatRuntimeStore.getState().params.checkpoint; - if (!value || value === currentCheckpoint) return; + const store = useChatRuntimeStore.getState(); + const currentCheckpoint = store.params.checkpoint; + const currentVariant = store.activeGgufVariant; + if (!value || (value === currentCheckpoint && (meta?.ggufVariant ?? null) === (currentVariant ?? null))) return; void (async () => { let switchNote: string | undefined; const activeThreadId = await resolveActiveSingleThreadId(view); @@ -591,6 +593,7 @@ export function ChatPage(): ReactElement { models={models} loraModels={loraModels} value={inferenceParams.checkpoint} + activeGgufVariant={activeGgufVariant} onValueChange={handleCheckpointChange} onEject={handleEject} variant="ghost" diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts index a0ea17e304..fece047cd8 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts @@ -144,7 +144,7 @@ export function useChatModelRuntime() { setLoras(lorasRes.loras.map(toLoraSummary)); if (statusRes.active_model) { - setCheckpoint(statusRes.active_model); + setCheckpoint(statusRes.active_model, statusRes.gguf_variant); } } catch (error) { const message = @@ -159,14 +159,15 @@ export function useChatModelRuntime() { const selectModel = useCallback( async (selection: string | SelectedModelInput) => { const modelId = typeof selection === "string" ? selection : selection.id; - if (!modelId || params.checkpoint === modelId) { + const ggufVariant = + typeof selection === "string" ? undefined : selection.ggufVariant; + const currentVariant = useChatRuntimeStore.getState().activeGgufVariant; + if (!modelId || (params.checkpoint === modelId && (ggufVariant ?? null) === (currentVariant ?? null))) { return; } const explicitIsLora = typeof selection === "string" ? undefined : selection.isLora; - const ggufVariant = - typeof selection === "string" ? undefined : selection.ggufVariant; const extraLoadingDescription = typeof selection === "string" ? undefined : selection.loadingDescription; const model = models.find((entry) => entry.id === modelId); diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts index 03825b5861..2e3b43d606 100644 --- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts +++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts @@ -39,13 +39,14 @@ type ChatRuntimeStore = { runningByThreadId: Record; autoTitle: boolean; modelsError: string | null; + activeGgufVariant: string | null; setParams: (params: InferenceParams) => void; setModels: (models: ChatModelSummary[]) => void; setLoras: (loras: ChatLoraSummary[]) => void; setThreadRunning: (threadId: string, running: boolean) => void; setAutoTitle: (enabled: boolean) => void; setModelsError: (error: string | null) => void; - setCheckpoint: (modelId: string) => void; + setCheckpoint: (modelId: string, ggufVariant?: string | null) => void; clearCheckpoint: () => void; }; @@ -56,6 +57,7 @@ export const useChatRuntimeStore = create((set) => ({ runningByThreadId: {}, autoTitle: loadBool(AUTO_TITLE_KEY, false), modelsError: null, + activeGgufVariant: null, setParams: (params) => set({ params }), setModels: (models) => set({ models }), setLoras: (loras) => set({ loras }), @@ -75,12 +77,13 @@ export const useChatRuntimeStore = create((set) => ({ return { autoTitle }; }), setModelsError: (modelsError) => set({ modelsError }), - setCheckpoint: (modelId) => + setCheckpoint: (modelId, ggufVariant) => set((state) => ({ params: { ...state.params, checkpoint: modelId, }, + activeGgufVariant: ggufVariant ?? null, })), clearCheckpoint: () => set((state) => ({ @@ -88,5 +91,6 @@ export const useChatRuntimeStore = create((set) => ({ ...state.params, checkpoint: "", }, + activeGgufVariant: null, })), })); diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index d0b37f5cce..edf8fecae3 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -69,6 +69,7 @@ export interface InferenceStatusResponse { active_model: string | null; is_vision: boolean; is_gguf?: boolean; + gguf_variant?: string | null; loading: string[]; loaded: string[]; }