diff --git a/setup.sh b/setup.sh index 368d62eb2f..8314ef6d75 100755 --- a/setup.sh +++ b/setup.sh @@ -224,11 +224,10 @@ fi # unsloth-zoo's GGUF export pipeline. We build: # - llama-server: for GGUF model inference # - llama-quantize: for GGUF export quantization (symlinked to root for check_llama_cpp()) -LLAMA_SERVER_BIN="$SCRIPT_DIR/llama.cpp/build/bin/llama-server" -if [ -f "$LLAMA_SERVER_BIN" ]; then - echo "" - echo "✅ llama-server already exists at $LLAMA_SERVER_BIN" -else +LLAMA_CPP_DIR="$SCRIPT_DIR/llama.cpp" +LLAMA_SERVER_BIN="$LLAMA_CPP_DIR/build/bin/llama-server" +rm -rf "$LLAMA_CPP_DIR" +{ # Check prerequisites if ! command -v cmake &>/dev/null; then echo "" @@ -240,17 +239,9 @@ else else echo "" echo "Building llama-server for GGUF inference..." - LLAMA_CPP_DIR="$SCRIPT_DIR/llama.cpp" BUILD_OK=true - if [ -d "$LLAMA_CPP_DIR/.git" ]; then - echo " llama.cpp repo already cloned, pulling latest..." - run_quiet "pull llama.cpp" git -C "$LLAMA_CPP_DIR" pull || true - else - # Remove any non-git llama.cpp directory (stale build artifacts) - rm -rf "$LLAMA_CPP_DIR" - run_quiet "clone llama.cpp" git clone --depth 1 https://github.com/ggml-org/llama.cpp.git "$LLAMA_CPP_DIR" || BUILD_OK=false - fi + run_quiet "clone llama.cpp" git clone --depth 1 https://github.com/ggml-org/llama.cpp.git "$LLAMA_CPP_DIR" || BUILD_OK=false if [ "$BUILD_OK" = true ]; then CMAKE_ARGS="" @@ -309,7 +300,7 @@ else echo "⚠️ llama-server build failed — GGUF inference won't be available, but everything else works" fi fi -fi +} # ── 9. Add shell alias (skip in Colab) ── # Note: venv activation does NOT persist across terminal sessions. diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 30fe660404..68bf871590 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -62,6 +62,10 @@ class LlamaCppBackend: def is_vision(self) -> bool: return self._is_vision + @property + def hf_variant(self) -> Optional[str]: + return self._hf_variant + # ── Binary discovery ────────────────────────────────────────── @staticmethod diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index fc3f788ac6..3a908dcfc3 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -59,6 +59,7 @@ class InferenceStatusResponse(BaseModel): active_model: Optional[str] = Field(None, description="Currently active model identifier") is_vision: bool = Field(False, description="Whether the active model is a vision model") is_gguf: bool = Field(False, description="Whether the active model is a GGUF model (llama.cpp)") + gguf_variant: Optional[str] = Field(None, description="GGUF quantization variant (e.g. Q4_K_M)") loading: List[str] = Field(default_factory=list, description="Models currently being loaded") loaded: List[str] = Field(default_factory=list, description="Models currently loaded") diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index bc450add0d..8d1c6667c0 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -5,7 +5,7 @@ import sys import time import uuid from pathlib import Path -from fastapi import APIRouter, HTTPException, Request +from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.responses import StreamingResponse, JSONResponse from typing import Optional import json @@ -50,6 +50,7 @@ from models.inference import ( CompletionChoice, CompletionMessage, ) +from auth.authentication import get_current_subject router = APIRouter() logger = logging.getLogger(__name__) @@ -71,7 +72,10 @@ def get_llama_cpp_backend() -> LlamaCppBackend: @router.post("/load", response_model=LoadResponse) -async def load_model(request: LoadRequest): +async def load_model( + request: LoadRequest, + current_subject: str = Depends(get_current_subject), +): """ Load a model for inference. @@ -194,7 +198,10 @@ async def load_model(request: LoadRequest): @router.post("/unload", response_model=UnloadResponse) -async def unload_model(request: UnloadRequest): +async def unload_model( + request: UnloadRequest, + current_subject: str = Depends(get_current_subject), +): """ Unload a model from memory. Routes to the correct backend (llama-server for GGUF, Unsloth otherwise). @@ -222,7 +229,10 @@ async def unload_model(request: UnloadRequest): @router.post("/generate/stream") -async def generate_stream(request: GenerateRequest): +async def generate_stream( + request: GenerateRequest, + current_subject: str = Depends(get_current_subject), +): """ Generate a chat response with Server-Sent Events (SSE) streaming. @@ -295,7 +305,9 @@ async def generate_stream(request: GenerateRequest): @router.get("/status", response_model=InferenceStatusResponse) -async def get_status(): +async def get_status( + current_subject: str = Depends(get_current_subject), +): """ Get current inference backend status. Reports whichever backend (Unsloth or llama-server) is currently active. @@ -309,6 +321,7 @@ async def get_status(): active_model=llama_backend.model_identifier, is_vision=llama_backend.is_vision, is_gguf=True, + gguf_variant=llama_backend.hf_variant, loading=[], loaded=[llama_backend.model_identifier], ) @@ -398,7 +411,11 @@ def _extract_content_parts( @router.post("/chat/completions") -async def openai_chat_completions(payload: ChatCompletionRequest, request: Request): +async def openai_chat_completions( + payload: ChatCompletionRequest, + request: Request, + current_subject: str = Depends(get_current_subject), +): """ OpenAI-compatible chat completions endpoint. diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 8bb9e60ed2..b84a14d226 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -491,21 +491,28 @@ def _extract_quant_label(filename: str) -> str: Extract quantization label like Q4_K_M, IQ4_XS, BF16 from a GGUF filename. Examples: - "gemma-3-4b-it-Q4_K_M.gguf" → "Q4_K_M" - "model-IQ4_NL.gguf" → "IQ4_NL" - "model-BF16.gguf" → "BF16" - "model-UD-IQ1_S.gguf" → "UD-IQ1_S" + "gemma-3-4b-it-Q4_K_M.gguf" → "Q4_K_M" + "model-IQ4_NL.gguf" → "IQ4_NL" + "model-BF16.gguf" → "BF16" + "model-UD-IQ1_S.gguf" → "UD-IQ1_S" + "model-UD-TQ1_0.gguf" → "UD-TQ1_0" + "MXFP4_MOE/model-MXFP4_MOE-0001.gguf"→ "MXFP4_MOE" """ import re - stem = filename.rsplit(".", 1)[0] # Remove .gguf - # Match known quantization patterns (UD- prefix, IQ, Q, BF/F variants) + # Use only the basename (rfilename may include directory) + basename = filename.rsplit("/", 1)[-1] + # Strip .gguf and any shard suffix (-00001-of-00010) + stem = re.sub(r'-\d{3,}-of-\d{3,}', '', basename.rsplit(".", 1)[0]) + # Match known quantization patterns match = re.search( r'(UD-)?' # Optional UD- prefix (Ultra Discrete) - r'(IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?' # IQ variants: IQ4_XS, IQ4_NL, IQ1_S - r'|Q[0-9]+_K_[A-Z]+' # K-quant: Q4_K_M, Q3_K_S - r'|Q[0-9]+_[0-9]+' # Standard: Q8_0, Q5_1 - r'|Q[0-9]+_K' # Short K-quant: Q6_K - r'|BF16|F16|F32)', # Full precision + r'(MXFP[0-9]+(?:_[A-Z0-9]+)*' # MXFP variants: MXFP4, MXFP4_MOE + r'|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?' # IQ variants: IQ4_XS, IQ4_NL, IQ1_S + r'|TQ[0-9]+_[0-9]+' # Ternary quant: TQ1_0, TQ2_0 + r'|Q[0-9]+_K_[A-Z]+' # K-quant: Q4_K_M, Q3_K_S + r'|Q[0-9]+_[0-9]+' # Standard: Q8_0, Q5_1 + r'|Q[0-9]+_K' # Short K-quant: Q6_K + r'|BF16|F16|F32)', # Full precision stem, re.IGNORECASE, ) if match: @@ -530,10 +537,13 @@ def list_gguf_variants( """ from huggingface_hub import model_info as hf_model_info - info = hf_model_info(repo_id, token=hf_token) + info = hf_model_info(repo_id, token=hf_token, files_metadata=True) variants: list[GgufVariantInfo] = [] has_vision = False + quant_totals: dict[str, int] = {} # quant -> total bytes + quant_first_file: dict[str, str] = {} # quant -> first filename (for display) + for sibling in info.siblings: fname = sibling.rfilename if not fname.endswith(".gguf"): @@ -546,10 +556,15 @@ def list_gguf_variants( continue quant = _extract_quant_label(fname) + quant_totals[quant] = quant_totals.get(quant, 0) + size + if quant not in quant_first_file: + quant_first_file[quant] = fname + + for quant, total_size in quant_totals.items(): variants.append(GgufVariantInfo( - filename=fname, + filename=quant_first_file[quant], quant=quant, - size_bytes=size, + size_bytes=total_size, )) return variants, has_vision diff --git a/studio/frontend/package.json b/studio/frontend/package.json index 432f25b1e6..63df4cc07d 100644 --- a/studio/frontend/package.json +++ b/studio/frontend/package.json @@ -65,7 +65,7 @@ "remark-gfm": "^4.0.1", "shadcn": "^3.8.4", "sonner": "^2.0.7", - "streamdown": "^2.2.0", + "streamdown": "^2.3.0", "tailwind-merge": "^3.4.0", "tailwindcss": "^4.1.18", "tw-animate-css": "^1.4.0", @@ -89,4 +89,4 @@ "typescript-eslint": "^8.55.0", "vite": "^7.3.1" } -} \ No newline at end of file +} diff --git a/studio/frontend/src/components/assistant-ui/markdown-text.tsx b/studio/frontend/src/components/assistant-ui/markdown-text.tsx index 3f18cfb940..a3f07ab885 100644 --- a/studio/frontend/src/components/assistant-ui/markdown-text.tsx +++ b/studio/frontend/src/components/assistant-ui/markdown-text.tsx @@ -1,14 +1,83 @@ "use client"; import { INTERNAL, useMessagePartText } from "@assistant-ui/react"; +import { copyToClipboard } from "@/lib/copy-to-clipboard"; +import { Copy02Icon, Tick02Icon } from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; import { code } from "@streamdown/code"; import { math } from "@streamdown/math"; import { mermaid } from "@streamdown/mermaid"; -import { Streamdown } from "streamdown"; +import { Block, type BlockProps, Streamdown } from "streamdown"; +import { useEffect, useRef, useState } from "react"; import "katex/dist/katex.min.css"; const { withSmoothContextProvider, useSmoothStatus } = INTERNAL; +function getMermaidSource(blockContent: string): string | null { + const source = blockContent.match(/```mermaid\s*([\s\S]*?)```/i)?.[1]?.trim(); + return source && source.length > 0 ? source : null; +} + +const COPY_RESET_MS = 2000; + +function MermaidCopyButton({ source }: { source: string }) { + const [copied, setCopied] = useState(false); + const resetTimeoutRef = useRef | null>(null); + + useEffect(() => { + return () => { + if (resetTimeoutRef.current) { + clearTimeout(resetTimeoutRef.current); + } + }; + }, []); + + return ( + + ); +} + +function StreamdownBlock(props: BlockProps) { + const hasMermaidFence = props.content.includes("```mermaid"); + const mermaidSource = getMermaidSource(props.content); + + if (props.isIncomplete && hasMermaidFence) { + return ( +
+ Loading diagram... +
+ ); + } + + if (mermaidSource) { + return ( +
+ + +
+ ); + } + + return ; +} + const MarkdownTextImpl = () => { const { text } = useMessagePartText(); const status = useSmoothStatus(); @@ -19,8 +88,16 @@ const MarkdownTextImpl = () => { mode="streaming" isAnimating={status.type === "running"} plugins={{ code, math, mermaid }} - controls={true} + controls={{ + mermaid: { + fullscreen: true, + download: true, + copy: false, + panZoom: true, + }, + }} shikiTheme={["github-light", "github-dark"]} + BlockComponent={StreamdownBlock} > {text} diff --git a/studio/frontend/src/components/assistant-ui/model-selector.tsx b/studio/frontend/src/components/assistant-ui/model-selector.tsx index d470034832..c6685f0214 100644 --- a/studio/frontend/src/components/assistant-ui/model-selector.tsx +++ b/studio/frontend/src/components/assistant-ui/model-selector.tsx @@ -27,6 +27,7 @@ interface ModelSelectorProps { loraModels?: LoraModelOption[]; value?: string; defaultValue?: string; + activeGgufVariant?: string | null; onValueChange?: (value: string, meta: ModelSelectorChangeMeta) => void; onEject?: () => void; variant?: "outline" | "ghost" | "muted"; @@ -158,6 +159,7 @@ export function ModelSelector({ loraModels = [], value, defaultValue, + activeGgufVariant, onValueChange, onEject, variant = "outline", @@ -202,9 +204,15 @@ export function ModelSelector({ return all; }, [loraModels, models]); - const currentModel = selected - ? optionById.get(selected) ?? { id: selected, name: selected } - : undefined; + const currentModel = useMemo(() => { + if (!selected) return undefined; + const found = optionById.get(selected); + if (activeGgufVariant) { + const desc = `GGUF · ${activeGgufVariant}`; + return found ? { ...found, description: desc } : { id: selected, name: selected, description: desc }; + } + return found ?? { id: selected, name: selected }; + }, [selected, optionById, activeGgufVariant]); function handleSelect(id: string, meta: ModelSelectorChangeMeta) { if (onValueChange) { diff --git a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx index 26f0e6e028..8a52747409 100644 --- a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx +++ b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx @@ -55,6 +55,7 @@ function ModelRow({ vramStatus, vramEst, gpuGb, + tooltipText, }: { label: string; meta?: string; @@ -63,6 +64,7 @@ function ModelRow({ vramStatus?: VramFitStatus | null; vramEst?: number; gpuGb?: number; + tooltipText?: ReactNode; }) { const exceeds = vramStatus === "exceeds"; const showVramTooltip = @@ -81,20 +83,20 @@ function ModelRow({ type="button" onClick={onClick} className={cn( - "flex w-full items-center justify-between gap-2 rounded-md px-2.5 py-1.5 text-left text-sm transition-colors hover:bg-accent", + "flex w-full items-center gap-2 rounded-md px-2.5 py-1.5 text-left text-sm transition-colors hover:bg-accent", selected && "bg-accent/60", exceeds && "opacity-50", )} > {label} - + {vramStatus === "exceeds" && ( OOM )} @@ -122,6 +124,17 @@ function ModelRow({ ); } + + if (tooltipText) { + return ( + + {content} + + {tooltipText} + + + ); + } return content; } @@ -130,9 +143,11 @@ function ModelRow({ function GgufVariantExpander({ repoId, onSelect, + gpuGb, }: { repoId: string; onSelect: (id: string, meta: ModelSelectorChangeMeta) => void; + gpuGb?: number; }) { const [variants, setVariants] = useState(null); const [defaultVariant, setDefaultVariant] = useState(null); @@ -209,28 +224,45 @@ function GgufVariantExpander({ Vision )} - {variants.map((v) => ( - - ))} + > + + {v.quant} + {v.quant === defaultVariant && ( + + recommended + + )} + + + {fitStatus === "exceeds" && ( + OOM + )} + {fitStatus === "tight" && ( + TIGHT + )} + {fitStatus === "fits" && ( + FIT + )} + + {formatBytes(v.size_bytes)} + + + + ); + })} ); } @@ -390,7 +422,7 @@ export function HubModelPicker({ gpuGb={gpu.available ? gpu.memoryTotalGb : undefined} /> {expandedGguf === id && ( - + )} ); @@ -425,7 +457,7 @@ export function HubModelPicker({ gpuGb={gpu.available ? gpu.memoryTotalGb : undefined} /> {expandedGguf === id && ( - + )} ); @@ -542,6 +574,14 @@ export function LoraModelPicker({ source: isExported ? "exported" : "lora", isLora: !isMerged && !isGguf, })} + tooltipText={ + <> + {adapter.name} + + {adapter.id} + + + } /> ); })} diff --git a/studio/frontend/src/components/markdown/mermaid-error.tsx b/studio/frontend/src/components/markdown/mermaid-error.tsx new file mode 100644 index 0000000000..4e352d4768 --- /dev/null +++ b/studio/frontend/src/components/markdown/mermaid-error.tsx @@ -0,0 +1,28 @@ +import type { MermaidErrorComponentProps } from "streamdown"; + +function hasSlashComment(chart: string): boolean { + return /(^|[^:])\/\/.*/m.test(chart); +} + +export function MermaidError({ + error, + chart, + retry, +}: MermaidErrorComponentProps) { + return ( +
+

Mermaid render failed

+

{error}

+ {hasSlashComment(chart) ? ( +

Hint: Mermaid comments use `%%`, not `//`.

+ ) : null} + +
+ ); +} diff --git a/studio/frontend/src/components/ui/combobox.tsx b/studio/frontend/src/components/ui/combobox.tsx index 8ccc40c95f..34b66ea64e 100644 --- a/studio/frontend/src/components/ui/combobox.tsx +++ b/studio/frontend/src/components/ui/combobox.tsx @@ -6,8 +6,8 @@ import { Combobox as ComboboxPrimitive } from "@base-ui/react"; import * as React from "react"; import { createContext, useContext, useState } from "react"; -import { Button } from "@/components/ui/button"; -import { useDialogPortalContainer } from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { useDialogPortalContainer } from "@/components/ui/dialog"; import { InputGroup, InputGroupAddon, @@ -144,21 +144,21 @@ function ComboboxContent({ className, side = "bottom", sideOffset = 6, - align = "start", - alignOffset = 0, - anchor, - container, - ...props -}: ComboboxPrimitive.Popup.Props & - Pick< - ComboboxPrimitive.Positioner.Props, - "side" | "align" | "sideOffset" | "alignOffset" | "anchor" - > & { - container?: HTMLElement | null; - }): React.ReactElement { - const dialogContainer = useDialogPortalContainer(); - return ( - + align = "start", + alignOffset = 0, + anchor, + container, + ...props +}: ComboboxPrimitive.Popup.Props & + Pick< + ComboboxPrimitive.Positioner.Props, + "side" | "align" | "sideOffset" | "alignOffset" | "anchor" + > & { + container?: HTMLElement | null; + }): React.ReactElement { + const dialogContainer = useDialogPortalContainer(); + return ( + = [ + { + value: "text", + label: "Text", + description: "Language models", + }, { value: "vision", label: "Vision", @@ -54,11 +59,6 @@ export const MODEL_TYPES: ReadonlyArray<{ label: "Embeddings", description: "Text embedding models", }, - { - value: "text", - label: "Text", - description: "Language models", - }, ]; export const CONTEXT_LENGTHS = [512, 1024, 2048, 4096, 8192, 16384, 32768]; diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx index 7e986721ab..14a457ad76 100644 --- a/studio/frontend/src/features/chat/chat-page.tsx +++ b/studio/frontend/src/features/chat/chat-page.tsx @@ -314,6 +314,7 @@ export function ChatPage(): ReactElement { ); const inferenceParams = useChatRuntimeStore((state) => state.params); const setInferenceParams = useChatRuntimeStore((state) => state.setParams); + const activeGgufVariant = useChatRuntimeStore((state) => state.activeGgufVariant); const autoTitle = useChatRuntimeStore((state) => state.autoTitle); const setAutoTitle = useChatRuntimeStore((state) => state.setAutoTitle); const modelsFromStore = useChatRuntimeStore((state) => state.models); @@ -336,9 +337,10 @@ export function ChatPage(): ReactElement { const handleCheckpointChange = useCallback( (value: string, meta?: { isLora: boolean; ggufVariant?: string }) => { - const currentCheckpoint = - useChatRuntimeStore.getState().params.checkpoint; - if (!value || value === currentCheckpoint) return; + const store = useChatRuntimeStore.getState(); + const currentCheckpoint = store.params.checkpoint; + const currentVariant = store.activeGgufVariant; + if (!value || (value === currentCheckpoint && (meta?.ggufVariant ?? null) === (currentVariant ?? null))) return; void (async () => { let switchNote: string | undefined; const activeThreadId = await resolveActiveSingleThreadId(view); @@ -591,6 +593,7 @@ export function ChatPage(): ReactElement { models={models} loraModels={loraModels} value={inferenceParams.checkpoint} + activeGgufVariant={activeGgufVariant} onValueChange={handleCheckpointChange} onEject={handleEject} variant="ghost" diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index 2d7ae781fa..e70b24fb54 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -157,6 +157,7 @@ export function ChatSettingsPanel({ }: ChatSettingsPanelProps) { const [presets, setPresets] = useState(BUILTIN_PRESETS); const [activePreset, setActivePreset] = useState("Default"); + const isBuiltinPreset = BUILTIN_PRESETS.some((p) => p.name === activePreset); function set(key: K) { return (v: InferenceParams[K]) => onParamsChange({ ...params, [key]: v }); @@ -165,7 +166,11 @@ export function ChatSettingsPanel({ function applyPreset(name: string) { const p = presets.find((pr) => pr.name === name); if (p) { - onParamsChange({ ...p.params, systemPrompt: params.systemPrompt }); + onParamsChange({ + ...p.params, + systemPrompt: params.systemPrompt, + checkpoint: params.checkpoint, + }); setActivePreset(name); } } @@ -219,24 +224,7 @@ export function ChatSettingsPanel({ {presets.map((p) => ( -
- {p.name} - {!BUILTIN_PRESETS.some((bp) => bp.name === p.name) && ( - - )} -
+ {p.name} ))} @@ -250,6 +238,20 @@ export function ChatSettingsPanel({ Save + diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts index a0ea17e304..fece047cd8 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts @@ -144,7 +144,7 @@ export function useChatModelRuntime() { setLoras(lorasRes.loras.map(toLoraSummary)); if (statusRes.active_model) { - setCheckpoint(statusRes.active_model); + setCheckpoint(statusRes.active_model, statusRes.gguf_variant); } } catch (error) { const message = @@ -159,14 +159,15 @@ export function useChatModelRuntime() { const selectModel = useCallback( async (selection: string | SelectedModelInput) => { const modelId = typeof selection === "string" ? selection : selection.id; - if (!modelId || params.checkpoint === modelId) { + const ggufVariant = + typeof selection === "string" ? undefined : selection.ggufVariant; + const currentVariant = useChatRuntimeStore.getState().activeGgufVariant; + if (!modelId || (params.checkpoint === modelId && (ggufVariant ?? null) === (currentVariant ?? null))) { return; } const explicitIsLora = typeof selection === "string" ? undefined : selection.isLora; - const ggufVariant = - typeof selection === "string" ? undefined : selection.ggufVariant; const extraLoadingDescription = typeof selection === "string" ? undefined : selection.loadingDescription; const model = models.find((entry) => entry.id === modelId); diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts index 03825b5861..2e3b43d606 100644 --- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts +++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts @@ -39,13 +39,14 @@ type ChatRuntimeStore = { runningByThreadId: Record; autoTitle: boolean; modelsError: string | null; + activeGgufVariant: string | null; setParams: (params: InferenceParams) => void; setModels: (models: ChatModelSummary[]) => void; setLoras: (loras: ChatLoraSummary[]) => void; setThreadRunning: (threadId: string, running: boolean) => void; setAutoTitle: (enabled: boolean) => void; setModelsError: (error: string | null) => void; - setCheckpoint: (modelId: string) => void; + setCheckpoint: (modelId: string, ggufVariant?: string | null) => void; clearCheckpoint: () => void; }; @@ -56,6 +57,7 @@ export const useChatRuntimeStore = create((set) => ({ runningByThreadId: {}, autoTitle: loadBool(AUTO_TITLE_KEY, false), modelsError: null, + activeGgufVariant: null, setParams: (params) => set({ params }), setModels: (models) => set({ models }), setLoras: (loras) => set({ loras }), @@ -75,12 +77,13 @@ export const useChatRuntimeStore = create((set) => ({ return { autoTitle }; }), setModelsError: (modelsError) => set({ modelsError }), - setCheckpoint: (modelId) => + setCheckpoint: (modelId, ggufVariant) => set((state) => ({ params: { ...state.params, checkpoint: modelId, }, + activeGgufVariant: ggufVariant ?? null, })), clearCheckpoint: () => set((state) => ({ @@ -88,5 +91,6 @@ export const useChatRuntimeStore = create((set) => ({ ...state.params, checkpoint: "", }, + activeGgufVariant: null, })), })); diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index d0b37f5cce..edf8fecae3 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -69,6 +69,7 @@ export interface InferenceStatusResponse { active_model: string | null; is_vision: boolean; is_gguf?: boolean; + gguf_variant?: string | null; loading: string[]; loaded: string[]; } diff --git a/studio/frontend/src/features/onboarding/components/steps/dataset-step.tsx b/studio/frontend/src/features/onboarding/components/steps/dataset-step.tsx index daf8e81691..f5ba2eb898 100644 --- a/studio/frontend/src/features/onboarding/components/steps/dataset-step.tsx +++ b/studio/frontend/src/features/onboarding/components/steps/dataset-step.tsx @@ -38,7 +38,7 @@ import { useHfTokenValidation, useInfiniteScroll, } from "@/hooks"; -import { cn, formatCompact } from "@/lib/utils"; +import { cn } from "@/lib/utils"; import { HfDatasetSubsetSplitSelectors, useTrainingConfigStore, @@ -254,19 +254,11 @@ export function DatasetStep() { > {(id: string) => { - const r = hfResults.find((r) => r.id === id); - const detail = r?.totalExamples - ? `${formatCompact(r.totalExamples)} rows` - : (r?.sizeCategory ?? null); return ( - + - + {id} @@ -277,15 +269,6 @@ export function DatasetStep() { {id} - {detail ? ( - - {detail} - - ) : r?.downloads != null ? ( - - ↓{formatCompact(r.downloads)} - - ) : null} ); }} diff --git a/studio/frontend/src/features/studio/sections/dataset-section.tsx b/studio/frontend/src/features/studio/sections/dataset-section.tsx index 42c00cb5d0..57e50bced5 100644 --- a/studio/frontend/src/features/studio/sections/dataset-section.tsx +++ b/studio/frontend/src/features/studio/sections/dataset-section.tsx @@ -33,7 +33,6 @@ import { useHfTokenValidation, useInfiniteScroll, } from "@/hooks"; -import { formatCompact } from "@/lib/utils"; import { HfDatasetSubsetSplitSelectors, useDatasetPreviewDialogStore, @@ -224,24 +223,11 @@ export function DatasetSection() { > {(id: string) => { - const r = hfResults.find((ds) => ds.id === id); - let detail: string | null = null; - if (r?.totalExamples) { - detail = `${formatCompact(r.totalExamples)} rows`; - } else if (r?.sizeCategory) { - detail = r.sizeCategory; - } else if (r?.downloads != null) { - detail = `↓${formatCompact(r.downloads)}`; - } return ( - + - + {id} @@ -252,11 +238,6 @@ export function DatasetSection() { {id} - {detail && ( - - {detail} - - )} ); }} diff --git a/studio/frontend/src/features/studio/sections/model-section.tsx b/studio/frontend/src/features/studio/sections/model-section.tsx index 3d3ede7c1f..7a15cea02d 100644 --- a/studio/frontend/src/features/studio/sections/model-section.tsx +++ b/studio/frontend/src/features/studio/sections/model-section.tsx @@ -331,10 +331,10 @@ export function ModelSection() { const source = model?.source === "hf_cache" ? "HF cache" : "Local dir"; return ( - + - + {model?.display_name ?? id} @@ -342,7 +342,7 @@ export function ModelSection() { {model?.path ?? id} - + {source} @@ -446,11 +446,11 @@ export function ModelSection() { - + {id} @@ -470,7 +470,7 @@ export function ModelSection() { )} - + {fitStatus === "exceeds" && ( OOM diff --git a/studio/frontend/src/hooks/use-hf-model-search.ts b/studio/frontend/src/hooks/use-hf-model-search.ts index 8fc0b32cf8..dd5f9cf8a7 100644 --- a/studio/frontend/src/hooks/use-hf-model-search.ts +++ b/studio/frontend/src/hooks/use-hf-model-search.ts @@ -133,8 +133,8 @@ export function useHfModelSearch( ...(accessToken ? { credentials: { accessToken } } : {}), }) as AsyncGenerator; } - // Dual-query: unsloth first, then general - return mergedModelIterator(trimmed, task, accessToken) as AsyncGenerator; + // Typed query: disable task filter so explicitly searched models still appear even if HF task metadata is wrong/missing. + return mergedModelIterator(trimmed, undefined, accessToken) as AsyncGenerator; }, [query, task, accessToken], );