merge nightly into feature/data-reciper-enchansments
This commit is contained in:
Shine1i
commit
95dd202ab3
21 changed files with 324 additions and 168 deletions
21
setup.sh
21
setup.sh
|
|
@ -224,11 +224,10 @@ fi
|
|||
# unsloth-zoo's GGUF export pipeline. We build:
|
||||
# - llama-server: for GGUF model inference
|
||||
# - llama-quantize: for GGUF export quantization (symlinked to root for check_llama_cpp())
|
||||
LLAMA_SERVER_BIN="$SCRIPT_DIR/llama.cpp/build/bin/llama-server"
|
||||
if [ -f "$LLAMA_SERVER_BIN" ]; then
|
||||
echo ""
|
||||
echo "✅ llama-server already exists at $LLAMA_SERVER_BIN"
|
||||
else
|
||||
LLAMA_CPP_DIR="$SCRIPT_DIR/llama.cpp"
|
||||
LLAMA_SERVER_BIN="$LLAMA_CPP_DIR/build/bin/llama-server"
|
||||
rm -rf "$LLAMA_CPP_DIR"
|
||||
{
|
||||
# Check prerequisites
|
||||
if ! command -v cmake &>/dev/null; then
|
||||
echo ""
|
||||
|
|
@ -240,17 +239,9 @@ else
|
|||
else
|
||||
echo ""
|
||||
echo "Building llama-server for GGUF inference..."
|
||||
LLAMA_CPP_DIR="$SCRIPT_DIR/llama.cpp"
|
||||
|
||||
BUILD_OK=true
|
||||
if [ -d "$LLAMA_CPP_DIR/.git" ]; then
|
||||
echo " llama.cpp repo already cloned, pulling latest..."
|
||||
run_quiet "pull llama.cpp" git -C "$LLAMA_CPP_DIR" pull || true
|
||||
else
|
||||
# Remove any non-git llama.cpp directory (stale build artifacts)
|
||||
rm -rf "$LLAMA_CPP_DIR"
|
||||
run_quiet "clone llama.cpp" git clone --depth 1 https://github.com/ggml-org/llama.cpp.git "$LLAMA_CPP_DIR" || BUILD_OK=false
|
||||
fi
|
||||
run_quiet "clone llama.cpp" git clone --depth 1 https://github.com/ggml-org/llama.cpp.git "$LLAMA_CPP_DIR" || BUILD_OK=false
|
||||
|
||||
if [ "$BUILD_OK" = true ]; then
|
||||
CMAKE_ARGS=""
|
||||
|
|
@ -309,7 +300,7 @@ else
|
|||
echo "⚠️ llama-server build failed — GGUF inference won't be available, but everything else works"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# ── 9. Add shell alias (skip in Colab) ──
|
||||
# Note: venv activation does NOT persist across terminal sessions.
|
||||
|
|
|
|||
|
|
@ -62,6 +62,10 @@ class LlamaCppBackend:
|
|||
def is_vision(self) -> bool:
|
||||
return self._is_vision
|
||||
|
||||
@property
|
||||
def hf_variant(self) -> Optional[str]:
|
||||
return self._hf_variant
|
||||
|
||||
# ── Binary discovery ──────────────────────────────────────────
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@ class InferenceStatusResponse(BaseModel):
|
|||
active_model: Optional[str] = Field(None, description="Currently active model identifier")
|
||||
is_vision: bool = Field(False, description="Whether the active model is a vision model")
|
||||
is_gguf: bool = Field(False, description="Whether the active model is a GGUF model (llama.cpp)")
|
||||
gguf_variant: Optional[str] = Field(None, description="GGUF quantization variant (e.g. Q4_K_M)")
|
||||
loading: List[str] = Field(default_factory=list, description="Models currently being loaded")
|
||||
loaded: List[str] = Field(default_factory=list, description="Models currently loaded")
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import sys
|
|||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from fastapi.responses import StreamingResponse, JSONResponse
|
||||
from typing import Optional
|
||||
import json
|
||||
|
|
@ -50,6 +50,7 @@ from models.inference import (
|
|||
CompletionChoice,
|
||||
CompletionMessage,
|
||||
)
|
||||
from auth.authentication import get_current_subject
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -71,7 +72,10 @@ def get_llama_cpp_backend() -> LlamaCppBackend:
|
|||
|
||||
|
||||
@router.post("/load", response_model=LoadResponse)
|
||||
async def load_model(request: LoadRequest):
|
||||
async def load_model(
|
||||
request: LoadRequest,
|
||||
current_subject: str = Depends(get_current_subject),
|
||||
):
|
||||
"""
|
||||
Load a model for inference.
|
||||
|
||||
|
|
@ -194,7 +198,10 @@ async def load_model(request: LoadRequest):
|
|||
|
||||
|
||||
@router.post("/unload", response_model=UnloadResponse)
|
||||
async def unload_model(request: UnloadRequest):
|
||||
async def unload_model(
|
||||
request: UnloadRequest,
|
||||
current_subject: str = Depends(get_current_subject),
|
||||
):
|
||||
"""
|
||||
Unload a model from memory.
|
||||
Routes to the correct backend (llama-server for GGUF, Unsloth otherwise).
|
||||
|
|
@ -222,7 +229,10 @@ async def unload_model(request: UnloadRequest):
|
|||
|
||||
|
||||
@router.post("/generate/stream")
|
||||
async def generate_stream(request: GenerateRequest):
|
||||
async def generate_stream(
|
||||
request: GenerateRequest,
|
||||
current_subject: str = Depends(get_current_subject),
|
||||
):
|
||||
"""
|
||||
Generate a chat response with Server-Sent Events (SSE) streaming.
|
||||
|
||||
|
|
@ -295,7 +305,9 @@ async def generate_stream(request: GenerateRequest):
|
|||
|
||||
|
||||
@router.get("/status", response_model=InferenceStatusResponse)
|
||||
async def get_status():
|
||||
async def get_status(
|
||||
current_subject: str = Depends(get_current_subject),
|
||||
):
|
||||
"""
|
||||
Get current inference backend status.
|
||||
Reports whichever backend (Unsloth or llama-server) is currently active.
|
||||
|
|
@ -309,6 +321,7 @@ async def get_status():
|
|||
active_model=llama_backend.model_identifier,
|
||||
is_vision=llama_backend.is_vision,
|
||||
is_gguf=True,
|
||||
gguf_variant=llama_backend.hf_variant,
|
||||
loading=[],
|
||||
loaded=[llama_backend.model_identifier],
|
||||
)
|
||||
|
|
@ -398,7 +411,11 @@ def _extract_content_parts(
|
|||
|
||||
|
||||
@router.post("/chat/completions")
|
||||
async def openai_chat_completions(payload: ChatCompletionRequest, request: Request):
|
||||
async def openai_chat_completions(
|
||||
payload: ChatCompletionRequest,
|
||||
request: Request,
|
||||
current_subject: str = Depends(get_current_subject),
|
||||
):
|
||||
"""
|
||||
OpenAI-compatible chat completions endpoint.
|
||||
|
||||
|
|
|
|||
|
|
@ -491,21 +491,28 @@ def _extract_quant_label(filename: str) -> str:
|
|||
Extract quantization label like Q4_K_M, IQ4_XS, BF16 from a GGUF filename.
|
||||
|
||||
Examples:
|
||||
"gemma-3-4b-it-Q4_K_M.gguf" → "Q4_K_M"
|
||||
"model-IQ4_NL.gguf" → "IQ4_NL"
|
||||
"model-BF16.gguf" → "BF16"
|
||||
"model-UD-IQ1_S.gguf" → "UD-IQ1_S"
|
||||
"gemma-3-4b-it-Q4_K_M.gguf" → "Q4_K_M"
|
||||
"model-IQ4_NL.gguf" → "IQ4_NL"
|
||||
"model-BF16.gguf" → "BF16"
|
||||
"model-UD-IQ1_S.gguf" → "UD-IQ1_S"
|
||||
"model-UD-TQ1_0.gguf" → "UD-TQ1_0"
|
||||
"MXFP4_MOE/model-MXFP4_MOE-0001.gguf"→ "MXFP4_MOE"
|
||||
"""
|
||||
import re
|
||||
stem = filename.rsplit(".", 1)[0] # Remove .gguf
|
||||
# Match known quantization patterns (UD- prefix, IQ, Q, BF/F variants)
|
||||
# Use only the basename (rfilename may include directory)
|
||||
basename = filename.rsplit("/", 1)[-1]
|
||||
# Strip .gguf and any shard suffix (-00001-of-00010)
|
||||
stem = re.sub(r'-\d{3,}-of-\d{3,}', '', basename.rsplit(".", 1)[0])
|
||||
# Match known quantization patterns
|
||||
match = re.search(
|
||||
r'(UD-)?' # Optional UD- prefix (Ultra Discrete)
|
||||
r'(IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?' # IQ variants: IQ4_XS, IQ4_NL, IQ1_S
|
||||
r'|Q[0-9]+_K_[A-Z]+' # K-quant: Q4_K_M, Q3_K_S
|
||||
r'|Q[0-9]+_[0-9]+' # Standard: Q8_0, Q5_1
|
||||
r'|Q[0-9]+_K' # Short K-quant: Q6_K
|
||||
r'|BF16|F16|F32)', # Full precision
|
||||
r'(MXFP[0-9]+(?:_[A-Z0-9]+)*' # MXFP variants: MXFP4, MXFP4_MOE
|
||||
r'|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?' # IQ variants: IQ4_XS, IQ4_NL, IQ1_S
|
||||
r'|TQ[0-9]+_[0-9]+' # Ternary quant: TQ1_0, TQ2_0
|
||||
r'|Q[0-9]+_K_[A-Z]+' # K-quant: Q4_K_M, Q3_K_S
|
||||
r'|Q[0-9]+_[0-9]+' # Standard: Q8_0, Q5_1
|
||||
r'|Q[0-9]+_K' # Short K-quant: Q6_K
|
||||
r'|BF16|F16|F32)', # Full precision
|
||||
stem, re.IGNORECASE,
|
||||
)
|
||||
if match:
|
||||
|
|
@ -530,10 +537,13 @@ def list_gguf_variants(
|
|||
"""
|
||||
from huggingface_hub import model_info as hf_model_info
|
||||
|
||||
info = hf_model_info(repo_id, token=hf_token)
|
||||
info = hf_model_info(repo_id, token=hf_token, files_metadata=True)
|
||||
variants: list[GgufVariantInfo] = []
|
||||
has_vision = False
|
||||
|
||||
quant_totals: dict[str, int] = {} # quant -> total bytes
|
||||
quant_first_file: dict[str, str] = {} # quant -> first filename (for display)
|
||||
|
||||
for sibling in info.siblings:
|
||||
fname = sibling.rfilename
|
||||
if not fname.endswith(".gguf"):
|
||||
|
|
@ -546,10 +556,15 @@ def list_gguf_variants(
|
|||
continue
|
||||
|
||||
quant = _extract_quant_label(fname)
|
||||
quant_totals[quant] = quant_totals.get(quant, 0) + size
|
||||
if quant not in quant_first_file:
|
||||
quant_first_file[quant] = fname
|
||||
|
||||
for quant, total_size in quant_totals.items():
|
||||
variants.append(GgufVariantInfo(
|
||||
filename=fname,
|
||||
filename=quant_first_file[quant],
|
||||
quant=quant,
|
||||
size_bytes=size,
|
||||
size_bytes=total_size,
|
||||
))
|
||||
|
||||
return variants, has_vision
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@
|
|||
"remark-gfm": "^4.0.1",
|
||||
"shadcn": "^3.8.4",
|
||||
"sonner": "^2.0.7",
|
||||
"streamdown": "^2.2.0",
|
||||
"streamdown": "^2.3.0",
|
||||
"tailwind-merge": "^3.4.0",
|
||||
"tailwindcss": "^4.1.18",
|
||||
"tw-animate-css": "^1.4.0",
|
||||
|
|
@ -89,4 +89,4 @@
|
|||
"typescript-eslint": "^8.55.0",
|
||||
"vite": "^7.3.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,14 +1,83 @@
|
|||
"use client";
|
||||
|
||||
import { INTERNAL, useMessagePartText } from "@assistant-ui/react";
|
||||
import { copyToClipboard } from "@/lib/copy-to-clipboard";
|
||||
import { Copy02Icon, Tick02Icon } from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import { code } from "@streamdown/code";
|
||||
import { math } from "@streamdown/math";
|
||||
import { mermaid } from "@streamdown/mermaid";
|
||||
import { Streamdown } from "streamdown";
|
||||
import { Block, type BlockProps, Streamdown } from "streamdown";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import "katex/dist/katex.min.css";
|
||||
|
||||
const { withSmoothContextProvider, useSmoothStatus } = INTERNAL;
|
||||
|
||||
function getMermaidSource(blockContent: string): string | null {
|
||||
const source = blockContent.match(/```mermaid\s*([\s\S]*?)```/i)?.[1]?.trim();
|
||||
return source && source.length > 0 ? source : null;
|
||||
}
|
||||
|
||||
const COPY_RESET_MS = 2000;
|
||||
|
||||
function MermaidCopyButton({ source }: { source: string }) {
|
||||
const [copied, setCopied] = useState(false);
|
||||
const resetTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (resetTimeoutRef.current) {
|
||||
clearTimeout(resetTimeoutRef.current);
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
className="absolute top-3.5 right-20 z-20 cursor-pointer text-muted-foreground transition-all hover:text-foreground"
|
||||
title="Copy Mermaid source"
|
||||
onClick={() => {
|
||||
if (!copyToClipboard(source)) return;
|
||||
setCopied(true);
|
||||
if (resetTimeoutRef.current) {
|
||||
clearTimeout(resetTimeoutRef.current);
|
||||
}
|
||||
resetTimeoutRef.current = setTimeout(() => {
|
||||
setCopied(false);
|
||||
resetTimeoutRef.current = null;
|
||||
}, COPY_RESET_MS);
|
||||
}}
|
||||
>
|
||||
<HugeiconsIcon icon={copied ? Tick02Icon : Copy02Icon} className="size-5" />
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
function StreamdownBlock(props: BlockProps) {
|
||||
const hasMermaidFence = props.content.includes("```mermaid");
|
||||
const mermaidSource = getMermaidSource(props.content);
|
||||
|
||||
if (props.isIncomplete && hasMermaidFence) {
|
||||
return (
|
||||
<div className="my-4 flex h-48 items-center justify-center rounded-xl border border-border bg-muted/30 text-sm text-muted-foreground animate-pulse">
|
||||
Loading diagram...
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (mermaidSource) {
|
||||
return (
|
||||
<div className="relative">
|
||||
<Block {...props} />
|
||||
<MermaidCopyButton source={mermaidSource} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return <Block {...props} />;
|
||||
}
|
||||
|
||||
const MarkdownTextImpl = () => {
|
||||
const { text } = useMessagePartText();
|
||||
const status = useSmoothStatus();
|
||||
|
|
@ -19,8 +88,16 @@ const MarkdownTextImpl = () => {
|
|||
mode="streaming"
|
||||
isAnimating={status.type === "running"}
|
||||
plugins={{ code, math, mermaid }}
|
||||
controls={true}
|
||||
controls={{
|
||||
mermaid: {
|
||||
fullscreen: true,
|
||||
download: true,
|
||||
copy: false,
|
||||
panZoom: true,
|
||||
},
|
||||
}}
|
||||
shikiTheme={["github-light", "github-dark"]}
|
||||
BlockComponent={StreamdownBlock}
|
||||
>
|
||||
{text}
|
||||
</Streamdown>
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ interface ModelSelectorProps {
|
|||
loraModels?: LoraModelOption[];
|
||||
value?: string;
|
||||
defaultValue?: string;
|
||||
activeGgufVariant?: string | null;
|
||||
onValueChange?: (value: string, meta: ModelSelectorChangeMeta) => void;
|
||||
onEject?: () => void;
|
||||
variant?: "outline" | "ghost" | "muted";
|
||||
|
|
@ -158,6 +159,7 @@ export function ModelSelector({
|
|||
loraModels = [],
|
||||
value,
|
||||
defaultValue,
|
||||
activeGgufVariant,
|
||||
onValueChange,
|
||||
onEject,
|
||||
variant = "outline",
|
||||
|
|
@ -202,9 +204,15 @@ export function ModelSelector({
|
|||
return all;
|
||||
}, [loraModels, models]);
|
||||
|
||||
const currentModel = selected
|
||||
? optionById.get(selected) ?? { id: selected, name: selected }
|
||||
: undefined;
|
||||
const currentModel = useMemo(() => {
|
||||
if (!selected) return undefined;
|
||||
const found = optionById.get(selected);
|
||||
if (activeGgufVariant) {
|
||||
const desc = `GGUF · ${activeGgufVariant}`;
|
||||
return found ? { ...found, description: desc } : { id: selected, name: selected, description: desc };
|
||||
}
|
||||
return found ?? { id: selected, name: selected };
|
||||
}, [selected, optionById, activeGgufVariant]);
|
||||
|
||||
function handleSelect(id: string, meta: ModelSelectorChangeMeta) {
|
||||
if (onValueChange) {
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ function ModelRow({
|
|||
vramStatus,
|
||||
vramEst,
|
||||
gpuGb,
|
||||
tooltipText,
|
||||
}: {
|
||||
label: string;
|
||||
meta?: string;
|
||||
|
|
@ -63,6 +64,7 @@ function ModelRow({
|
|||
vramStatus?: VramFitStatus | null;
|
||||
vramEst?: number;
|
||||
gpuGb?: number;
|
||||
tooltipText?: ReactNode;
|
||||
}) {
|
||||
const exceeds = vramStatus === "exceeds";
|
||||
const showVramTooltip =
|
||||
|
|
@ -81,20 +83,20 @@ function ModelRow({
|
|||
type="button"
|
||||
onClick={onClick}
|
||||
className={cn(
|
||||
"flex w-full items-center justify-between gap-2 rounded-md px-2.5 py-1.5 text-left text-sm transition-colors hover:bg-accent",
|
||||
"flex w-full items-center gap-2 rounded-md px-2.5 py-1.5 text-left text-sm transition-colors hover:bg-accent",
|
||||
selected && "bg-accent/60",
|
||||
exceeds && "opacity-50",
|
||||
)}
|
||||
>
|
||||
<span
|
||||
className={cn(
|
||||
"min-w-0 flex-1 truncate",
|
||||
"block min-w-0 flex-1 truncate",
|
||||
exceeds && "line-through decoration-muted-foreground/50",
|
||||
)}
|
||||
>
|
||||
{label}
|
||||
</span>
|
||||
<span className="flex items-center gap-1.5 shrink-0">
|
||||
<span className="ml-auto flex items-center gap-1.5 shrink-0">
|
||||
{vramStatus === "exceeds" && (
|
||||
<span className="text-[9px] font-medium text-red-400">OOM</span>
|
||||
)}
|
||||
|
|
@ -122,6 +124,17 @@ function ModelRow({
|
|||
</Tooltip>
|
||||
);
|
||||
}
|
||||
|
||||
if (tooltipText) {
|
||||
return (
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>{content}</TooltipTrigger>
|
||||
<TooltipContent side="left" className="max-w-xs break-all">
|
||||
{tooltipText}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
);
|
||||
}
|
||||
return content;
|
||||
}
|
||||
|
||||
|
|
@ -130,9 +143,11 @@ function ModelRow({
|
|||
function GgufVariantExpander({
|
||||
repoId,
|
||||
onSelect,
|
||||
gpuGb,
|
||||
}: {
|
||||
repoId: string;
|
||||
onSelect: (id: string, meta: ModelSelectorChangeMeta) => void;
|
||||
gpuGb?: number;
|
||||
}) {
|
||||
const [variants, setVariants] = useState<GgufVariantDetail[] | null>(null);
|
||||
const [defaultVariant, setDefaultVariant] = useState<string | null>(null);
|
||||
|
|
@ -209,28 +224,45 @@ function GgufVariantExpander({
|
|||
<span className="text-[9px] font-medium text-blue-400">Vision</span>
|
||||
)}
|
||||
</div>
|
||||
{variants.map((v) => (
|
||||
<button
|
||||
key={v.filename}
|
||||
type="button"
|
||||
onClick={() => handleVariantClick(v.quant)}
|
||||
className={cn(
|
||||
"flex w-full items-center justify-between gap-2 rounded-md px-2.5 py-1 text-left text-sm transition-colors hover:bg-accent",
|
||||
)}
|
||||
>
|
||||
<span className="min-w-0 flex-1 truncate font-mono text-xs">
|
||||
{v.quant}
|
||||
{v.quant === defaultVariant && (
|
||||
<span className="ml-1.5 text-[9px] font-sans font-medium text-primary/70">
|
||||
recommended
|
||||
</span>
|
||||
{variants.map((v) => {
|
||||
const sizeGb = v.size_bytes / (1024 ** 3);
|
||||
const fitStatus = gpuGb != null && gpuGb > 0 && sizeGb > 0
|
||||
? checkVramFit(sizeGb, gpuGb)
|
||||
: null;
|
||||
return (
|
||||
<button
|
||||
key={v.filename}
|
||||
type="button"
|
||||
onClick={() => handleVariantClick(v.quant)}
|
||||
className={cn(
|
||||
"flex w-full items-center justify-between gap-2 rounded-md px-2.5 py-1 text-left text-sm transition-colors hover:bg-accent",
|
||||
)}
|
||||
</span>
|
||||
<span className="text-[10px] text-muted-foreground shrink-0">
|
||||
{formatBytes(v.size_bytes)}
|
||||
</span>
|
||||
</button>
|
||||
))}
|
||||
>
|
||||
<span className="min-w-0 flex-1 truncate font-mono text-xs">
|
||||
{v.quant}
|
||||
{v.quant === defaultVariant && (
|
||||
<span className="ml-1.5 text-[9px] font-sans font-medium text-primary/70">
|
||||
recommended
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
<span className="flex items-center gap-1.5 shrink-0">
|
||||
{fitStatus === "exceeds" && (
|
||||
<span className="text-[9px] font-medium text-red-400">OOM</span>
|
||||
)}
|
||||
{fitStatus === "tight" && (
|
||||
<span className="text-[9px] font-medium text-amber-400">TIGHT</span>
|
||||
)}
|
||||
{fitStatus === "fits" && (
|
||||
<span className="text-[9px] font-medium text-emerald-500/90">FIT</span>
|
||||
)}
|
||||
<span className="text-[10px] text-muted-foreground">
|
||||
{formatBytes(v.size_bytes)}
|
||||
</span>
|
||||
</span>
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -390,7 +422,7 @@ export function HubModelPicker({
|
|||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
/>
|
||||
{expandedGguf === id && (
|
||||
<GgufVariantExpander repoId={id} onSelect={onSelect} />
|
||||
<GgufVariantExpander repoId={id} onSelect={onSelect} gpuGb={gpu.available ? gpu.memoryTotalGb : undefined} />
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
|
|
@ -425,7 +457,7 @@ export function HubModelPicker({
|
|||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
/>
|
||||
{expandedGguf === id && (
|
||||
<GgufVariantExpander repoId={id} onSelect={onSelect} />
|
||||
<GgufVariantExpander repoId={id} onSelect={onSelect} gpuGb={gpu.available ? gpu.memoryTotalGb : undefined} />
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
|
|
@ -542,6 +574,14 @@ export function LoraModelPicker({
|
|||
source: isExported ? "exported" : "lora",
|
||||
isLora: !isMerged && !isGguf,
|
||||
})}
|
||||
tooltipText={
|
||||
<>
|
||||
<span className="block break-words">{adapter.name}</span>
|
||||
<span className="block mt-1 text-[10px] text-muted-foreground break-all">
|
||||
{adapter.id}
|
||||
</span>
|
||||
</>
|
||||
}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
|
|
|
|||
28
studio/frontend/src/components/markdown/mermaid-error.tsx
Normal file
28
studio/frontend/src/components/markdown/mermaid-error.tsx
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
import type { MermaidErrorComponentProps } from "streamdown";
|
||||
|
||||
function hasSlashComment(chart: string): boolean {
|
||||
return /(^|[^:])\/\/.*/m.test(chart);
|
||||
}
|
||||
|
||||
export function MermaidError({
|
||||
error,
|
||||
chart,
|
||||
retry,
|
||||
}: MermaidErrorComponentProps) {
|
||||
return (
|
||||
<div className="my-4 rounded-lg border border-red-300 bg-red-50 p-3 text-red-800">
|
||||
<p className="text-sm font-semibold">Mermaid render failed</p>
|
||||
<p className="mt-1 break-words font-mono text-xs">{error}</p>
|
||||
{hasSlashComment(chart) ? (
|
||||
<p className="mt-1 text-xs">Hint: Mermaid comments use `%%`, not `//`.</p>
|
||||
) : null}
|
||||
<button
|
||||
type="button"
|
||||
onClick={retry}
|
||||
className="mt-2 rounded border border-red-300 px-2 py-1 text-xs hover:bg-red-100"
|
||||
>
|
||||
Retry
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -6,8 +6,8 @@ import { Combobox as ComboboxPrimitive } from "@base-ui/react";
|
|||
import * as React from "react";
|
||||
import { createContext, useContext, useState } from "react";
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { useDialogPortalContainer } from "@/components/ui/dialog";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { useDialogPortalContainer } from "@/components/ui/dialog";
|
||||
import {
|
||||
InputGroup,
|
||||
InputGroupAddon,
|
||||
|
|
@ -144,21 +144,21 @@ function ComboboxContent({
|
|||
className,
|
||||
side = "bottom",
|
||||
sideOffset = 6,
|
||||
align = "start",
|
||||
alignOffset = 0,
|
||||
anchor,
|
||||
container,
|
||||
...props
|
||||
}: ComboboxPrimitive.Popup.Props &
|
||||
Pick<
|
||||
ComboboxPrimitive.Positioner.Props,
|
||||
"side" | "align" | "sideOffset" | "alignOffset" | "anchor"
|
||||
> & {
|
||||
container?: HTMLElement | null;
|
||||
}): React.ReactElement {
|
||||
const dialogContainer = useDialogPortalContainer();
|
||||
return (
|
||||
<ComboboxPrimitive.Portal container={container ?? dialogContainer ?? undefined}>
|
||||
align = "start",
|
||||
alignOffset = 0,
|
||||
anchor,
|
||||
container,
|
||||
...props
|
||||
}: ComboboxPrimitive.Popup.Props &
|
||||
Pick<
|
||||
ComboboxPrimitive.Positioner.Props,
|
||||
"side" | "align" | "sideOffset" | "alignOffset" | "anchor"
|
||||
> & {
|
||||
container?: HTMLElement | null;
|
||||
}): React.ReactElement {
|
||||
const dialogContainer = useDialogPortalContainer();
|
||||
return (
|
||||
<ComboboxPrimitive.Portal container={container ?? dialogContainer ?? undefined}>
|
||||
<ComboboxPrimitive.Positioner
|
||||
side={side}
|
||||
sideOffset={sideOffset}
|
||||
|
|
@ -206,7 +206,7 @@ function ComboboxItem({
|
|||
<ComboboxPrimitive.Item
|
||||
data-slot="combobox-item"
|
||||
className={cn(
|
||||
"data-highlighted:bg-accent data-highlighted:text-accent-foreground not-data-[variant=destructive]:data-highlighted:**:text-accent-foreground gap-2.5 rounded-xl corner-squircle py-2 pr-8 pl-3 text-sm [&_svg:not([class*='size-'])]:size-4 relative flex w-full cursor-pointer items-center outline-hidden select-none data-[disabled]:pointer-events-none data-[disabled]:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0",
|
||||
"data-highlighted:bg-accent data-highlighted:text-accent-foreground not-data-[variant=destructive]:data-highlighted:**:text-accent-foreground gap-2 rounded-xl corner-squircle py-2 pr-2 pl-3 text-sm [&[aria-selected=true]]:pr-7 [&_svg:not([class*='size-'])]:size-4 relative flex w-full cursor-pointer items-center outline-hidden select-none data-[disabled]:pointer-events-none data-[disabled]:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0",
|
||||
className,
|
||||
)}
|
||||
{...props}
|
||||
|
|
|
|||
|
|
@ -39,6 +39,11 @@ export const MODEL_TYPES: ReadonlyArray<{
|
|||
label: string;
|
||||
description: string;
|
||||
}> = [
|
||||
{
|
||||
value: "text",
|
||||
label: "Text",
|
||||
description: "Language models",
|
||||
},
|
||||
{
|
||||
value: "vision",
|
||||
label: "Vision",
|
||||
|
|
@ -54,11 +59,6 @@ export const MODEL_TYPES: ReadonlyArray<{
|
|||
label: "Embeddings",
|
||||
description: "Text embedding models",
|
||||
},
|
||||
{
|
||||
value: "text",
|
||||
label: "Text",
|
||||
description: "Language models",
|
||||
},
|
||||
];
|
||||
|
||||
export const CONTEXT_LENGTHS = [512, 1024, 2048, 4096, 8192, 16384, 32768];
|
||||
|
|
|
|||
|
|
@ -314,6 +314,7 @@ export function ChatPage(): ReactElement {
|
|||
);
|
||||
const inferenceParams = useChatRuntimeStore((state) => state.params);
|
||||
const setInferenceParams = useChatRuntimeStore((state) => state.setParams);
|
||||
const activeGgufVariant = useChatRuntimeStore((state) => state.activeGgufVariant);
|
||||
const autoTitle = useChatRuntimeStore((state) => state.autoTitle);
|
||||
const setAutoTitle = useChatRuntimeStore((state) => state.setAutoTitle);
|
||||
const modelsFromStore = useChatRuntimeStore((state) => state.models);
|
||||
|
|
@ -336,9 +337,10 @@ export function ChatPage(): ReactElement {
|
|||
|
||||
const handleCheckpointChange = useCallback(
|
||||
(value: string, meta?: { isLora: boolean; ggufVariant?: string }) => {
|
||||
const currentCheckpoint =
|
||||
useChatRuntimeStore.getState().params.checkpoint;
|
||||
if (!value || value === currentCheckpoint) return;
|
||||
const store = useChatRuntimeStore.getState();
|
||||
const currentCheckpoint = store.params.checkpoint;
|
||||
const currentVariant = store.activeGgufVariant;
|
||||
if (!value || (value === currentCheckpoint && (meta?.ggufVariant ?? null) === (currentVariant ?? null))) return;
|
||||
void (async () => {
|
||||
let switchNote: string | undefined;
|
||||
const activeThreadId = await resolveActiveSingleThreadId(view);
|
||||
|
|
@ -591,6 +593,7 @@ export function ChatPage(): ReactElement {
|
|||
models={models}
|
||||
loraModels={loraModels}
|
||||
value={inferenceParams.checkpoint}
|
||||
activeGgufVariant={activeGgufVariant}
|
||||
onValueChange={handleCheckpointChange}
|
||||
onEject={handleEject}
|
||||
variant="ghost"
|
||||
|
|
|
|||
|
|
@ -157,6 +157,7 @@ export function ChatSettingsPanel({
|
|||
}: ChatSettingsPanelProps) {
|
||||
const [presets, setPresets] = useState<Preset[]>(BUILTIN_PRESETS);
|
||||
const [activePreset, setActivePreset] = useState("Default");
|
||||
const isBuiltinPreset = BUILTIN_PRESETS.some((p) => p.name === activePreset);
|
||||
|
||||
function set<K extends keyof InferenceParams>(key: K) {
|
||||
return (v: InferenceParams[K]) => onParamsChange({ ...params, [key]: v });
|
||||
|
|
@ -165,7 +166,11 @@ export function ChatSettingsPanel({
|
|||
function applyPreset(name: string) {
|
||||
const p = presets.find((pr) => pr.name === name);
|
||||
if (p) {
|
||||
onParamsChange({ ...p.params, systemPrompt: params.systemPrompt });
|
||||
onParamsChange({
|
||||
...p.params,
|
||||
systemPrompt: params.systemPrompt,
|
||||
checkpoint: params.checkpoint,
|
||||
});
|
||||
setActivePreset(name);
|
||||
}
|
||||
}
|
||||
|
|
@ -219,24 +224,7 @@ export function ChatSettingsPanel({
|
|||
<SelectContent>
|
||||
{presets.map((p) => (
|
||||
<SelectItem key={p.name} value={p.name}>
|
||||
<div className="flex w-full items-center justify-between gap-2">
|
||||
<span>{p.name}</span>
|
||||
{!BUILTIN_PRESETS.some((bp) => bp.name === p.name) && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
deletePreset(p.name);
|
||||
}}
|
||||
className="text-muted-foreground hover:text-destructive"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={Delete02Icon}
|
||||
className="size-3"
|
||||
/>
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
{p.name}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
|
|
@ -250,6 +238,20 @@ export function ChatSettingsPanel({
|
|||
<HugeiconsIcon icon={FloppyDiskIcon} className="size-3.5" />
|
||||
Save
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => deletePreset(activePreset)}
|
||||
disabled={isBuiltinPreset}
|
||||
className="flex h-8 items-center gap-1.5 rounded-md border px-2.5 text-xs text-muted-foreground transition-colors hover:bg-accent disabled:cursor-not-allowed disabled:opacity-50"
|
||||
title={
|
||||
isBuiltinPreset
|
||||
? "Built-in presets cannot be deleted"
|
||||
: "Delete selected preset"
|
||||
}
|
||||
>
|
||||
<HugeiconsIcon icon={Delete02Icon} className="size-3.5" />
|
||||
Delete
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
|
|||
|
|
@ -144,7 +144,7 @@ export function useChatModelRuntime() {
|
|||
setLoras(lorasRes.loras.map(toLoraSummary));
|
||||
|
||||
if (statusRes.active_model) {
|
||||
setCheckpoint(statusRes.active_model);
|
||||
setCheckpoint(statusRes.active_model, statusRes.gguf_variant);
|
||||
}
|
||||
} catch (error) {
|
||||
const message =
|
||||
|
|
@ -159,14 +159,15 @@ export function useChatModelRuntime() {
|
|||
const selectModel = useCallback(
|
||||
async (selection: string | SelectedModelInput) => {
|
||||
const modelId = typeof selection === "string" ? selection : selection.id;
|
||||
if (!modelId || params.checkpoint === modelId) {
|
||||
const ggufVariant =
|
||||
typeof selection === "string" ? undefined : selection.ggufVariant;
|
||||
const currentVariant = useChatRuntimeStore.getState().activeGgufVariant;
|
||||
if (!modelId || (params.checkpoint === modelId && (ggufVariant ?? null) === (currentVariant ?? null))) {
|
||||
return;
|
||||
}
|
||||
|
||||
const explicitIsLora =
|
||||
typeof selection === "string" ? undefined : selection.isLora;
|
||||
const ggufVariant =
|
||||
typeof selection === "string" ? undefined : selection.ggufVariant;
|
||||
const extraLoadingDescription =
|
||||
typeof selection === "string" ? undefined : selection.loadingDescription;
|
||||
const model = models.find((entry) => entry.id === modelId);
|
||||
|
|
|
|||
|
|
@ -39,13 +39,14 @@ type ChatRuntimeStore = {
|
|||
runningByThreadId: Record<string, boolean>;
|
||||
autoTitle: boolean;
|
||||
modelsError: string | null;
|
||||
activeGgufVariant: string | null;
|
||||
setParams: (params: InferenceParams) => void;
|
||||
setModels: (models: ChatModelSummary[]) => void;
|
||||
setLoras: (loras: ChatLoraSummary[]) => void;
|
||||
setThreadRunning: (threadId: string, running: boolean) => void;
|
||||
setAutoTitle: (enabled: boolean) => void;
|
||||
setModelsError: (error: string | null) => void;
|
||||
setCheckpoint: (modelId: string) => void;
|
||||
setCheckpoint: (modelId: string, ggufVariant?: string | null) => void;
|
||||
clearCheckpoint: () => void;
|
||||
};
|
||||
|
||||
|
|
@ -56,6 +57,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set) => ({
|
|||
runningByThreadId: {},
|
||||
autoTitle: loadBool(AUTO_TITLE_KEY, false),
|
||||
modelsError: null,
|
||||
activeGgufVariant: null,
|
||||
setParams: (params) => set({ params }),
|
||||
setModels: (models) => set({ models }),
|
||||
setLoras: (loras) => set({ loras }),
|
||||
|
|
@ -75,12 +77,13 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set) => ({
|
|||
return { autoTitle };
|
||||
}),
|
||||
setModelsError: (modelsError) => set({ modelsError }),
|
||||
setCheckpoint: (modelId) =>
|
||||
setCheckpoint: (modelId, ggufVariant) =>
|
||||
set((state) => ({
|
||||
params: {
|
||||
...state.params,
|
||||
checkpoint: modelId,
|
||||
},
|
||||
activeGgufVariant: ggufVariant ?? null,
|
||||
})),
|
||||
clearCheckpoint: () =>
|
||||
set((state) => ({
|
||||
|
|
@ -88,5 +91,6 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set) => ({
|
|||
...state.params,
|
||||
checkpoint: "",
|
||||
},
|
||||
activeGgufVariant: null,
|
||||
})),
|
||||
}));
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@ export interface InferenceStatusResponse {
|
|||
active_model: string | null;
|
||||
is_vision: boolean;
|
||||
is_gguf?: boolean;
|
||||
gguf_variant?: string | null;
|
||||
loading: string[];
|
||||
loaded: string[];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ import {
|
|||
useHfTokenValidation,
|
||||
useInfiniteScroll,
|
||||
} from "@/hooks";
|
||||
import { cn, formatCompact } from "@/lib/utils";
|
||||
import { cn } from "@/lib/utils";
|
||||
import {
|
||||
HfDatasetSubsetSplitSelectors,
|
||||
useTrainingConfigStore,
|
||||
|
|
@ -254,19 +254,11 @@ export function DatasetStep() {
|
|||
>
|
||||
<ComboboxList className="p-1 !max-h-none !overflow-visible">
|
||||
{(id: string) => {
|
||||
const r = hfResults.find((r) => r.id === id);
|
||||
const detail = r?.totalExamples
|
||||
? `${formatCompact(r.totalExamples)} rows`
|
||||
: (r?.sizeCategory ?? null);
|
||||
return (
|
||||
<ComboboxItem
|
||||
key={id}
|
||||
value={id}
|
||||
className="justify-between"
|
||||
>
|
||||
<ComboboxItem key={id} value={id} className="gap-2">
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild={true}>
|
||||
<span className="min-w-0 flex-1 truncate">
|
||||
<span className="block min-w-0 flex-1 truncate">
|
||||
{id}
|
||||
</span>
|
||||
</TooltipTrigger>
|
||||
|
|
@ -277,15 +269,6 @@ export function DatasetStep() {
|
|||
{id}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
{detail ? (
|
||||
<span className="text-[10px] text-muted-foreground shrink-0">
|
||||
{detail}
|
||||
</span>
|
||||
) : r?.downloads != null ? (
|
||||
<span className="text-[10px] text-muted-foreground shrink-0">
|
||||
↓{formatCompact(r.downloads)}
|
||||
</span>
|
||||
) : null}
|
||||
</ComboboxItem>
|
||||
);
|
||||
}}
|
||||
|
|
|
|||
|
|
@ -33,7 +33,6 @@ import {
|
|||
useHfTokenValidation,
|
||||
useInfiniteScroll,
|
||||
} from "@/hooks";
|
||||
import { formatCompact } from "@/lib/utils";
|
||||
import {
|
||||
HfDatasetSubsetSplitSelectors,
|
||||
useDatasetPreviewDialogStore,
|
||||
|
|
@ -224,24 +223,11 @@ export function DatasetSection() {
|
|||
>
|
||||
<ComboboxList className="p-1 !max-h-none !overflow-visible">
|
||||
{(id: string) => {
|
||||
const r = hfResults.find((ds) => ds.id === id);
|
||||
let detail: string | null = null;
|
||||
if (r?.totalExamples) {
|
||||
detail = `${formatCompact(r.totalExamples)} rows`;
|
||||
} else if (r?.sizeCategory) {
|
||||
detail = r.sizeCategory;
|
||||
} else if (r?.downloads != null) {
|
||||
detail = `↓${formatCompact(r.downloads)}`;
|
||||
}
|
||||
return (
|
||||
<ComboboxItem
|
||||
key={id}
|
||||
value={id}
|
||||
className="justify-between"
|
||||
>
|
||||
<ComboboxItem key={id} value={id} className="gap-2">
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild={true}>
|
||||
<span className="min-w-0 flex-1 truncate">
|
||||
<span className="block min-w-0 flex-1 truncate">
|
||||
{id}
|
||||
</span>
|
||||
</TooltipTrigger>
|
||||
|
|
@ -252,11 +238,6 @@ export function DatasetSection() {
|
|||
{id}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
{detail && (
|
||||
<span className="text-[10px] text-muted-foreground shrink-0">
|
||||
{detail}
|
||||
</span>
|
||||
)}
|
||||
</ComboboxItem>
|
||||
);
|
||||
}}
|
||||
|
|
|
|||
|
|
@ -331,10 +331,10 @@ export function ModelSection() {
|
|||
const source =
|
||||
model?.source === "hf_cache" ? "HF cache" : "Local dir";
|
||||
return (
|
||||
<ComboboxItem key={id} value={id} className="justify-between">
|
||||
<ComboboxItem key={id} value={id} className="gap-2">
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild={true}>
|
||||
<span className="min-w-0 flex-1 truncate">
|
||||
<span className="block min-w-0 flex-1 truncate">
|
||||
{model?.display_name ?? id}
|
||||
</span>
|
||||
</TooltipTrigger>
|
||||
|
|
@ -342,7 +342,7 @@ export function ModelSection() {
|
|||
{model?.path ?? id}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
<span className="shrink-0 text-[10px] text-muted-foreground">
|
||||
<span className="ml-auto shrink-0 text-[10px] text-muted-foreground">
|
||||
{source}
|
||||
</span>
|
||||
</ComboboxItem>
|
||||
|
|
@ -446,11 +446,11 @@ export function ModelSection() {
|
|||
<ComboboxItem
|
||||
key={id}
|
||||
value={id}
|
||||
className={`justify-between ${exceeds ? "opacity-50" : ""}`}
|
||||
className={`gap-2 ${exceeds ? "opacity-50" : ""}`}
|
||||
>
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild={true}>
|
||||
<span className={`min-w-0 flex-1 truncate ${exceeds ? "line-through decoration-muted-foreground/50" : ""}`}>
|
||||
<span className={`block min-w-0 flex-1 truncate ${exceeds ? "line-through decoration-muted-foreground/50" : ""}`}>
|
||||
{id}
|
||||
</span>
|
||||
</TooltipTrigger>
|
||||
|
|
@ -470,7 +470,7 @@ export function ModelSection() {
|
|||
)}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
<span className="flex items-center gap-1.5 shrink-0">
|
||||
<span className="ml-auto flex items-center gap-1.5 shrink-0">
|
||||
{fitStatus === "exceeds" && (
|
||||
<span className="text-[9px] font-medium text-red-400">
|
||||
OOM
|
||||
|
|
|
|||
|
|
@ -133,8 +133,8 @@ export function useHfModelSearch(
|
|||
...(accessToken ? { credentials: { accessToken } } : {}),
|
||||
}) as AsyncGenerator<unknown>;
|
||||
}
|
||||
// Dual-query: unsloth first, then general
|
||||
return mergedModelIterator(trimmed, task, accessToken) as AsyncGenerator<unknown>;
|
||||
// Typed query: disable task filter so explicitly searched models still appear even if HF task metadata is wrong/missing.
|
||||
return mergedModelIterator(trimmed, undefined, accessToken) as AsyncGenerator<unknown>;
|
||||
},
|
||||
[query, task, accessToken],
|
||||
);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue