diff --git a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx index b24536d2f7..bf60f9c51a 100644 --- a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx +++ b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx @@ -191,11 +191,32 @@ function GgufVariantExpander({ [repoId, onSelect], ); + // If the backend-recommended variant is OOM, pick the largest fitting + // variant instead; if all are OOM, recommend the smallest one. + const effectiveRecommended = useMemo(() => { + if (!variants || !gpuGb || gpuGb <= 0) return defaultVariant; + const isOom = (v: GgufVariantDetail) => { + const gb = v.size_bytes / (1024 ** 3); + return gb > 0 && checkVramFit(gb, gpuGb) === "exceeds"; + }; + const defaultV = variants.find((v) => v.quant === defaultVariant); + if (defaultV && !isOom(defaultV)) return defaultVariant; + // Default is OOM -- pick largest non-OOM variant (best quality that fits) + const fitting = variants.filter((v) => !isOom(v)); + if (fitting.length > 0) { + fitting.sort((a, b) => b.size_bytes - a.size_bytes); + return fitting[0].quant; + } + // All OOM -- recommend smallest (most likely to run with --fit) + const sorted = [...variants].sort((a, b) => a.size_bytes - b.size_bytes); + return sorted[0].quant; + }, [variants, defaultVariant, gpuGb]); + const sortedVariants = useMemo(() => { if (!variants) return variants; return [...variants].sort((a, b) => { - const aIsRec = a.quant === defaultVariant; - const bIsRec = b.quant === defaultVariant; + const aIsRec = a.quant === effectiveRecommended; + const bIsRec = b.quant === effectiveRecommended; if (aIsRec !== bIsRec) return aIsRec ? -1 : 1; const aGb = a.size_bytes / (1024 ** 3); @@ -206,7 +227,7 @@ function GgufVariantExpander({ return b.size_bytes - a.size_bytes; }); - }, [variants, defaultVariant, gpuGb]); + }, [variants, effectiveRecommended, gpuGb]); if (loading) { return ( @@ -257,7 +278,7 @@ function GgufVariantExpander({ > {v.quant} - {v.quant === defaultVariant && ( + {v.quant === effectiveRecommended && ( recommended diff --git a/studio/frontend/src/hooks/use-gpu-info.ts b/studio/frontend/src/hooks/use-gpu-info.ts index 5c913eee65..8cb614b50f 100644 --- a/studio/frontend/src/hooks/use-gpu-info.ts +++ b/studio/frontend/src/hooks/use-gpu-info.ts @@ -30,11 +30,12 @@ async function fetchGpuOnce(): Promise { const data = await res.json(); const gpuData = data?.gpu; if (!gpuData?.available || !gpuData.devices?.length) return DEFAULT_GPU; - const dev = gpuData.devices[0]; + const devices = gpuData.devices as Array<{ name?: string; memory_total_gb?: number }>; + const totalGb = devices.reduce((sum, d) => sum + (d.memory_total_gb ?? 0), 0); const info: GpuInfo = { available: true, - name: dev.name ?? "Unknown", - memoryTotalGb: dev.memory_total_gb ?? 0, + name: devices[0]?.name ?? "Unknown", + memoryTotalGb: totalGb, }; cachedGpu = info; return info;