ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

GGUF chat only for CPU

Browse source
This commit is contained in:
Manan17 2026-03-16 21:14:53 +00:00
commit c2cd02dc8f
6 changed files with 23 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

13
studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx
View file

@ -396,9 +396,12 @@ export function HubModelPicker({
return s;
}, [cachedGguf, cachedModels]);
const chatOnly = usePlatformStore((s) => s.isChatOnly());
const recommendedIds = useMemo(() => {
const all = dedupe([...models.map((model) => model.id), value ?? ""])
.filter((id) => !downloadedSet.has(id.toLowerCase()));
.filter((id) => !downloadedSet.has(id.toLowerCase()))
.filter((id) => !chatOnly || isGgufRepo(id));
// Cap at 4 GGUFs + 4 non-GGUFs so the list stays manageable
const gguf: string[] = [];
const hub: string[] = [];
@ -407,7 +410,7 @@ export function HubModelPicker({
else if (!isGgufRepo(id) && hub.length < 4) hub.push(id);
}
return [...gguf, ...hub];
}, [models, value, downloadedSet]);
}, [models, value, downloadedSet, chatOnly]);
const { paramCountById: recommendedParamCountById } =
useRecommendedModelVram(recommendedIds);
@ -415,8 +418,6 @@ export function HubModelPicker({
const showHfSection = debouncedQuery.trim().length > 0;
const recommendedSet = useMemo(() => new Set(recommendedIds), [recommendedIds]);
const chatOnly = usePlatformStore((s) => s.isChatOnly());
const hfIds = useMemo(() => {
if (!showHfSection) return [];
return results
@ -519,7 +520,7 @@ export function HubModelPicker({
<Spinner className="size-3 text-muted-foreground" />
<span className="text-xs text-muted-foreground">Loading models…</span>
</div>
) : !showHfSection && (cachedGguf.length > 0 || cachedModels.length > 0) ? (
) : !showHfSection && (cachedGguf.length > 0 || (!chatOnly && cachedModels.length > 0)) ? (
<>
<ListLabel>{"\uD83E\uDDA5"} Downloaded</ListLabel>
{cachedGguf.map((c) => (
@ -536,7 +537,7 @@ export function HubModelPicker({
)}
</div>
))}
{cachedModels.map((c) => (
{!chatOnly && cachedModels.map((c) => (
<ModelRow
key={c.repo_id}
label={c.repo_id}

9
studio/frontend/src/config/env.ts
View file

@ -16,14 +16,16 @@ export type DeviceType = "mac" | "windows" | "linux" | string;
interface PlatformState {
deviceType: DeviceType;
chatOnly: boolean;
fetched: boolean;
isChatOnly: () => boolean;
}
export const usePlatformStore = create<PlatformState>()((_, get) => ({
deviceType: "linux",
chatOnly: false,
fetched: false,
isChatOnly: () => get().deviceType === "mac",
isChatOnly: () => get().chatOnly,
}));
export async function fetchDeviceType(): Promise<DeviceType> {
@ -33,9 +35,10 @@ export async function fetchDeviceType(): Promise<DeviceType> {
try {
const res = await fetch("/api/health");
if (res.ok) {
const data = (await res.json()) as { device_type?: string };
const data = (await res.json()) as { device_type?: string; chat_only?: boolean };
const deviceType = data.device_type ?? "linux";
usePlatformStore.setState({ deviceType, fetched: true });
const chatOnly = data.chat_only ?? deviceType === "mac";
usePlatformStore.setState({ deviceType, chatOnly, fetched: true });
return deviceType;
}
} catch (err) {