GGUF chat only for CPU

studio/backend/core/inference/defaults.py

@@ -3,8 +3,6 @@
 
 """Default model lists for inference, split by platform."""
 
-import sys
-
 DEFAULT_MODELS_GGUF = [
     "unsloth/Llama-3.2-1B-Instruct-GGUF",
     "unsloth/Llama-3.2-3B-Instruct-GGUF",
@@ -25,6 +23,7 @@ DEFAULT_MODELS_STANDARD = [
 
 
 def get_default_models() -> list[str]:
-    if sys.platform == "darwin":
+    import utils.hardware.hardware as hw
+    if hw.CHAT_ONLY:
         return list(DEFAULT_MODELS_GGUF)
     return list(DEFAULT_MODELS_STANDARD)

studio/backend/main.py

@@ -148,11 +148,14 @@ async def health_check():
     platform_map = {"darwin": "mac", "win32": "windows", "linux": "linux"}
     device_type = platform_map.get(sys.platform, sys.platform)
 
+    chat_only = _hw_module.CHAT_ONLY
+
     return {
         "status": "healthy",
         "timestamp": datetime.now().isoformat(),
         "service": "Unsloth UI Backend",
         "device_type": device_type,
+        "chat_only": chat_only,
     }
 
 

studio/backend/utils/hardware/__init__.py

@@ -8,6 +8,7 @@ Hardware detection and GPU utilities
 from .hardware import (
     DeviceType,
     DEVICE,
+    CHAT_ONLY,
     detect_hardware,
     get_device,
     is_apple_silicon,
@@ -24,6 +25,7 @@ from .hardware import (
 __all__ = [
     "DeviceType",
     "DEVICE",
+    "CHAT_ONLY",
     "detect_hardware",
     "get_device",
     "is_apple_silicon",

studio/backend/utils/hardware/hardware.py

@@ -39,6 +39,7 @@ class DeviceType(str, Enum):
 # ========== Global State (set once by detect_hardware) ==========
 
 DEVICE: Optional[DeviceType] = None
+CHAT_ONLY: bool = True  # No CUDA GPU → GGUF chat only (Mac, CPU-only, etc.)
 
 
 # ========== Detection ==========
@@ -81,7 +82,7 @@ def detect_hardware() -> DeviceType:
       2. MLX   (Apple Silicon via MLX framework)
       3. CPU   (fallback)
     """
-    global DEVICE
+    global DEVICE, CHAT_ONLY
 
     # --- CUDA: try PyTorch ---
     if _has_torch():
@@ -89,6 +90,7 @@ def detect_hardware() -> DeviceType:
 
         if torch.cuda.is_available():
             DEVICE = DeviceType.CUDA
+            CHAT_ONLY = False
             device_name = torch.cuda.get_device_properties(0).name
             print(f"Hardware detected: CUDA — {device_name}")
             return DEVICE

studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx

@@ -396,9 +396,12 @@ export function HubModelPicker({
     return s;
   }, [cachedGguf, cachedModels]);
 
+  const chatOnly = usePlatformStore((s) => s.isChatOnly());
+
   const recommendedIds = useMemo(() => {
     const all = dedupe([...models.map((model) => model.id), value ?? ""])
-      .filter((id) => !downloadedSet.has(id.toLowerCase()));
+      .filter((id) => !downloadedSet.has(id.toLowerCase()))
+      .filter((id) => !chatOnly || isGgufRepo(id));
     // Cap at 4 GGUFs + 4 non-GGUFs so the list stays manageable
     const gguf: string[] = [];
     const hub: string[] = [];
@@ -407,7 +410,7 @@ export function HubModelPicker({
       else if (!isGgufRepo(id) && hub.length < 4) hub.push(id);
     }
     return [...gguf, ...hub];
-  }, [models, value, downloadedSet]);
+  }, [models, value, downloadedSet, chatOnly]);
 
   const { paramCountById: recommendedParamCountById } =
     useRecommendedModelVram(recommendedIds);
@@ -415,8 +418,6 @@ export function HubModelPicker({
   const showHfSection = debouncedQuery.trim().length > 0;
   const recommendedSet = useMemo(() => new Set(recommendedIds), [recommendedIds]);
 
-  const chatOnly = usePlatformStore((s) => s.isChatOnly());
-
   const hfIds = useMemo(() => {
     if (!showHfSection) return [];
     return results
@@ -519,7 +520,7 @@ export function HubModelPicker({
               <Spinner className="size-3 text-muted-foreground" />
               <span className="text-xs text-muted-foreground">Loading models…</span>
             </div>
-          ) : !showHfSection && (cachedGguf.length > 0 || cachedModels.length > 0) ? (
+          ) : !showHfSection && (cachedGguf.length > 0 || (!chatOnly && cachedModels.length > 0)) ? (
             <>
               <ListLabel>{"\uD83E\uDDA5"} Downloaded</ListLabel>
               {cachedGguf.map((c) => (
@@ -536,7 +537,7 @@ export function HubModelPicker({
                   )}
                 </div>
               ))}
-              {cachedModels.map((c) => (
+              {!chatOnly && cachedModels.map((c) => (
                 <ModelRow
                   key={c.repo_id}
                   label={c.repo_id}

studio/frontend/src/config/env.ts

@@ -16,14 +16,16 @@ export type DeviceType = "mac" | "windows" | "linux" | string;
 
 interface PlatformState {
   deviceType: DeviceType;
+  chatOnly: boolean;
   fetched: boolean;
   isChatOnly: () => boolean;
 }
 
 export const usePlatformStore = create<PlatformState>()((_, get) => ({
   deviceType: "linux",
+  chatOnly: false,
   fetched: false,
-  isChatOnly: () => get().deviceType === "mac",
+  isChatOnly: () => get().chatOnly,
 }));
 
 export async function fetchDeviceType(): Promise<DeviceType> {
@@ -33,9 +35,10 @@ export async function fetchDeviceType(): Promise<DeviceType> {
   try {
     const res = await fetch("/api/health");
     if (res.ok) {
-      const data = (await res.json()) as { device_type?: string };
+      const data = (await res.json()) as { device_type?: string; chat_only?: boolean };
       const deviceType = data.device_type ?? "linux";
-      usePlatformStore.setState({ deviceType, fetched: true });
+      const chatOnly = data.chat_only ?? deviceType === "mac";
+      usePlatformStore.setState({ deviceType, chatOnly, fetched: true });
       return deviceType;
     }
   } catch (err) {