GGUF chat only for CPU

studio/backend/core/inference/defaults.py

@@ -3,8 +3,6 @@
 
 """Default model lists for inference, split by platform."""
 
-import sys
-
 DEFAULT_MODELS_GGUF = [
     "unsloth/Llama-3.2-1B-Instruct-GGUF",
     "unsloth/Llama-3.2-3B-Instruct-GGUF",
@@ -25,6 +23,7 @@ DEFAULT_MODELS_STANDARD = [
 
 
 def get_default_models() -> list[str]:
-    if sys.platform == "darwin":
+    import utils.hardware.hardware as hw
+    if hw.CHAT_ONLY:
         return list(DEFAULT_MODELS_GGUF)
     return list(DEFAULT_MODELS_STANDARD)

studio/backend/main.py

@@ -148,11 +148,14 @@ async def health_check():
     platform_map = {"darwin": "mac", "win32": "windows", "linux": "linux"}
     device_type = platform_map.get(sys.platform, sys.platform)
 
+    chat_only = _hw_module.CHAT_ONLY
+
     return {
         "status": "healthy",
         "timestamp": datetime.now().isoformat(),
         "service": "Unsloth UI Backend",
         "device_type": device_type,
+        "chat_only": chat_only,
     }
 
 

studio/backend/utils/hardware/__init__.py

@@ -8,6 +8,7 @@ Hardware detection and GPU utilities
 from .hardware import (
     DeviceType,
     DEVICE,
+    CHAT_ONLY,
     detect_hardware,
     get_device,
     is_apple_silicon,
@@ -24,6 +25,7 @@ from .hardware import (
 __all__ = [
     "DeviceType",
     "DEVICE",
+    "CHAT_ONLY",
     "detect_hardware",
     "get_device",
     "is_apple_silicon",

studio/backend/utils/hardware/hardware.py

@@ -39,6 +39,7 @@ class DeviceType(str, Enum):
 # ========== Global State (set once by detect_hardware) ==========
 
 DEVICE: Optional[DeviceType] = None
+CHAT_ONLY: bool = True  # No CUDA GPU → GGUF chat only (Mac, CPU-only, etc.)
 
 
 # ========== Detection ==========
@@ -81,7 +82,7 @@ def detect_hardware() -> DeviceType:
       2. MLX   (Apple Silicon via MLX framework)
       3. CPU   (fallback)
     """
-    global DEVICE
+    global DEVICE, CHAT_ONLY
 
     # --- CUDA: try PyTorch ---
     if _has_torch():
@@ -89,6 +90,7 @@ def detect_hardware() -> DeviceType:
 
         if torch.cuda.is_available():
             DEVICE = DeviceType.CUDA
+            CHAT_ONLY = False
             device_name = torch.cuda.get_device_properties(0).name
             print(f"Hardware detected: CUDA — {device_name}")
             return DEVICE