diff --git a/studio/backend/core/inference/defaults.py b/studio/backend/core/inference/defaults.py index c8e23deb09..d5e9ca2e97 100644 --- a/studio/backend/core/inference/defaults.py +++ b/studio/backend/core/inference/defaults.py @@ -3,7 +3,7 @@ """Default model lists for inference, split by platform.""" -import sys +import utils.hardware.hardware as hw DEFAULT_MODELS_GGUF = [ "unsloth/Llama-3.2-1B-Instruct-GGUF", @@ -25,6 +25,7 @@ DEFAULT_MODELS_STANDARD = [ def get_default_models() -> list[str]: - if sys.platform == "darwin": + hw.get_device() # ensure detect_hardware() has run + if hw.CHAT_ONLY: return list(DEFAULT_MODELS_GGUF) return list(DEFAULT_MODELS_STANDARD) diff --git a/studio/backend/main.py b/studio/backend/main.py index 3ab846306e..5f580ff269 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -148,11 +148,14 @@ async def health_check(): platform_map = {"darwin": "mac", "win32": "windows", "linux": "linux"} device_type = platform_map.get(sys.platform, sys.platform) + chat_only = _hw_module.CHAT_ONLY + return { "status": "healthy", "timestamp": datetime.now().isoformat(), "service": "Unsloth UI Backend", "device_type": device_type, + "chat_only": chat_only, } diff --git a/studio/backend/utils/hardware/__init__.py b/studio/backend/utils/hardware/__init__.py index c1f8b62010..cfd780568e 100644 --- a/studio/backend/utils/hardware/__init__.py +++ b/studio/backend/utils/hardware/__init__.py @@ -8,6 +8,7 @@ Hardware detection and GPU utilities from .hardware import ( DeviceType, DEVICE, + CHAT_ONLY, detect_hardware, get_device, is_apple_silicon, @@ -24,6 +25,7 @@ from .hardware import ( __all__ = [ "DeviceType", "DEVICE", + "CHAT_ONLY", "detect_hardware", "get_device", "is_apple_silicon", diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py index c743dcd897..8f0545893a 100644 --- a/studio/backend/utils/hardware/hardware.py +++ b/studio/backend/utils/hardware/hardware.py @@ -39,6 +39,7 @@ class DeviceType(str, Enum): # ========== Global State (set once by detect_hardware) ========== DEVICE: Optional[DeviceType] = None +CHAT_ONLY: bool = True # No CUDA GPU → GGUF chat only (Mac, CPU-only, etc.) # ========== Detection ========== @@ -81,7 +82,8 @@ def detect_hardware() -> DeviceType: 2. MLX (Apple Silicon via MLX framework) 3. CPU (fallback) """ - global DEVICE + global DEVICE, CHAT_ONLY + CHAT_ONLY = True # reset — only CUDA sets it to False # --- CUDA: try PyTorch --- if _has_torch(): @@ -89,6 +91,7 @@ def detect_hardware() -> DeviceType: if torch.cuda.is_available(): DEVICE = DeviceType.CUDA + CHAT_ONLY = False device_name = torch.cuda.get_device_properties(0).name print(f"Hardware detected: CUDA — {device_name}") return DEVICE diff --git a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx index 2284cd194f..34f0f9b533 100644 --- a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx +++ b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx @@ -396,9 +396,12 @@ export function HubModelPicker({ return s; }, [cachedGguf, cachedModels]); + const chatOnly = usePlatformStore((s) => s.isChatOnly()); + const recommendedIds = useMemo(() => { const all = dedupe([...models.map((model) => model.id), value ?? ""]) - .filter((id) => !downloadedSet.has(id.toLowerCase())); + .filter((id) => !downloadedSet.has(id.toLowerCase())) + .filter((id) => !chatOnly || isGgufRepo(id)); // Cap at 4 GGUFs + 4 non-GGUFs so the list stays manageable const gguf: string[] = []; const hub: string[] = []; @@ -407,7 +410,7 @@ export function HubModelPicker({ else if (!isGgufRepo(id) && hub.length < 4) hub.push(id); } return [...gguf, ...hub]; - }, [models, value, downloadedSet]); + }, [models, value, downloadedSet, chatOnly]); const { paramCountById: recommendedParamCountById } = useRecommendedModelVram(recommendedIds); @@ -415,8 +418,6 @@ export function HubModelPicker({ const showHfSection = debouncedQuery.trim().length > 0; const recommendedSet = useMemo(() => new Set(recommendedIds), [recommendedIds]); - const chatOnly = usePlatformStore((s) => s.isChatOnly()); - const hfIds = useMemo(() => { if (!showHfSection) return []; return results @@ -519,7 +520,7 @@ export function HubModelPicker({ Loading models… - ) : !showHfSection && (cachedGguf.length > 0 || cachedModels.length > 0) ? ( + ) : !showHfSection && (cachedGguf.length > 0 || (!chatOnly && cachedModels.length > 0)) ? ( <> {"\uD83E\uDDA5"} Downloaded {cachedGguf.map((c) => ( @@ -536,7 +537,7 @@ export function HubModelPicker({ )} ))} - {cachedModels.map((c) => ( + {!chatOnly && cachedModels.map((c) => ( boolean; } export const usePlatformStore = create()((_, get) => ({ deviceType: "linux", + chatOnly: false, fetched: false, - isChatOnly: () => get().deviceType === "mac", + isChatOnly: () => get().chatOnly, })); export async function fetchDeviceType(): Promise { @@ -33,9 +35,10 @@ export async function fetchDeviceType(): Promise { try { const res = await fetch("/api/health"); if (res.ok) { - const data = (await res.json()) as { device_type?: string }; + const data = (await res.json()) as { device_type?: string; chat_only?: boolean }; const deviceType = data.device_type ?? "linux"; - usePlatformStore.setState({ deviceType, fetched: true }); + const chatOnly = data.chat_only ?? deviceType === "mac"; + usePlatformStore.setState({ deviceType, chatOnly, fetched: true }); return deviceType; } } catch (err) {