From 217e8f036c902b8e58caee88a70f872cb28f6a32 Mon Sep 17 00:00:00 2001 From: alkinun Date: Mon, 27 Jul 2026 09:28:39 +0300 Subject: [PATCH] fix(studio): report Vulkan GPUs in system UI (#7476) * fix(studio): report Vulkan GPUs in system UI * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix(studio): separate Vulkan inference GPU reporting * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix(studio): keep retrying Vulkan probe refreshes * fix(studio): preserve known zero GPU budgets --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- README.md | 18 +- studio/backend/main.py | 69 +++++-- studio/backend/tests/test_gpu_selection.py | 103 +++++++++++ .../tests/test_system_vulkan_gpu_info.py | 171 ++++++++++++++++++ studio/backend/utils/hardware/__init__.py | 2 + studio/backend/utils/hardware/hardware.py | 59 +++++- .../src/components/floating-monitor.tsx | 39 +++- studio/frontend/src/features/hub/hub-page.tsx | 24 ++- .../components/model-selector/pickers.tsx | 115 ++++++++---- .../model-selector/recommended-fit.ts | 26 ++- .../features/settings/tabs/resources-tab.tsx | 44 ++++- studio/frontend/src/hooks/index.ts | 2 +- studio/frontend/src/hooks/use-gpu-info.ts | 78 ++++++-- studio/frontend/src/hooks/use-system.ts | 41 +++-- 14 files changed, 692 insertions(+), 99 deletions(-) create mode 100644 studio/backend/tests/test_system_vulkan_gpu_info.py diff --git a/README.md b/README.md index 514454f985..1facb87c11 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ Unsloth Studio (Beta) works on **Windows, Linux, WSL** and **macOS**. * **NVIDIA:** Training works on RTX 30/40/50, Blackwell, DGX Spark, Station and more * **macOS:** Training, MLX and GGUF inference are ALL supported. * **AMD:** Training, RL, chat and deployment work on Windows, WSL and Linux. [Read the AMD guide](https://unsloth.ai/docs/basics/amd). -* **Vulkan:** GGUF inference is supported on [compatible GPUs, including Intel GPUs](https://github.com/unslothai/unsloth/pull/5819). +* **Vulkan:** GGUF inference is supported on [compatible GPUs, including Intel GPUs](https://github.com/unslothai/unsloth/pull/5819). Vulkan accelerates GGUF inference only; training still requires a supported PyTorch or MLX backend. * **Multi-GPU:** Available now, with a major upgrade on the way #### macOS, Linux, WSL: @@ -112,12 +112,28 @@ curl -fsSL https://unsloth.ai/install.sh | sh ``` Use the same command to update. +To force the Vulkan llama.cpp backend, set `UNSLOTH_FORCE_VULKAN=1` **before installing or updating**. The setting selects the llama.cpp binary bundle, so setting it only when launching Studio cannot replace an existing CPU bundle: + +```bash +export UNSLOTH_FORCE_VULKAN=1 +curl -fsSL https://unsloth.ai/install.sh | sh +``` + #### Windows: ```powershell irm https://unsloth.ai/install.ps1 | iex ``` Use the same command to update. +To force the Vulkan llama.cpp backend, set the environment variable before running the installer or updater: + +```powershell +$env:UNSLOTH_FORCE_VULKAN=1 +irm https://unsloth.ai/install.ps1 | iex +``` + +Re-running the current installer replaces a previously selected CPU bundle when the backend differs. A separate Vulkan SDK is not required; the GPU driver must provide a working Vulkan runtime. + #### Launch ```bash unsloth studio -p 8888 diff --git a/studio/backend/main.py b/studio/backend/main.py index 5af25efa74..1f793341ea 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -40,7 +40,7 @@ if sys.platform == "win32": _SYSTEM_GPU_CACHE_TTL_SECONDS = 10.0 _system_gpu_cache_lock = threading.Lock() -_system_gpu_cache: Optional[tuple[float, dict[str, Any]]] = None +_system_gpu_cache: Optional[tuple[float, tuple[dict[str, Any], dict[str, Any]]]] = None # ── Windows AMD ROCm DLL injection ────────────────────────────────────────── # Python 3.8+ ignores PATH for extension modules; register ROCm bin dirs with @@ -1149,10 +1149,14 @@ async def shutdown_server(request: Request, current_subject: str = Depends(get_c return {"status": "shutting_down"} -def _get_cached_system_gpu_info(logger) -> dict[str, Any]: - """Return merged GPU visibility/utilization with bounded live-probe churn.""" +def _get_cached_system_gpu_info(logger) -> tuple[dict[str, Any], dict[str, Any]]: + """Return training and inference GPU info with bounded live-probe churn.""" import time - from utils.hardware import get_backend_visible_gpu_info, get_visible_gpu_utilization + from utils.hardware import ( + get_backend_visible_gpu_info, + get_visible_gpu_utilization, + get_vulkan_inference_gpu_info, + ) global _system_gpu_cache now = time.monotonic() @@ -1174,7 +1178,20 @@ def _get_cached_system_gpu_info(logger) -> dict[str, Any]: logger.debug(f"Failed to get GPU utilization info: {e}") utilization_info = {"devices": []} - util_devices = {d.get("index"): d for d in utilization_info.get("devices", [])} + # Device indices are backend-specific. Never overlay CUDA/ROCm metrics + # onto compact Vulkan ordinals merely because both happen to start at 0. + visibility_backend = visibility_info.get("backend") + utilization_backend = utilization_info.get("backend") + metrics_match = ( + not visibility_backend + or not utilization_backend + or visibility_backend == utilization_backend + ) + util_devices = ( + {d.get("index"): d for d in utilization_info.get("devices", [])} + if metrics_match + else {} + ) enriched_devices = [] for dev in visibility_info.get("devices", []): @@ -1184,14 +1201,19 @@ def _get_cached_system_gpu_info(logger) -> dict[str, Any]: total_vram = util.get("vram_total_gb") or dev.get("memory_total_gb") or 0 # Keep None (usage unknown, e.g. Windows ROCm perf counter) so the UI # shows unknown, not a fabricated 0 used / full free. - used_vram = util.get("vram_used_gb") + used_vram = util.get("vram_used_gb", dev.get("vram_used_gb")) + reported_free_vram = util.get("vram_free_gb", dev.get("vram_free_gb")) enriched_dev = dict(dev) enriched_dev["vram_used_gb"] = used_vram enriched_dev["vram_free_gb"] = ( - round(total_vram - used_vram, 2) if total_vram and used_vram is not None else None + round(total_vram - used_vram, 2) + if total_vram and used_vram is not None + else reported_free_vram + ) + enriched_dev["vram_utilization_pct"] = util.get( + "vram_utilization_pct", dev.get("vram_utilization_pct") ) - enriched_dev["vram_utilization_pct"] = util.get("vram_utilization_pct") enriched_devices.append(enriched_dev) # Whether GGUF loads accept an explicit gpu_ids pick: /load and @@ -1207,13 +1229,37 @@ def _get_cached_system_gpu_info(logger) -> dict[str, Any]: except Exception as e: logger.debug(f"Could not resolve gpu_ids support: {e}") gpu_ids_supported = True + # Preserve backend/index metadata from the visibility probe. In + # particular, a CPU training host can expose a Vulkan inference GPU and + # the UI must label that device as Vulkan rather than falling back to the + # top-level CPU training backend. gpu_info = { + **visibility_info, "available": visibility_info.get("available", False), "devices": enriched_devices, "gguf_gpu_ids_supported": gpu_ids_supported, } - _system_gpu_cache = (time.monotonic(), gpu_info) - return gpu_info + + # Keep inference placement separate on train-capable hosts where a + # forced Vulkan llama.cpp bundle can enumerate a different device set. + # If Vulkan is installed but its probe fails, retain the unavailable + # Vulkan shape instead of budgeting training GPUs that llama.cpp cannot use. + if visibility_info.get("backend") == "vulkan": + inference_gpu_info = gpu_info + else: + vulkan_info = get_vulkan_inference_gpu_info() + inference_gpu_info = ( + { + **vulkan_info, + "gguf_gpu_ids_supported": False, + } + if vulkan_info is not None + else gpu_info + ) + + combined_info = (gpu_info, inference_gpu_info) + _system_gpu_cache = (time.monotonic(), combined_info) + return combined_info @app.get("/api/system") @@ -1234,7 +1280,7 @@ def get_system_info(current_subject: str = Depends(get_current_subject)): logger = logging.getLogger(__name__) - gpu_info = _get_cached_system_gpu_info(logger) + gpu_info, inference_gpu_info = _get_cached_system_gpu_info(logger) memory = psutil.virtual_memory() @@ -1301,6 +1347,7 @@ def get_system_info(current_subject: str = Depends(get_current_subject)): "percent_used": disk.percent if disk else 0, }, "gpu": gpu_info, + "inference_gpu": inference_gpu_info, "ml_packages": ml_packages, # Export capability + torch-aware reason. See /api/system/hardware. **export_capability(), diff --git a/studio/backend/tests/test_gpu_selection.py b/studio/backend/tests/test_gpu_selection.py index 3dab7ef368..362c751baa 100644 --- a/studio/backend/tests/test_gpu_selection.py +++ b/studio/backend/tests/test_gpu_selection.py @@ -28,6 +28,7 @@ from utils.hardware import ( get_offloaded_device_map_entries, get_parent_visible_gpu_ids, get_visible_gpu_utilization, + get_vulkan_inference_gpu_info, prepare_gpu_selection, resolve_requested_gpu_ids, ) @@ -411,6 +412,108 @@ class TestVisibleGpuUtilization(_GpuCacheResetMixin, unittest.TestCase): self.assertEqual(result["devices"][0]["index"], 0) self.assertEqual(result["devices"][0]["visible_ordinal"], 0) + def test_discrete_vulkan_inference_gpu_info(self): + with ( + patch( + "core.inference.llama_cpp.LlamaCppBackend._is_vulkan_backend", + return_value = True, + ), + patch( + "core.inference.llama_cpp.LlamaCppBackend._get_gpu_memory", + return_value = [(0, 7402, 8192)], + ), + ): + result = get_vulkan_inference_gpu_info() + + self.assertTrue(result["available"]) + self.assertEqual(result["backend"], "vulkan") + self.assertEqual(result["index_kind"], "relative") + self.assertEqual(result["parent_visible_gpu_ids"], []) + self.assertEqual( + result["devices"], + [ + { + "index": 0, + "index_kind": "relative", + "visible_ordinal": 0, + "name": "Vulkan0", + "memory_total_gb": 8.0, + "vram_used_gb": 0.77, + "vram_free_gb": 7.23, + "vram_utilization_pct": 9.6, + "shared_memory": False, + } + ], + ) + + def test_vulkan_igpu_info_uses_capped_free_budget(self): + with ( + patch( + "core.inference.llama_cpp.LlamaCppBackend._is_vulkan_backend", + return_value = True, + ), + patch( + "core.inference.llama_cpp.LlamaCppBackend._get_gpu_memory", + return_value = [(0, 12288, 0)], + ), + ): + result = get_vulkan_inference_gpu_info() + + device = result["devices"][0] + self.assertEqual(device["memory_total_gb"], 12.0) + self.assertEqual(device["vram_free_gb"], 12.0) + self.assertIsNone(device["vram_used_gb"]) + self.assertIsNone(device["vram_utilization_pct"]) + self.assertTrue(device["shared_memory"]) + + def test_forced_vulkan_overrides_torch_gpu_visibility_for_inference(self): + with ( + patch("utils.hardware.hardware.get_device", return_value = DeviceType.CUDA), + patch( + "core.inference.llama_cpp.LlamaCppBackend._is_vulkan_backend", + return_value = True, + ), + patch( + "core.inference.llama_cpp.LlamaCppBackend._get_gpu_memory", + return_value = [(1, 6144, 8192)], + ), + patch( + "utils.hardware.nvidia.get_backend_visible_gpu_info", + return_value = { + "available": True, + "backend": "cuda", + "devices": [{"index": 0, "name": "CUDA0", "memory_total_gb": 24.0}], + }, + ), + patch( + "utils.hardware.hardware._get_parent_visible_gpu_spec", + return_value = {"raw": None, "numeric_ids": None}, + ), + ): + training_result = get_backend_visible_gpu_info() + inference_result = get_vulkan_inference_gpu_info() + + self.assertEqual(training_result["backend"], "cuda") + self.assertEqual(inference_result["backend"], "vulkan") + self.assertEqual(inference_result["devices"][0]["index"], 1) + + def test_vulkan_install_without_devices_reports_unavailable(self): + with ( + patch( + "core.inference.llama_cpp.LlamaCppBackend._is_vulkan_backend", + return_value = True, + ), + patch( + "core.inference.llama_cpp.LlamaCppBackend._get_gpu_memory", + return_value = [], + ), + ): + result = get_vulkan_inference_gpu_info() + + self.assertFalse(result["available"]) + self.assertEqual(result["backend"], "vulkan") + self.assertEqual(result["devices"], []) + class TestGpuAutoSelection(_GpuCacheResetMixin, unittest.TestCase): def test_get_device_map_uses_explicit_gpu_selection(self): diff --git a/studio/backend/tests/test_system_vulkan_gpu_info.py b/studio/backend/tests/test_system_vulkan_gpu_info.py new file mode 100644 index 0000000000..4742b6b4bb --- /dev/null +++ b/studio/backend/tests/test_system_vulkan_gpu_info.py @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +from types import SimpleNamespace + +import main + + +def test_system_gpu_info_preserves_vulkan_visibility_metrics(monkeypatch): + import utils.hardware as hardware + + vulkan_device = { + "index": 0, + "index_kind": "relative", + "visible_ordinal": 0, + "name": "Vulkan0", + "memory_total_gb": 8.0, + "vram_used_gb": 0.77, + "vram_free_gb": 7.23, + "vram_utilization_pct": 9.6, + "shared_memory": False, + } + monkeypatch.setattr( + hardware, + "get_backend_visible_gpu_info", + lambda: { + "available": False, + "backend": "cpu", + "devices": [], + "index_kind": "relative", + }, + ) + monkeypatch.setattr( + hardware, + "get_visible_gpu_utilization", + lambda: {"available": False, "backend": "cpu", "devices": []}, + ) + monkeypatch.setattr( + hardware, + "get_vulkan_inference_gpu_info", + lambda: { + "available": True, + "backend": "vulkan", + "devices": [vulkan_device], + "index_kind": "relative", + }, + ) + + from core.inference.llama_cpp import LlamaCppBackend + + monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda: True)) + monkeypatch.setattr(main, "_system_gpu_cache", None) + + gpu, inference_gpu = main._get_cached_system_gpu_info(SimpleNamespace(debug = lambda *args: None)) + + assert gpu["available"] is False + assert gpu["backend"] == "cpu" + assert gpu["index_kind"] == "relative" + assert gpu["gguf_gpu_ids_supported"] is False + assert gpu["devices"] == [] + assert inference_gpu["backend"] == "vulkan" + assert inference_gpu["devices"] == [vulkan_device] + + +def test_system_gpu_info_keeps_forced_vulkan_separate_from_training_metrics(monkeypatch): + import utils.hardware as hardware + + monkeypatch.setattr( + hardware, + "get_backend_visible_gpu_info", + lambda: { + "available": True, + "backend": "cuda", + "devices": [{"index": 0, "name": "CUDA0", "memory_total_gb": 24.0}], + }, + ) + monkeypatch.setattr( + hardware, + "get_visible_gpu_utilization", + lambda: { + "available": True, + "backend": "cuda", + "devices": [ + { + "index": 0, + "vram_total_gb": 24.0, + "vram_used_gb": 6.0, + "vram_utilization_pct": 25.0, + } + ], + }, + ) + monkeypatch.setattr( + hardware, + "get_vulkan_inference_gpu_info", + lambda: { + "available": True, + "backend": "vulkan", + "devices": [ + { + "index": 0, + "name": "Vulkan0", + "memory_total_gb": 8.0, + "vram_used_gb": 1.0, + "vram_free_gb": 7.0, + "vram_utilization_pct": 12.5, + "shared_memory": False, + } + ], + "index_kind": "relative", + }, + ) + + from core.inference.llama_cpp import LlamaCppBackend + from utils.hardware import DeviceType + + monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda: True)) + monkeypatch.setattr(hardware, "get_device", lambda: DeviceType.CUDA) + monkeypatch.setattr(main, "_system_gpu_cache", None) + + gpu, inference_gpu = main._get_cached_system_gpu_info(SimpleNamespace(debug = lambda *args: None)) + + assert gpu["backend"] == "cuda" + assert gpu["devices"][0]["vram_used_gb"] == 6.0 + assert inference_gpu["backend"] == "vulkan" + assert inference_gpu["devices"][0]["vram_used_gb"] == 1.0 + assert inference_gpu["gguf_gpu_ids_supported"] is False + + +def test_system_gpu_info_does_not_merge_metrics_across_backend_index_spaces(monkeypatch): + import utils.hardware as hardware + + vulkan_device = { + "index": 0, + "name": "Vulkan0", + "memory_total_gb": 8.0, + "vram_used_gb": 1.0, + "vram_free_gb": 7.0, + "vram_utilization_pct": 12.5, + } + monkeypatch.setattr( + hardware, + "get_backend_visible_gpu_info", + lambda: {"available": True, "backend": "vulkan", "devices": [vulkan_device]}, + ) + monkeypatch.setattr( + hardware, + "get_visible_gpu_utilization", + lambda: { + "available": True, + "backend": "cuda", + "devices": [ + { + "index": 0, + "vram_total_gb": 24.0, + "vram_used_gb": 20.0, + "vram_utilization_pct": 83.3, + } + ], + }, + ) + + from core.inference.llama_cpp import LlamaCppBackend + + monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda: True)) + monkeypatch.setattr(main, "_system_gpu_cache", None) + + gpu, inference_gpu = main._get_cached_system_gpu_info(SimpleNamespace(debug = lambda *args: None)) + + assert gpu["devices"] == [vulkan_device] + assert inference_gpu == gpu diff --git a/studio/backend/utils/hardware/__init__.py b/studio/backend/utils/hardware/__init__.py index 138238533f..72e768a799 100644 --- a/studio/backend/utils/hardware/__init__.py +++ b/studio/backend/utils/hardware/__init__.py @@ -19,6 +19,7 @@ from .hardware import ( get_gpu_utilization, get_visible_gpu_utilization, get_backend_visible_gpu_info, + get_vulkan_inference_gpu_info, get_physical_gpu_count, get_visible_gpu_count, get_parent_visible_gpu_ids, @@ -72,6 +73,7 @@ __all__ = [ "get_gpu_utilization", "get_visible_gpu_utilization", "get_backend_visible_gpu_info", + "get_vulkan_inference_gpu_info", "get_physical_gpu_count", "get_visible_gpu_count", "get_parent_visible_gpu_ids", diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py index 38ebc0b6d4..5c9af51581 100644 --- a/studio/backend/utils/hardware/hardware.py +++ b/studio/backend/utils/hardware/hardware.py @@ -296,7 +296,7 @@ def detect_hardware() -> DeviceType: CHAT_ONLY_REASON = "intel_mac" # Intel Mac: no PyTorch/MLX -> GGUF-only by design. else: CHAT_ONLY_REASON = "no_gpu" - print("Hardware detected: CPU (no GPU backend available)") + print("Hardware detected: CPU training backend (no PyTorch/MLX GPU backend available)") return DEVICE @@ -2575,8 +2575,65 @@ def _backend_visible_devices_env() -> Optional[str]: return os.environ.get("CUDA_VISIBLE_DEVICES") +def get_vulkan_inference_gpu_info() -> Optional[Dict[str, Any]]: + """Return llama.cpp Vulkan devices, or None when Vulkan is not installed.""" + # Vulkan is a llama.cpp inference backend, not a PyTorch training device, so + # keep it separate from the PyTorch/MLX training-device report. + try: + from core.inference.llama_cpp import LlamaCppBackend + except Exception as e: + logger.debug("Could not inspect the llama.cpp Vulkan backend: %s", e) + return None + + try: + if not LlamaCppBackend._is_vulkan_backend(): + return None + except Exception as e: + logger.debug("Could not identify the llama.cpp Vulkan backend: %s", e) + return None + + result = { + "available": False, + "backend": "vulkan", + "backend_cuda_visible_devices": None, + "parent_visible_gpu_ids": [], + "devices": [], + "index_kind": "relative", + } + try: + for ordinal, free_mib, total_mib in LlamaCppBackend._get_gpu_memory(): + # Integrated Vulkan GPUs report total=0 because their memory is + # shared. Publish the capped free value as their usable inference + # budget and mark it so clients do not add system RAM again. + shared_memory = total_mib == 0 + budget_mib = total_mib or free_mib + used_mib = max(0, total_mib - free_mib) if total_mib else None + result["devices"].append( + { + "index": ordinal, + "index_kind": "relative", + "visible_ordinal": ordinal, + "name": f"Vulkan{ordinal}", + "memory_total_gb": round(budget_mib / 1024, 2), + "vram_used_gb": round(used_mib / 1024, 2) if used_mib is not None else None, + "vram_free_gb": round(free_mib / 1024, 2), + "vram_utilization_pct": round((used_mib / total_mib) * 100, 1) + if used_mib is not None and total_mib > 0 + else None, + "shared_memory": shared_memory, + } + ) + except Exception as e: + logger.debug("Vulkan GPU visibility query failed: %s", e) + return result + + result["available"] = bool(result["devices"]) + return result + + def get_backend_visible_gpu_info() -> Dict[str, Any]: device = get_device() + if device in (DeviceType.CUDA, DeviceType.XPU): parent_visible_ids = get_parent_visible_gpu_ids() # Try native SMI first (nvidia-smi; skipped for ROCm). diff --git a/studio/frontend/src/components/floating-monitor.tsx b/studio/frontend/src/components/floating-monitor.tsx index 80501a518c..d86b9ab9af 100644 --- a/studio/frontend/src/components/floating-monitor.tsx +++ b/studio/frontend/src/components/floating-monitor.tsx @@ -4,7 +4,10 @@ import { Button } from "@/components/ui/button"; import { Progress } from "@/components/ui/progress"; import { useMonitorOverlayStore } from "@/features/settings"; -import { useSystemInfo } from "@/hooks/use-system"; +import { + aggregateGpuMemoryTotalGb, + useSystemInfo, +} from "@/hooks/use-system"; import { useT } from "@/i18n"; import { cn } from "@/lib/utils"; import { CpuIcon, GripVerticalIcon, XIcon } from "lucide-react"; @@ -65,11 +68,20 @@ export function FloatingMonitor() { const ramUsed = Math.max(0, ramTotal - ramAvailable); const ramPercent = clampPercent(systemInfo.memory?.percent_used ?? 0); - const devices = systemInfo.gpu?.devices ?? []; - const vramTotal = devices.reduce( - (sum, device) => sum + (device.memory_total_gb ?? 0), - 0, - ); + const displayedGpu = systemInfo.gpu?.available + ? systemInfo.gpu + : (systemInfo.inference_gpu ?? systemInfo.gpu); + const separateInferenceGpu = + systemInfo.gpu?.available && + systemInfo.inference_gpu && + systemInfo.inference_gpu.backend !== systemInfo.gpu.backend + ? systemInfo.inference_gpu + : null; + const inferenceVramTotal = separateInferenceGpu + ? aggregateGpuMemoryTotalGb(separateInferenceGpu.devices) + : 0; + const devices = displayedGpu?.devices ?? []; + const vramTotal = aggregateGpuMemoryTotalGb(devices); // null usage = unknown (e.g. Windows ROCm perf counter): treating it as 0 // fabricates a 0-used readout, so the aggregate is unknown if any device is. const vramUsageKnown = @@ -83,7 +95,7 @@ export function FloatingMonitor() { ); const unknownLabel = t("settings.resources.environment.unknown"); - const hasGpu = (systemInfo.gpu?.available ?? false) && devices.length > 0; + const hasGpu = (displayedGpu?.available ?? false) && devices.length > 0; return ( @@ -188,6 +200,19 @@ export function FloatingMonitor() { /> )} + {separateInferenceGpu && ( +
+ GGUF inference + + {separateInferenceGpu.backend ?? "GPU"} + {separateInferenceGpu.available + ? inferenceVramTotal + ? ` · ${formatGiB(inferenceVramTotal)}` + : "" + : " · unavailable"} + +
+ )} diff --git a/studio/frontend/src/features/hub/hub-page.tsx b/studio/frontend/src/features/hub/hub-page.tsx index 9fc452b4ed..6259f6c8a2 100644 --- a/studio/frontend/src/features/hub/hub-page.tsx +++ b/studio/frontend/src/features/hub/hub-page.tsx @@ -29,7 +29,7 @@ import { resolveInitialConfig, } from "@/features/model-picker"; import { useDebouncedValue } from "@/hooks/use-debounced-value"; -import { useGpuInfo } from "@/hooks/use-gpu-info"; +import { useGpuInfo, useInferenceGpuInfo } from "@/hooks/use-gpu-info"; import { cn } from "@/lib/utils"; import { useNavigate, useSearch } from "@tanstack/react-router"; import { @@ -334,6 +334,7 @@ function selectedRepoMatchesRuntime( export function ModelsPage() { const navigate = useNavigate(); const gpu = useGpuInfo(); + const inferenceGpu = useInferenceGpuInfo(); const online = useOnlineStatus(); const deviceType = usePlatformStore((s) => s.deviceType); const hubSearch = useSearch({ from: "/hub" }); @@ -757,7 +758,10 @@ export function ModelsPage() { // matching the chat model selector. (!fitOnDeviceOnly || row.isAvailableOnDevice || - hfModelFitsDevice(row.result, gpu)), + hfModelFitsDevice( + row.result, + row.result.isGguf ? inferenceGpu : gpu, + )), ); }, [ discoverRows, @@ -769,6 +773,7 @@ export function ModelsPage() { activeChannel, fitOnDeviceOnly, gpu, + inferenceGpu, ]); const listRows = filteredDiscoverRows; @@ -799,7 +804,7 @@ export function ModelsPage() { (row) => !fitOnDeviceOnly || row.isAvailableOnDevice || - hfModelFitsDevice(row.result, gpu), + hfModelFitsDevice(row.result, inferenceGpu), ), [ hubFeed.trending.results, @@ -807,6 +812,7 @@ export function ModelsPage() { modelDiscoveryInventorySignature, fitOnDeviceOnly, gpu, + inferenceGpu, ], ); const feedRows = useMemo(() => { @@ -1254,9 +1260,11 @@ export function ModelsPage() { loadingPhase: loadProgress?.phase, minMemory, vramInfo, - gpuGb: gpu.available ? gpu.memoryTotalGb : undefined, + gpuGb: inferenceGpu.available ? inferenceGpu.memoryTotalGb : undefined, systemRamGb: - gpu.systemRamAvailableGb > 0 ? gpu.systemRamAvailableGb : undefined, + inferenceGpu.systemRamAvailableGb > 0 + ? inferenceGpu.systemRamAvailableGb + : undefined, }), [ isActive, @@ -1265,9 +1273,9 @@ export function ModelsPage() { loadProgress?.phase, minMemory, vramInfo, - gpu.available, - gpu.memoryTotalGb, - gpu.systemRamAvailableGb, + inferenceGpu.available, + inferenceGpu.memoryTotalGb, + inferenceGpu.systemRamAvailableGb, ], ); diff --git a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx index 71b20058f4..ad1586bc41 100644 --- a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx +++ b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx @@ -52,7 +52,7 @@ import { useHfTokenStore, useOnlineStatus, } from "@/features/hub"; -import { useDebouncedValue, useGpuInfo } from "@/hooks"; +import { useDebouncedValue, useGpuInfo, useInferenceGpuInfo } from "@/hooks"; import { extractParamLabel } from "@/lib/model-size"; import { toast } from "@/lib/toast"; import { cn, formatCompact } from "@/lib/utils"; @@ -720,6 +720,7 @@ function GgufVariantExpander({ onSelect, gpuGb, systemRamGb, + budgetKnown = false, hfToken, parentOptionKey, onNavigatePastStart, @@ -735,6 +736,7 @@ function GgufVariantExpander({ onSelect: (id: string, meta: ModelSelectorChangeMeta) => void; gpuGb?: number; systemRamGb?: number; + budgetKnown?: boolean; /** HF token threaded into the variant fetch so private/gated repos resolve * their GGUF variants (and update badges). */ hfToken?: string; @@ -854,8 +856,9 @@ function GgufVariantExpander({ const getGgufFit = useCallback( (sizeBytes: number): "fits" | "tight" | "oom" => { - // No device budget at all: can't classify, so don't show OOM badges. - if (totalBudgetGb <= 0) return "fits"; + // Preserve permissive behavior only when no budget was measured. A known + // zero Vulkan budget means every non-empty variant is OOM. + if (totalBudgetGb <= 0) return budgetKnown ? "oom" : "fits"; const gb = sizeBytes / 1024 ** 3; if (gb <= 0 || gb <= gpuBudgetGb) return "fits"; // No-GPU / unified-memory hosts (Mac) have only the RAM budget, so the tier @@ -864,13 +867,17 @@ function GgufVariantExpander({ if (gb <= totalBudgetGb) return "tight"; return "oom"; }, - [gpuBudgetGb, totalBudgetGb], + [budgetKnown, gpuBudgetGb, totalBudgetGb], ); // If the recommended variant is OOM, pick the largest fitting one; // if all are OOM, recommend the smallest. const effectiveRecommended = useMemo(() => { - if (!variants || variants.length === 0 || totalBudgetGb <= 0) { + if ( + !variants || + variants.length === 0 || + (totalBudgetGb <= 0 && !budgetKnown) + ) { return defaultVariant; } const defaultV = variants.find((v) => v.quant === defaultVariant); @@ -885,7 +892,7 @@ function GgufVariantExpander({ // All OOM -- recommend smallest (most likely to partially run) const sorted = [...variants].sort((a, b) => a.size_bytes - b.size_bytes); return sorted[0]?.quant ?? defaultVariant; - }, [variants, defaultVariant, totalBudgetGb, getGgufFit]); + }, [variants, defaultVariant, totalBudgetGb, budgetKnown, getGgufFit]); const sortedVariants = useMemo(() => { if (!variants) return variants; @@ -1396,6 +1403,7 @@ export function HubModelPicker({ onEject?: () => void; }) { const gpu = useGpuInfo(); + const inferenceGpu = useInferenceGpuInfo(); // Live model id from the runtime store (backend-mirrored active_model), not the dropdown // highlight which can be a staged pick. Disables the update action for it. const loadedModelId = useChatRuntimeStore((s) => s.params.checkpoint); @@ -1854,7 +1862,7 @@ export function HubModelPicker({ return rows.filter((r) => { // Downloaded models always show, regardless of device fit. if (downloadedSet.has(r.id.toLowerCase())) return true; - return hfModelFitsDevice(r, gpu); + return hfModelFitsDevice(r, r.isGguf ? inferenceGpu : gpu); }); }, [ recommendedSearch.results, @@ -1864,6 +1872,7 @@ export function HubModelPicker({ formatFilter, isMac, gpu, + inferenceGpu, isChatSupported, ]); @@ -1904,14 +1913,17 @@ export function HubModelPicker({ r.estimatedSizeBytes ?? (params ? estimateQuantBytes(params) : undefined); const hasDeviceBudget = - gpu.memoryTotalGb > 0 || gpu.systemRamAvailableGb > 0; + inferenceGpu.budgetKnown || + inferenceGpu.memoryTotalGb > 0 || + inferenceGpu.systemRamAvailableGb > 0; const exceeds = hasDeviceBudget && sizeBytes != null && !fitsDevice({ sizeBytes, - gpuGb: gpu.memoryTotalGb, - systemRamGb: gpu.systemRamAvailableGb, + gpuGb: inferenceGpu.memoryTotalGb, + systemRamGb: inferenceGpu.systemRamAvailableGb, + budgetKnown: inferenceGpu.budgetKnown, }); map.set(r.id, { meta, @@ -1928,7 +1940,7 @@ export function HubModelPicker({ map.set(r.id, { meta, status, est }); } return map; - }, [recommendedSearch.results, isKnownGgufRepo, gpu]); + }, [recommendedSearch.results, isKnownGgufRepo, gpu, inferenceGpu]); // Tag-accurate capabilities keyed by repo id, pooled from both HF listings. // Rows look it up by id and fall back to name detection when absent. @@ -2249,7 +2261,7 @@ export function HubModelPicker({ totalParams: recommendedParamCountById.get(id), isGguf: isKnownGgufRepo(id), }, - gpu, + isKnownGgufRepo(id) ? inferenceGpu : gpu, ), ) ); @@ -2263,6 +2275,7 @@ export function HubModelPicker({ downloadedSet, recommendedParamCountById, gpu, + inferenceGpu, ]); const recommendedSet = useMemo( @@ -2280,7 +2293,7 @@ export function HubModelPicker({ (r) => !fitOnDeviceOnly || downloadedSet.has(r.id.toLowerCase()) || - hfModelFitsDevice(r, gpu), + hfModelFitsDevice(r, r.isGguf ? inferenceGpu : gpu), ) .map((result) => result.id) .filter((id) => !isHiddenModelId(id)) @@ -2309,6 +2322,7 @@ export function HubModelPicker({ fitOnDeviceOnly, downloadedSet, gpu, + inferenceGpu, isMac, ]); @@ -2905,8 +2919,9 @@ export function HubModelPicker({ parentOptionKey={optionKey} onNavigatePastStart={() => hubModelList.focusOption(optionKey)} onNavigatePastEnd={() => hubModelList.moveFocus(optionKey, "next")} - gpuGb={gpu.available ? gpu.memoryTotalGb : undefined} - systemRamGb={gpu.systemRamAvailableGb || undefined} + gpuGb={inferenceGpu.available ? inferenceGpu.memoryTotalGb : undefined} + systemRamGb={inferenceGpu.systemRamAvailableGb || undefined} + budgetKnown={inferenceGpu.budgetKnown} variantActions={{ onUpdate: (quant, expectedBytes) => updateGgufVariant(c.repo_id, quant, expectedBytes), @@ -3364,7 +3379,7 @@ export function HubModelPicker({ loraModelList={hubModelList} expandedGguf={expandedGguf} setExpandedGguf={setExpandedGguf} - gpu={gpu} + gpu={inferenceGpu} /> )} @@ -3691,13 +3706,14 @@ export function HubModelPicker({ hubModelList.moveFocus(optionKey, "next") } gpuGb={ - gpu.available - ? gpu.memoryTotalGb + inferenceGpu.available + ? inferenceGpu.memoryTotalGb : undefined } systemRamGb={ - gpu.systemRamAvailableGb || undefined + inferenceGpu.systemRamAvailableGb || undefined } + budgetKnown={inferenceGpu.budgetKnown} /> )} @@ -3816,11 +3832,14 @@ export function HubModelPicker({ hubModelList.moveFocus(optionKey, "next") } gpuGb={ - gpu.available ? gpu.memoryTotalGb : undefined + inferenceGpu.available + ? inferenceGpu.memoryTotalGb + : undefined } systemRamGb={ - gpu.systemRamAvailableGb || undefined + inferenceGpu.systemRamAvailableGb || undefined } + budgetKnown={inferenceGpu.budgetKnown} /> )} @@ -3929,11 +3948,14 @@ export function HubModelPicker({ hubModelList.moveFocus(optionKey, "next") } gpuGb={ - gpu.available ? gpu.memoryTotalGb : undefined + inferenceGpu.available + ? inferenceGpu.memoryTotalGb + : undefined } systemRamGb={ - gpu.systemRamAvailableGb || undefined + inferenceGpu.systemRamAvailableGb || undefined } + budgetKnown={inferenceGpu.budgetKnown} /> )} @@ -3997,7 +4019,13 @@ export function HubModelPicker({ vramStatus={info?.status ?? null} vramEst={info?.est} gpuGb={ - gpu.available ? gpu.memoryTotalGb : undefined + isG + ? inferenceGpu.available + ? inferenceGpu.memoryTotalGb + : undefined + : gpu.available + ? gpu.memoryTotalGb + : undefined } onArrowDownIntoChildren={ expandedGguf === id @@ -4019,11 +4047,14 @@ export function HubModelPicker({ hubModelList.moveFocus(optionKey, "next") } gpuGb={ - gpu.available ? gpu.memoryTotalGb : undefined + inferenceGpu.available + ? inferenceGpu.memoryTotalGb + : undefined } systemRamGb={ - gpu.systemRamAvailableGb || undefined + inferenceGpu.systemRamAvailableGb || undefined } + budgetKnown={inferenceGpu.budgetKnown} variantActions={{ onDelete: async (quant) => { await deleteCachedModel( @@ -4102,7 +4133,13 @@ export function HubModelPicker({ isKnownGgufRepo(id) ? undefined : vram?.est } gpuGb={ - gpu.available ? gpu.memoryTotalGb : undefined + isKnownGgufRepo(id) + ? inferenceGpu.available + ? inferenceGpu.memoryTotalGb + : undefined + : gpu.available + ? gpu.memoryTotalGb + : undefined } onArrowDownIntoChildren={ expandedGguf === id @@ -4128,11 +4165,14 @@ export function HubModelPicker({ hubModelList.moveFocus(optionKey, "next") } gpuGb={ - gpu.available ? gpu.memoryTotalGb : undefined + inferenceGpu.available + ? inferenceGpu.memoryTotalGb + : undefined } systemRamGb={ - gpu.systemRamAvailableGb || undefined + inferenceGpu.systemRamAvailableGb || undefined } + budgetKnown={inferenceGpu.budgetKnown} variantActions={{ onDelete: async (quant) => { await deleteCachedModel( @@ -4207,7 +4247,13 @@ export function HubModelPicker({ } vramEst={isSearchGguf ? undefined : vram?.est} gpuGb={ - gpu.available ? gpu.memoryTotalGb : undefined + isSearchGguf + ? inferenceGpu.available + ? inferenceGpu.memoryTotalGb + : undefined + : gpu.available + ? gpu.memoryTotalGb + : undefined } onArrowDownIntoChildren={ expandedGguf === id @@ -4233,11 +4279,14 @@ export function HubModelPicker({ hubModelList.moveFocus(optionKey, "next") } gpuGb={ - gpu.available ? gpu.memoryTotalGb : undefined + inferenceGpu.available + ? inferenceGpu.memoryTotalGb + : undefined } systemRamGb={ - gpu.systemRamAvailableGb || undefined + inferenceGpu.systemRamAvailableGb || undefined } + budgetKnown={inferenceGpu.budgetKnown} variantActions={{ onDelete: async (quant) => { await deleteCachedModel( @@ -4320,6 +4369,7 @@ function FineTunedRows({ setExpandedGguf: Dispatch>; gpu: { available: boolean; + budgetKnown: boolean; memoryTotalGb: number; systemRamAvailableGb: number; }; @@ -4456,6 +4506,7 @@ function FineTunedRows({ } gpuGb={gpu.available ? gpu.memoryTotalGb : undefined} systemRamGb={gpu.systemRamAvailableGb || undefined} + budgetKnown={gpu.budgetKnown} sourceOverride={isExportedGguf ? "exported" : undefined} variantActions={{ deleteTitle: "Delete exported GGUF variant?", diff --git a/studio/frontend/src/features/model-picker/components/model-selector/recommended-fit.ts b/studio/frontend/src/features/model-picker/components/model-selector/recommended-fit.ts index b8fe47c706..b7abe06313 100644 --- a/studio/frontend/src/features/model-picker/components/model-selector/recommended-fit.ts +++ b/studio/frontend/src/features/model-picker/components/model-selector/recommended-fit.ts @@ -97,13 +97,21 @@ export function fitsDevice(opts: { estimatedVramGb?: number; gpuGb?: number; systemRamGb?: number; + budgetKnown?: boolean; requireKnown?: boolean; }): boolean { - const { sizeBytes, estimatedVramGb, gpuGb, systemRamGb, requireKnown } = opts; + const { + sizeBytes, + estimatedVramGb, + gpuGb, + systemRamGb, + budgetKnown, + requireKnown, + } = opts; // Unified-memory hosts (Mac / no discrete GPU) report system RAM but no GPU, // so the budget must include RAM. Only an entirely unknown budget fits freely. const budgetGb = Math.max(0, gpuGb ?? 0) * 0.7 + Math.max(0, systemRamGb ?? 0) * 0.7; - if (budgetGb <= 0) return true; + if (budgetGb <= 0) return !budgetKnown; if (sizeBytes && sizeBytes > 0) { return sizeBytes / 1024 ** 3 <= budgetGb; } @@ -129,9 +137,18 @@ export function hfModelFitsDevice( estimatedSizeBytes?: number; isGguf?: boolean; }, - gpu: { memoryTotalGb: number; systemRamAvailableGb: number }, + gpu: { + memoryTotalGb: number; + systemRamAvailableGb: number; + budgetKnown?: boolean; + }, ): boolean { - if (gpu.memoryTotalGb <= 0 && gpu.systemRamAvailableGb <= 0) return true; + if ( + gpu.memoryTotalGb <= 0 && + gpu.systemRamAvailableGb <= 0 && + !gpu.budgetKnown + ) + return true; const params = model.totalParams ?? paramsFromId(model.id); const quantBytes = params ? estimateQuantBytes(params) : undefined; const sizeBytes = isGgufId(model.id, model.isGguf) @@ -141,6 +158,7 @@ export function hfModelFitsDevice( sizeBytes, gpuGb: gpu.memoryTotalGb, systemRamGb: gpu.systemRamAvailableGb, + budgetKnown: gpu.budgetKnown, requireKnown: true, }); } diff --git a/studio/frontend/src/features/settings/tabs/resources-tab.tsx b/studio/frontend/src/features/settings/tabs/resources-tab.tsx index b22a54cb0b..f45ddeba7c 100644 --- a/studio/frontend/src/features/settings/tabs/resources-tab.tsx +++ b/studio/frontend/src/features/settings/tabs/resources-tab.tsx @@ -10,7 +10,11 @@ import { openModelsDir, pickHuggingFaceCacheDir, } from "@/features/native-intents"; -import { useSystemInfo, type GpuDevice } from "@/hooks/use-system"; +import { + aggregateGpuMemoryTotalGb, + useSystemInfo, + type GpuDevice, +} from "@/hooks/use-system"; import { isTauri } from "@/lib/api-base"; import { copyToClipboard } from "@/lib/copy-to-clipboard"; import { toast } from "@/lib/toast"; @@ -184,6 +188,18 @@ export function ResourcesTab() { const [hfCacheLoaded, setHfCacheLoaded] = useState(false); const [cacheBrowserOpen, setCacheBrowserOpen] = useState(false); const [cacheSaving, setCacheSaving] = useState(false); + const displayedGpu = systemInfo.gpu?.available + ? systemInfo.gpu + : (systemInfo.inference_gpu ?? systemInfo.gpu); + const separateInferenceGpu = + systemInfo.gpu?.available && + systemInfo.inference_gpu && + systemInfo.inference_gpu.backend !== systemInfo.gpu.backend + ? systemInfo.inference_gpu + : null; + const inferenceVramTotal = separateInferenceGpu + ? aggregateGpuMemoryTotalGb(separateInferenceGpu.devices) + : 0; useEffect(() => { let cancelled = false; @@ -203,17 +219,14 @@ export function ResourcesTab() { }, []); const metrics = useMemo(() => { - const devices = systemInfo.gpu?.devices ?? []; + const devices = displayedGpu?.devices ?? []; const ramTotal = systemInfo.memory?.total_gb ?? 0; const ramAvailable = systemInfo.memory?.available_gb ?? 0; const ramUsed = Math.max(0, ramTotal - ramAvailable); const diskTotal = systemInfo.disk?.total_gb ?? 0; const diskFree = systemInfo.disk?.free_gb ?? 0; const diskUsed = Math.max(0, diskTotal - diskFree); - const vramTotal = devices.reduce( - (sum, device) => sum + (device.memory_total_gb ?? 0), - 0, - ); + const vramTotal = aggregateGpuMemoryTotalGb(devices); // null usage = unknown (e.g. Windows ROCm perf counter): treating it as 0 // fabricates a 0-used total, so the aggregate is unknown if any device is. const vramUsageKnown = @@ -252,7 +265,7 @@ export function ResourcesTab() { vramPercent, vramUsageKnown, }; - }, [systemInfo]); + }, [displayedGpu, systemInfo]); const handleCacheFolder = async () => { if (!hfCache) return; @@ -312,9 +325,9 @@ export function ResourcesTab() { : t("settings.resources.environment.unknown"); const cpuFrequencyLabel = formatFrequency(systemInfo.cpu?.frequency_mhz); const hasGpu = - (systemInfo.gpu?.available ?? false) && metrics.devices.length > 0; + (displayedGpu?.available ?? false) && metrics.devices.length > 0; const backendLabel = ( - systemInfo.gpu?.backend ?? systemInfo.device_backend ?? "cpu" + displayedGpu?.backend ?? systemInfo.device_backend ?? "cpu" ).toUpperCase(); const modelsFolderPath = hfCache ? hfCache.cacheHome @@ -426,6 +439,19 @@ export function ResourcesTab() { + {separateInferenceGpu && ( +
+ GGUF inference + + {separateInferenceGpu.backend ?? "GPU"} + {separateInferenceGpu.available + ? inferenceVramTotal + ? ` · ${formatGiB(inferenceVramTotal)}` + : "" + : " · unavailable"} + +
+ )} {hasGpu ? ( metrics.devices.map((device, index) => { const ordinal = deviceOrdinal(device); diff --git a/studio/frontend/src/hooks/index.ts b/studio/frontend/src/hooks/index.ts index 33289ffe9e..371ede0892 100644 --- a/studio/frontend/src/hooks/index.ts +++ b/studio/frontend/src/hooks/index.ts @@ -2,7 +2,7 @@ // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 export { useDebouncedValue } from "./use-debounced-value"; -export { useGpuInfo } from "./use-gpu-info"; +export { useGpuInfo, useInferenceGpuInfo } from "./use-gpu-info"; export { useGpuUtilization } from "./use-gpu-utilization"; export { useHardwareInfo } from "./use-hardware-info"; export { useHfDatasetSplits } from "./use-hf-dataset-splits"; diff --git a/studio/frontend/src/hooks/use-gpu-info.ts b/studio/frontend/src/hooks/use-gpu-info.ts index db2cc021be..c7eafc7e18 100644 --- a/studio/frontend/src/hooks/use-gpu-info.ts +++ b/studio/frontend/src/hooks/use-gpu-info.ts @@ -3,10 +3,14 @@ import { authFetch } from "@/features/auth"; import { useEffect, useState } from "react"; -import type { SystemInfoResponse } from "./use-system"; +import { + aggregateGpuMemoryTotalGb, + type SystemInfoResponse, +} from "./use-system"; export interface GpuInfo { available: boolean; + budgetKnown: boolean; name: string; memoryTotalGb: number; cpuCore: number; @@ -30,6 +34,7 @@ export interface SystemGpuDevice { const DEFAULT_GPU: GpuInfo = { available: false, + budgetKnown: false, name: "Unknown", memoryTotalGb: 0, cpuCore: 0, @@ -42,8 +47,8 @@ const DEFAULT_GPU: GpuInfo = { let cachedSystem: SystemInfoResponse | null = null; let systemPromise: Promise | null = null; -async function fetchSystemOnce(): Promise { - if (cachedSystem) return cachedSystem; +async function fetchSystemOnce(force = false): Promise { + if (!force && cachedSystem) return cachedSystem; if (systemPromise) return systemPromise; systemPromise = (async () => { try { @@ -52,14 +57,18 @@ async function fetchSystemOnce(): Promise { cachedSystem = (await res.json()) as SystemInfoResponse; return cachedSystem; } catch { - systemPromise = null; // reset so a later call retries (backend not ready) return null; + } finally { + systemPromise = null; } })(); return systemPromise; } -function toGpuInfo(data: SystemInfoResponse | null): GpuInfo { +function toGpuInfo( + data: SystemInfoResponse | null, + source: "gpu" | "inference_gpu" = "gpu", +): GpuInfo { // CPU/RAM exist even on GPU-less hosts (e.g. Mac), so populate them on every // path: unified-memory math still needs a RAM budget to work with. const base = { @@ -68,16 +77,25 @@ function toGpuInfo(data: SystemInfoResponse | null): GpuInfo { systemRamAvailableGb: data?.memory?.available_gb ?? 0, systemRamTotalGb: data?.memory?.total_gb ?? 0, }; - const gpuData = data?.gpu; + const gpuData = + source === "inference_gpu" + ? (data?.inference_gpu ?? data?.gpu) + : data?.gpu; const devices = gpuData?.devices ?? []; if (!gpuData?.available || !devices.length) { - return { ...DEFAULT_GPU, ...base }; + return { ...DEFAULT_GPU, ...base, budgetKnown: data !== null }; } return { ...base, + // A Vulkan iGPU's reported budget is capped shared system RAM, not an + // independent VRAM pool. Do not offer the same RAM again for CPU offload. + systemRamAvailableGb: devices.some((device) => device.shared_memory) + ? 0 + : base.systemRamAvailableGb, available: true, + budgetKnown: true, name: devices[0]?.name ?? "Unknown", - memoryTotalGb: devices.reduce((sum, d) => sum + (d.memory_total_gb ?? 0), 0), + memoryTotalGb: aggregateGpuMemoryTotalGb(devices), }; } @@ -104,24 +122,56 @@ function toGpuDevices(data: SystemInfoResponse | null): SystemGpuDevice[] { } /** Aggregate GPU info from /api/system; shares one module-level fetch across all GPU hooks. */ -export function useGpuInfo(): GpuInfo { +function useGpuInfoSource(source: "gpu" | "inference_gpu"): GpuInfo { const [gpu, setGpu] = useState( - cachedSystem ? toGpuInfo(cachedSystem) : DEFAULT_GPU, + cachedSystem ? toGpuInfo(cachedSystem, source) : DEFAULT_GPU, ); useEffect(() => { // No early return on cachedSystem: a consumer mounting as the cache fills // (between render and effect) would otherwise stay stuck at the default. let cancelled = false; - fetchSystemOnce().then((d) => { - if (!cancelled) setGpu(toGpuInfo(d)); - }); + let retryId: number | undefined; + const update = (force = false, retryVulkan = false) => { + fetchSystemOnce(force).then((d) => { + if (cancelled) return; + if (!d) { + // Once an unavailable Vulkan backend starts polling, a transient API + // failure must preserve the current state and continue the same loop. + if (retryVulkan) { + retryId = window.setTimeout(() => update(true, true), 3000); + } + return; + } + setGpu(toGpuInfo(d, source)); + const inferenceGpu = d.inference_gpu; + if ( + source === "inference_gpu" && + inferenceGpu?.backend === "vulkan" && + !inferenceGpu.available + ) { + retryId = window.setTimeout(() => update(true, true), 3000); + } + }); + }; + update(); return () => { cancelled = true; + if (retryId !== undefined) window.clearTimeout(retryId); }; - }, []); + }, [source]); return gpu; } +/** Training-capable GPU info from the PyTorch/MLX hardware detector. */ +export function useGpuInfo(): GpuInfo { + return useGpuInfoSource("gpu"); +} + +/** GGUF inference GPU info, including a separately installed Vulkan backend. */ +export function useInferenceGpuInfo(): GpuInfo { + return useGpuInfoSource("inference_gpu"); +} + /** All backend-visible GPUs (index, name, total VRAM); shares the same fetch. */ export function useGpuDevices(): SystemGpuDevice[] { const [devices, setDevices] = useState( diff --git a/studio/frontend/src/hooks/use-system.ts b/studio/frontend/src/hooks/use-system.ts index 8cfe2bace4..c118532a99 100644 --- a/studio/frontend/src/hooks/use-system.ts +++ b/studio/frontend/src/hooks/use-system.ts @@ -13,6 +13,33 @@ export interface GpuDevice { vram_used_gb?: number; vram_free_gb?: number; vram_utilization_pct?: number | null; + /** True when the reported GPU budget comes from shared system memory. */ + shared_memory?: boolean; +} + +export interface SystemGpuInfo { + available: boolean; + backend?: string; + /** Whether GGUF loads accept explicit physical GPU IDs. */ + gguf_gpu_ids_supported?: boolean; + backend_cuda_visible_devices?: string | null; + parent_visible_gpu_ids?: number[]; + index_kind?: string; + devices: GpuDevice[]; +} + +/** Sum dedicated VRAM while counting a shared host-memory pool only once. */ +export function aggregateGpuMemoryTotalGb(devices: GpuDevice[]): number { + const dedicated = devices + .filter((device) => !device.shared_memory) + .reduce((sum, device) => sum + (device.memory_total_gb ?? 0), 0); + const shared = Math.max( + 0, + ...devices + .filter((device) => device.shared_memory) + .map((device) => device.memory_total_gb ?? 0), + ); + return dedicated + shared; } export interface SystemInfoResponse { @@ -37,17 +64,9 @@ export interface SystemInfoResponse { free_gb: number; percent_used: number; }; - gpu: { - available: boolean; - backend?: string; - /** Whether GGUF loads accept an explicit gpu_ids pick (false on XPU hosts - * and Vulkan-only builds, where /load and /validate 400 picks). */ - gguf_gpu_ids_supported?: boolean; - backend_cuda_visible_devices?: string | null; - parent_visible_gpu_ids?: number[]; - index_kind?: string; - devices: GpuDevice[]; - }; + gpu: SystemGpuInfo; + /** Devices available to GGUF inference; differs when llama.cpp uses Vulkan. */ + inference_gpu?: SystemGpuInfo; ml_packages: { torch?: string; transformers?: string;