fix(studio): report Vulkan GPUs in system UI (#7476)
* fix(studio): report Vulkan GPUs in system UI * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix(studio): separate Vulkan inference GPU reporting * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix(studio): keep retrying Vulkan probe refreshes * fix(studio): preserve known zero GPU budgets --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
alkinun
GitHub
parent
9eaf5c29a5
commit
217e8f036c
14 changed files with 692 additions and 99 deletions
18
README.md
18
README.md
|
|
@ -103,7 +103,7 @@ Unsloth Studio (Beta) works on **Windows, Linux, WSL** and **macOS**.
|
|||
* **NVIDIA:** Training works on RTX 30/40/50, Blackwell, DGX Spark, Station and more
|
||||
* **macOS:** Training, MLX and GGUF inference are ALL supported.
|
||||
* **AMD:** Training, RL, chat and deployment work on Windows, WSL and Linux. [Read the AMD guide](https://unsloth.ai/docs/basics/amd).
|
||||
* **Vulkan:** GGUF inference is supported on [compatible GPUs, including Intel GPUs](https://github.com/unslothai/unsloth/pull/5819).
|
||||
* **Vulkan:** GGUF inference is supported on [compatible GPUs, including Intel GPUs](https://github.com/unslothai/unsloth/pull/5819). Vulkan accelerates GGUF inference only; training still requires a supported PyTorch or MLX backend.
|
||||
* **Multi-GPU:** Available now, with a major upgrade on the way
|
||||
|
||||
#### macOS, Linux, WSL:
|
||||
|
|
@ -112,12 +112,28 @@ curl -fsSL https://unsloth.ai/install.sh | sh
|
|||
```
|
||||
Use the same command to update.
|
||||
|
||||
To force the Vulkan llama.cpp backend, set `UNSLOTH_FORCE_VULKAN=1` **before installing or updating**. The setting selects the llama.cpp binary bundle, so setting it only when launching Studio cannot replace an existing CPU bundle:
|
||||
|
||||
```bash
|
||||
export UNSLOTH_FORCE_VULKAN=1
|
||||
curl -fsSL https://unsloth.ai/install.sh | sh
|
||||
```
|
||||
|
||||
#### Windows:
|
||||
```powershell
|
||||
irm https://unsloth.ai/install.ps1 | iex
|
||||
```
|
||||
Use the same command to update.
|
||||
|
||||
To force the Vulkan llama.cpp backend, set the environment variable before running the installer or updater:
|
||||
|
||||
```powershell
|
||||
$env:UNSLOTH_FORCE_VULKAN=1
|
||||
irm https://unsloth.ai/install.ps1 | iex
|
||||
```
|
||||
|
||||
Re-running the current installer replaces a previously selected CPU bundle when the backend differs. A separate Vulkan SDK is not required; the GPU driver must provide a working Vulkan runtime.
|
||||
|
||||
#### Launch
|
||||
```bash
|
||||
unsloth studio -p 8888
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ if sys.platform == "win32":
|
|||
|
||||
_SYSTEM_GPU_CACHE_TTL_SECONDS = 10.0
|
||||
_system_gpu_cache_lock = threading.Lock()
|
||||
_system_gpu_cache: Optional[tuple[float, dict[str, Any]]] = None
|
||||
_system_gpu_cache: Optional[tuple[float, tuple[dict[str, Any], dict[str, Any]]]] = None
|
||||
|
||||
# ── Windows AMD ROCm DLL injection ──────────────────────────────────────────
|
||||
# Python 3.8+ ignores PATH for extension modules; register ROCm bin dirs with
|
||||
|
|
@ -1149,10 +1149,14 @@ async def shutdown_server(request: Request, current_subject: str = Depends(get_c
|
|||
return {"status": "shutting_down"}
|
||||
|
||||
|
||||
def _get_cached_system_gpu_info(logger) -> dict[str, Any]:
|
||||
"""Return merged GPU visibility/utilization with bounded live-probe churn."""
|
||||
def _get_cached_system_gpu_info(logger) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
"""Return training and inference GPU info with bounded live-probe churn."""
|
||||
import time
|
||||
from utils.hardware import get_backend_visible_gpu_info, get_visible_gpu_utilization
|
||||
from utils.hardware import (
|
||||
get_backend_visible_gpu_info,
|
||||
get_visible_gpu_utilization,
|
||||
get_vulkan_inference_gpu_info,
|
||||
)
|
||||
|
||||
global _system_gpu_cache
|
||||
now = time.monotonic()
|
||||
|
|
@ -1174,7 +1178,20 @@ def _get_cached_system_gpu_info(logger) -> dict[str, Any]:
|
|||
logger.debug(f"Failed to get GPU utilization info: {e}")
|
||||
utilization_info = {"devices": []}
|
||||
|
||||
util_devices = {d.get("index"): d for d in utilization_info.get("devices", [])}
|
||||
# Device indices are backend-specific. Never overlay CUDA/ROCm metrics
|
||||
# onto compact Vulkan ordinals merely because both happen to start at 0.
|
||||
visibility_backend = visibility_info.get("backend")
|
||||
utilization_backend = utilization_info.get("backend")
|
||||
metrics_match = (
|
||||
not visibility_backend
|
||||
or not utilization_backend
|
||||
or visibility_backend == utilization_backend
|
||||
)
|
||||
util_devices = (
|
||||
{d.get("index"): d for d in utilization_info.get("devices", [])}
|
||||
if metrics_match
|
||||
else {}
|
||||
)
|
||||
enriched_devices = []
|
||||
|
||||
for dev in visibility_info.get("devices", []):
|
||||
|
|
@ -1184,14 +1201,19 @@ def _get_cached_system_gpu_info(logger) -> dict[str, Any]:
|
|||
total_vram = util.get("vram_total_gb") or dev.get("memory_total_gb") or 0
|
||||
# Keep None (usage unknown, e.g. Windows ROCm perf counter) so the UI
|
||||
# shows unknown, not a fabricated 0 used / full free.
|
||||
used_vram = util.get("vram_used_gb")
|
||||
used_vram = util.get("vram_used_gb", dev.get("vram_used_gb"))
|
||||
reported_free_vram = util.get("vram_free_gb", dev.get("vram_free_gb"))
|
||||
|
||||
enriched_dev = dict(dev)
|
||||
enriched_dev["vram_used_gb"] = used_vram
|
||||
enriched_dev["vram_free_gb"] = (
|
||||
round(total_vram - used_vram, 2) if total_vram and used_vram is not None else None
|
||||
round(total_vram - used_vram, 2)
|
||||
if total_vram and used_vram is not None
|
||||
else reported_free_vram
|
||||
)
|
||||
enriched_dev["vram_utilization_pct"] = util.get(
|
||||
"vram_utilization_pct", dev.get("vram_utilization_pct")
|
||||
)
|
||||
enriched_dev["vram_utilization_pct"] = util.get("vram_utilization_pct")
|
||||
enriched_devices.append(enriched_dev)
|
||||
|
||||
# Whether GGUF loads accept an explicit gpu_ids pick: /load and
|
||||
|
|
@ -1207,13 +1229,37 @@ def _get_cached_system_gpu_info(logger) -> dict[str, Any]:
|
|||
except Exception as e:
|
||||
logger.debug(f"Could not resolve gpu_ids support: {e}")
|
||||
gpu_ids_supported = True
|
||||
# Preserve backend/index metadata from the visibility probe. In
|
||||
# particular, a CPU training host can expose a Vulkan inference GPU and
|
||||
# the UI must label that device as Vulkan rather than falling back to the
|
||||
# top-level CPU training backend.
|
||||
gpu_info = {
|
||||
**visibility_info,
|
||||
"available": visibility_info.get("available", False),
|
||||
"devices": enriched_devices,
|
||||
"gguf_gpu_ids_supported": gpu_ids_supported,
|
||||
}
|
||||
_system_gpu_cache = (time.monotonic(), gpu_info)
|
||||
return gpu_info
|
||||
|
||||
# Keep inference placement separate on train-capable hosts where a
|
||||
# forced Vulkan llama.cpp bundle can enumerate a different device set.
|
||||
# If Vulkan is installed but its probe fails, retain the unavailable
|
||||
# Vulkan shape instead of budgeting training GPUs that llama.cpp cannot use.
|
||||
if visibility_info.get("backend") == "vulkan":
|
||||
inference_gpu_info = gpu_info
|
||||
else:
|
||||
vulkan_info = get_vulkan_inference_gpu_info()
|
||||
inference_gpu_info = (
|
||||
{
|
||||
**vulkan_info,
|
||||
"gguf_gpu_ids_supported": False,
|
||||
}
|
||||
if vulkan_info is not None
|
||||
else gpu_info
|
||||
)
|
||||
|
||||
combined_info = (gpu_info, inference_gpu_info)
|
||||
_system_gpu_cache = (time.monotonic(), combined_info)
|
||||
return combined_info
|
||||
|
||||
|
||||
@app.get("/api/system")
|
||||
|
|
@ -1234,7 +1280,7 @@ def get_system_info(current_subject: str = Depends(get_current_subject)):
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
gpu_info = _get_cached_system_gpu_info(logger)
|
||||
gpu_info, inference_gpu_info = _get_cached_system_gpu_info(logger)
|
||||
|
||||
memory = psutil.virtual_memory()
|
||||
|
||||
|
|
@ -1301,6 +1347,7 @@ def get_system_info(current_subject: str = Depends(get_current_subject)):
|
|||
"percent_used": disk.percent if disk else 0,
|
||||
},
|
||||
"gpu": gpu_info,
|
||||
"inference_gpu": inference_gpu_info,
|
||||
"ml_packages": ml_packages,
|
||||
# Export capability + torch-aware reason. See /api/system/hardware.
|
||||
**export_capability(),
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ from utils.hardware import (
|
|||
get_offloaded_device_map_entries,
|
||||
get_parent_visible_gpu_ids,
|
||||
get_visible_gpu_utilization,
|
||||
get_vulkan_inference_gpu_info,
|
||||
prepare_gpu_selection,
|
||||
resolve_requested_gpu_ids,
|
||||
)
|
||||
|
|
@ -411,6 +412,108 @@ class TestVisibleGpuUtilization(_GpuCacheResetMixin, unittest.TestCase):
|
|||
self.assertEqual(result["devices"][0]["index"], 0)
|
||||
self.assertEqual(result["devices"][0]["visible_ordinal"], 0)
|
||||
|
||||
def test_discrete_vulkan_inference_gpu_info(self):
|
||||
with (
|
||||
patch(
|
||||
"core.inference.llama_cpp.LlamaCppBackend._is_vulkan_backend",
|
||||
return_value = True,
|
||||
),
|
||||
patch(
|
||||
"core.inference.llama_cpp.LlamaCppBackend._get_gpu_memory",
|
||||
return_value = [(0, 7402, 8192)],
|
||||
),
|
||||
):
|
||||
result = get_vulkan_inference_gpu_info()
|
||||
|
||||
self.assertTrue(result["available"])
|
||||
self.assertEqual(result["backend"], "vulkan")
|
||||
self.assertEqual(result["index_kind"], "relative")
|
||||
self.assertEqual(result["parent_visible_gpu_ids"], [])
|
||||
self.assertEqual(
|
||||
result["devices"],
|
||||
[
|
||||
{
|
||||
"index": 0,
|
||||
"index_kind": "relative",
|
||||
"visible_ordinal": 0,
|
||||
"name": "Vulkan0",
|
||||
"memory_total_gb": 8.0,
|
||||
"vram_used_gb": 0.77,
|
||||
"vram_free_gb": 7.23,
|
||||
"vram_utilization_pct": 9.6,
|
||||
"shared_memory": False,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
def test_vulkan_igpu_info_uses_capped_free_budget(self):
|
||||
with (
|
||||
patch(
|
||||
"core.inference.llama_cpp.LlamaCppBackend._is_vulkan_backend",
|
||||
return_value = True,
|
||||
),
|
||||
patch(
|
||||
"core.inference.llama_cpp.LlamaCppBackend._get_gpu_memory",
|
||||
return_value = [(0, 12288, 0)],
|
||||
),
|
||||
):
|
||||
result = get_vulkan_inference_gpu_info()
|
||||
|
||||
device = result["devices"][0]
|
||||
self.assertEqual(device["memory_total_gb"], 12.0)
|
||||
self.assertEqual(device["vram_free_gb"], 12.0)
|
||||
self.assertIsNone(device["vram_used_gb"])
|
||||
self.assertIsNone(device["vram_utilization_pct"])
|
||||
self.assertTrue(device["shared_memory"])
|
||||
|
||||
def test_forced_vulkan_overrides_torch_gpu_visibility_for_inference(self):
|
||||
with (
|
||||
patch("utils.hardware.hardware.get_device", return_value = DeviceType.CUDA),
|
||||
patch(
|
||||
"core.inference.llama_cpp.LlamaCppBackend._is_vulkan_backend",
|
||||
return_value = True,
|
||||
),
|
||||
patch(
|
||||
"core.inference.llama_cpp.LlamaCppBackend._get_gpu_memory",
|
||||
return_value = [(1, 6144, 8192)],
|
||||
),
|
||||
patch(
|
||||
"utils.hardware.nvidia.get_backend_visible_gpu_info",
|
||||
return_value = {
|
||||
"available": True,
|
||||
"backend": "cuda",
|
||||
"devices": [{"index": 0, "name": "CUDA0", "memory_total_gb": 24.0}],
|
||||
},
|
||||
),
|
||||
patch(
|
||||
"utils.hardware.hardware._get_parent_visible_gpu_spec",
|
||||
return_value = {"raw": None, "numeric_ids": None},
|
||||
),
|
||||
):
|
||||
training_result = get_backend_visible_gpu_info()
|
||||
inference_result = get_vulkan_inference_gpu_info()
|
||||
|
||||
self.assertEqual(training_result["backend"], "cuda")
|
||||
self.assertEqual(inference_result["backend"], "vulkan")
|
||||
self.assertEqual(inference_result["devices"][0]["index"], 1)
|
||||
|
||||
def test_vulkan_install_without_devices_reports_unavailable(self):
|
||||
with (
|
||||
patch(
|
||||
"core.inference.llama_cpp.LlamaCppBackend._is_vulkan_backend",
|
||||
return_value = True,
|
||||
),
|
||||
patch(
|
||||
"core.inference.llama_cpp.LlamaCppBackend._get_gpu_memory",
|
||||
return_value = [],
|
||||
),
|
||||
):
|
||||
result = get_vulkan_inference_gpu_info()
|
||||
|
||||
self.assertFalse(result["available"])
|
||||
self.assertEqual(result["backend"], "vulkan")
|
||||
self.assertEqual(result["devices"], [])
|
||||
|
||||
|
||||
class TestGpuAutoSelection(_GpuCacheResetMixin, unittest.TestCase):
|
||||
def test_get_device_map_uses_explicit_gpu_selection(self):
|
||||
|
|
|
|||
171
studio/backend/tests/test_system_vulkan_gpu_info.py
Normal file
171
studio/backend/tests/test_system_vulkan_gpu_info.py
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import main
|
||||
|
||||
|
||||
def test_system_gpu_info_preserves_vulkan_visibility_metrics(monkeypatch):
|
||||
import utils.hardware as hardware
|
||||
|
||||
vulkan_device = {
|
||||
"index": 0,
|
||||
"index_kind": "relative",
|
||||
"visible_ordinal": 0,
|
||||
"name": "Vulkan0",
|
||||
"memory_total_gb": 8.0,
|
||||
"vram_used_gb": 0.77,
|
||||
"vram_free_gb": 7.23,
|
||||
"vram_utilization_pct": 9.6,
|
||||
"shared_memory": False,
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
hardware,
|
||||
"get_backend_visible_gpu_info",
|
||||
lambda: {
|
||||
"available": False,
|
||||
"backend": "cpu",
|
||||
"devices": [],
|
||||
"index_kind": "relative",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
hardware,
|
||||
"get_visible_gpu_utilization",
|
||||
lambda: {"available": False, "backend": "cpu", "devices": []},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
hardware,
|
||||
"get_vulkan_inference_gpu_info",
|
||||
lambda: {
|
||||
"available": True,
|
||||
"backend": "vulkan",
|
||||
"devices": [vulkan_device],
|
||||
"index_kind": "relative",
|
||||
},
|
||||
)
|
||||
|
||||
from core.inference.llama_cpp import LlamaCppBackend
|
||||
|
||||
monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda: True))
|
||||
monkeypatch.setattr(main, "_system_gpu_cache", None)
|
||||
|
||||
gpu, inference_gpu = main._get_cached_system_gpu_info(SimpleNamespace(debug = lambda *args: None))
|
||||
|
||||
assert gpu["available"] is False
|
||||
assert gpu["backend"] == "cpu"
|
||||
assert gpu["index_kind"] == "relative"
|
||||
assert gpu["gguf_gpu_ids_supported"] is False
|
||||
assert gpu["devices"] == []
|
||||
assert inference_gpu["backend"] == "vulkan"
|
||||
assert inference_gpu["devices"] == [vulkan_device]
|
||||
|
||||
|
||||
def test_system_gpu_info_keeps_forced_vulkan_separate_from_training_metrics(monkeypatch):
|
||||
import utils.hardware as hardware
|
||||
|
||||
monkeypatch.setattr(
|
||||
hardware,
|
||||
"get_backend_visible_gpu_info",
|
||||
lambda: {
|
||||
"available": True,
|
||||
"backend": "cuda",
|
||||
"devices": [{"index": 0, "name": "CUDA0", "memory_total_gb": 24.0}],
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
hardware,
|
||||
"get_visible_gpu_utilization",
|
||||
lambda: {
|
||||
"available": True,
|
||||
"backend": "cuda",
|
||||
"devices": [
|
||||
{
|
||||
"index": 0,
|
||||
"vram_total_gb": 24.0,
|
||||
"vram_used_gb": 6.0,
|
||||
"vram_utilization_pct": 25.0,
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
hardware,
|
||||
"get_vulkan_inference_gpu_info",
|
||||
lambda: {
|
||||
"available": True,
|
||||
"backend": "vulkan",
|
||||
"devices": [
|
||||
{
|
||||
"index": 0,
|
||||
"name": "Vulkan0",
|
||||
"memory_total_gb": 8.0,
|
||||
"vram_used_gb": 1.0,
|
||||
"vram_free_gb": 7.0,
|
||||
"vram_utilization_pct": 12.5,
|
||||
"shared_memory": False,
|
||||
}
|
||||
],
|
||||
"index_kind": "relative",
|
||||
},
|
||||
)
|
||||
|
||||
from core.inference.llama_cpp import LlamaCppBackend
|
||||
from utils.hardware import DeviceType
|
||||
|
||||
monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda: True))
|
||||
monkeypatch.setattr(hardware, "get_device", lambda: DeviceType.CUDA)
|
||||
monkeypatch.setattr(main, "_system_gpu_cache", None)
|
||||
|
||||
gpu, inference_gpu = main._get_cached_system_gpu_info(SimpleNamespace(debug = lambda *args: None))
|
||||
|
||||
assert gpu["backend"] == "cuda"
|
||||
assert gpu["devices"][0]["vram_used_gb"] == 6.0
|
||||
assert inference_gpu["backend"] == "vulkan"
|
||||
assert inference_gpu["devices"][0]["vram_used_gb"] == 1.0
|
||||
assert inference_gpu["gguf_gpu_ids_supported"] is False
|
||||
|
||||
|
||||
def test_system_gpu_info_does_not_merge_metrics_across_backend_index_spaces(monkeypatch):
|
||||
import utils.hardware as hardware
|
||||
|
||||
vulkan_device = {
|
||||
"index": 0,
|
||||
"name": "Vulkan0",
|
||||
"memory_total_gb": 8.0,
|
||||
"vram_used_gb": 1.0,
|
||||
"vram_free_gb": 7.0,
|
||||
"vram_utilization_pct": 12.5,
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
hardware,
|
||||
"get_backend_visible_gpu_info",
|
||||
lambda: {"available": True, "backend": "vulkan", "devices": [vulkan_device]},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
hardware,
|
||||
"get_visible_gpu_utilization",
|
||||
lambda: {
|
||||
"available": True,
|
||||
"backend": "cuda",
|
||||
"devices": [
|
||||
{
|
||||
"index": 0,
|
||||
"vram_total_gb": 24.0,
|
||||
"vram_used_gb": 20.0,
|
||||
"vram_utilization_pct": 83.3,
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
from core.inference.llama_cpp import LlamaCppBackend
|
||||
|
||||
monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda: True))
|
||||
monkeypatch.setattr(main, "_system_gpu_cache", None)
|
||||
|
||||
gpu, inference_gpu = main._get_cached_system_gpu_info(SimpleNamespace(debug = lambda *args: None))
|
||||
|
||||
assert gpu["devices"] == [vulkan_device]
|
||||
assert inference_gpu == gpu
|
||||
|
|
@ -19,6 +19,7 @@ from .hardware import (
|
|||
get_gpu_utilization,
|
||||
get_visible_gpu_utilization,
|
||||
get_backend_visible_gpu_info,
|
||||
get_vulkan_inference_gpu_info,
|
||||
get_physical_gpu_count,
|
||||
get_visible_gpu_count,
|
||||
get_parent_visible_gpu_ids,
|
||||
|
|
@ -72,6 +73,7 @@ __all__ = [
|
|||
"get_gpu_utilization",
|
||||
"get_visible_gpu_utilization",
|
||||
"get_backend_visible_gpu_info",
|
||||
"get_vulkan_inference_gpu_info",
|
||||
"get_physical_gpu_count",
|
||||
"get_visible_gpu_count",
|
||||
"get_parent_visible_gpu_ids",
|
||||
|
|
|
|||
|
|
@ -296,7 +296,7 @@ def detect_hardware() -> DeviceType:
|
|||
CHAT_ONLY_REASON = "intel_mac" # Intel Mac: no PyTorch/MLX -> GGUF-only by design.
|
||||
else:
|
||||
CHAT_ONLY_REASON = "no_gpu"
|
||||
print("Hardware detected: CPU (no GPU backend available)")
|
||||
print("Hardware detected: CPU training backend (no PyTorch/MLX GPU backend available)")
|
||||
return DEVICE
|
||||
|
||||
|
||||
|
|
@ -2575,8 +2575,65 @@ def _backend_visible_devices_env() -> Optional[str]:
|
|||
return os.environ.get("CUDA_VISIBLE_DEVICES")
|
||||
|
||||
|
||||
def get_vulkan_inference_gpu_info() -> Optional[Dict[str, Any]]:
|
||||
"""Return llama.cpp Vulkan devices, or None when Vulkan is not installed."""
|
||||
# Vulkan is a llama.cpp inference backend, not a PyTorch training device, so
|
||||
# keep it separate from the PyTorch/MLX training-device report.
|
||||
try:
|
||||
from core.inference.llama_cpp import LlamaCppBackend
|
||||
except Exception as e:
|
||||
logger.debug("Could not inspect the llama.cpp Vulkan backend: %s", e)
|
||||
return None
|
||||
|
||||
try:
|
||||
if not LlamaCppBackend._is_vulkan_backend():
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.debug("Could not identify the llama.cpp Vulkan backend: %s", e)
|
||||
return None
|
||||
|
||||
result = {
|
||||
"available": False,
|
||||
"backend": "vulkan",
|
||||
"backend_cuda_visible_devices": None,
|
||||
"parent_visible_gpu_ids": [],
|
||||
"devices": [],
|
||||
"index_kind": "relative",
|
||||
}
|
||||
try:
|
||||
for ordinal, free_mib, total_mib in LlamaCppBackend._get_gpu_memory():
|
||||
# Integrated Vulkan GPUs report total=0 because their memory is
|
||||
# shared. Publish the capped free value as their usable inference
|
||||
# budget and mark it so clients do not add system RAM again.
|
||||
shared_memory = total_mib == 0
|
||||
budget_mib = total_mib or free_mib
|
||||
used_mib = max(0, total_mib - free_mib) if total_mib else None
|
||||
result["devices"].append(
|
||||
{
|
||||
"index": ordinal,
|
||||
"index_kind": "relative",
|
||||
"visible_ordinal": ordinal,
|
||||
"name": f"Vulkan{ordinal}",
|
||||
"memory_total_gb": round(budget_mib / 1024, 2),
|
||||
"vram_used_gb": round(used_mib / 1024, 2) if used_mib is not None else None,
|
||||
"vram_free_gb": round(free_mib / 1024, 2),
|
||||
"vram_utilization_pct": round((used_mib / total_mib) * 100, 1)
|
||||
if used_mib is not None and total_mib > 0
|
||||
else None,
|
||||
"shared_memory": shared_memory,
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug("Vulkan GPU visibility query failed: %s", e)
|
||||
return result
|
||||
|
||||
result["available"] = bool(result["devices"])
|
||||
return result
|
||||
|
||||
|
||||
def get_backend_visible_gpu_info() -> Dict[str, Any]:
|
||||
device = get_device()
|
||||
|
||||
if device in (DeviceType.CUDA, DeviceType.XPU):
|
||||
parent_visible_ids = get_parent_visible_gpu_ids()
|
||||
# Try native SMI first (nvidia-smi; skipped for ROCm).
|
||||
|
|
|
|||
|
|
@ -4,7 +4,10 @@
|
|||
import { Button } from "@/components/ui/button";
|
||||
import { Progress } from "@/components/ui/progress";
|
||||
import { useMonitorOverlayStore } from "@/features/settings";
|
||||
import { useSystemInfo } from "@/hooks/use-system";
|
||||
import {
|
||||
aggregateGpuMemoryTotalGb,
|
||||
useSystemInfo,
|
||||
} from "@/hooks/use-system";
|
||||
import { useT } from "@/i18n";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { CpuIcon, GripVerticalIcon, XIcon } from "lucide-react";
|
||||
|
|
@ -65,11 +68,20 @@ export function FloatingMonitor() {
|
|||
const ramUsed = Math.max(0, ramTotal - ramAvailable);
|
||||
const ramPercent = clampPercent(systemInfo.memory?.percent_used ?? 0);
|
||||
|
||||
const devices = systemInfo.gpu?.devices ?? [];
|
||||
const vramTotal = devices.reduce(
|
||||
(sum, device) => sum + (device.memory_total_gb ?? 0),
|
||||
0,
|
||||
);
|
||||
const displayedGpu = systemInfo.gpu?.available
|
||||
? systemInfo.gpu
|
||||
: (systemInfo.inference_gpu ?? systemInfo.gpu);
|
||||
const separateInferenceGpu =
|
||||
systemInfo.gpu?.available &&
|
||||
systemInfo.inference_gpu &&
|
||||
systemInfo.inference_gpu.backend !== systemInfo.gpu.backend
|
||||
? systemInfo.inference_gpu
|
||||
: null;
|
||||
const inferenceVramTotal = separateInferenceGpu
|
||||
? aggregateGpuMemoryTotalGb(separateInferenceGpu.devices)
|
||||
: 0;
|
||||
const devices = displayedGpu?.devices ?? [];
|
||||
const vramTotal = aggregateGpuMemoryTotalGb(devices);
|
||||
// null usage = unknown (e.g. Windows ROCm perf counter): treating it as 0
|
||||
// fabricates a 0-used readout, so the aggregate is unknown if any device is.
|
||||
const vramUsageKnown =
|
||||
|
|
@ -83,7 +95,7 @@ export function FloatingMonitor() {
|
|||
);
|
||||
const unknownLabel = t("settings.resources.environment.unknown");
|
||||
|
||||
const hasGpu = (systemInfo.gpu?.available ?? false) && devices.length > 0;
|
||||
const hasGpu = (displayedGpu?.available ?? false) && devices.length > 0;
|
||||
|
||||
return (
|
||||
<AnimatePresence>
|
||||
|
|
@ -188,6 +200,19 @@ export function FloatingMonitor() {
|
|||
/>
|
||||
</div>
|
||||
)}
|
||||
{separateInferenceGpu && (
|
||||
<div className="flex justify-between gap-2 text-ui-11 font-mono">
|
||||
<span className="text-muted-foreground">GGUF inference</span>
|
||||
<span className="uppercase text-foreground">
|
||||
{separateInferenceGpu.backend ?? "GPU"}
|
||||
{separateInferenceGpu.available
|
||||
? inferenceVramTotal
|
||||
? ` · ${formatGiB(inferenceVramTotal)}`
|
||||
: ""
|
||||
: " · unavailable"}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</motion.div>
|
||||
</motion.div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ import {
|
|||
resolveInitialConfig,
|
||||
} from "@/features/model-picker";
|
||||
import { useDebouncedValue } from "@/hooks/use-debounced-value";
|
||||
import { useGpuInfo } from "@/hooks/use-gpu-info";
|
||||
import { useGpuInfo, useInferenceGpuInfo } from "@/hooks/use-gpu-info";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { useNavigate, useSearch } from "@tanstack/react-router";
|
||||
import {
|
||||
|
|
@ -334,6 +334,7 @@ function selectedRepoMatchesRuntime(
|
|||
export function ModelsPage() {
|
||||
const navigate = useNavigate();
|
||||
const gpu = useGpuInfo();
|
||||
const inferenceGpu = useInferenceGpuInfo();
|
||||
const online = useOnlineStatus();
|
||||
const deviceType = usePlatformStore((s) => s.deviceType);
|
||||
const hubSearch = useSearch({ from: "/hub" });
|
||||
|
|
@ -757,7 +758,10 @@ export function ModelsPage() {
|
|||
// matching the chat model selector.
|
||||
(!fitOnDeviceOnly ||
|
||||
row.isAvailableOnDevice ||
|
||||
hfModelFitsDevice(row.result, gpu)),
|
||||
hfModelFitsDevice(
|
||||
row.result,
|
||||
row.result.isGguf ? inferenceGpu : gpu,
|
||||
)),
|
||||
);
|
||||
}, [
|
||||
discoverRows,
|
||||
|
|
@ -769,6 +773,7 @@ export function ModelsPage() {
|
|||
activeChannel,
|
||||
fitOnDeviceOnly,
|
||||
gpu,
|
||||
inferenceGpu,
|
||||
]);
|
||||
|
||||
const listRows = filteredDiscoverRows;
|
||||
|
|
@ -799,7 +804,7 @@ export function ModelsPage() {
|
|||
(row) =>
|
||||
!fitOnDeviceOnly ||
|
||||
row.isAvailableOnDevice ||
|
||||
hfModelFitsDevice(row.result, gpu),
|
||||
hfModelFitsDevice(row.result, inferenceGpu),
|
||||
),
|
||||
[
|
||||
hubFeed.trending.results,
|
||||
|
|
@ -807,6 +812,7 @@ export function ModelsPage() {
|
|||
modelDiscoveryInventorySignature,
|
||||
fitOnDeviceOnly,
|
||||
gpu,
|
||||
inferenceGpu,
|
||||
],
|
||||
);
|
||||
const feedRows = useMemo(() => {
|
||||
|
|
@ -1254,9 +1260,11 @@ export function ModelsPage() {
|
|||
loadingPhase: loadProgress?.phase,
|
||||
minMemory,
|
||||
vramInfo,
|
||||
gpuGb: gpu.available ? gpu.memoryTotalGb : undefined,
|
||||
gpuGb: inferenceGpu.available ? inferenceGpu.memoryTotalGb : undefined,
|
||||
systemRamGb:
|
||||
gpu.systemRamAvailableGb > 0 ? gpu.systemRamAvailableGb : undefined,
|
||||
inferenceGpu.systemRamAvailableGb > 0
|
||||
? inferenceGpu.systemRamAvailableGb
|
||||
: undefined,
|
||||
}),
|
||||
[
|
||||
isActive,
|
||||
|
|
@ -1265,9 +1273,9 @@ export function ModelsPage() {
|
|||
loadProgress?.phase,
|
||||
minMemory,
|
||||
vramInfo,
|
||||
gpu.available,
|
||||
gpu.memoryTotalGb,
|
||||
gpu.systemRamAvailableGb,
|
||||
inferenceGpu.available,
|
||||
inferenceGpu.memoryTotalGb,
|
||||
inferenceGpu.systemRamAvailableGb,
|
||||
],
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ import {
|
|||
useHfTokenStore,
|
||||
useOnlineStatus,
|
||||
} from "@/features/hub";
|
||||
import { useDebouncedValue, useGpuInfo } from "@/hooks";
|
||||
import { useDebouncedValue, useGpuInfo, useInferenceGpuInfo } from "@/hooks";
|
||||
import { extractParamLabel } from "@/lib/model-size";
|
||||
import { toast } from "@/lib/toast";
|
||||
import { cn, formatCompact } from "@/lib/utils";
|
||||
|
|
@ -720,6 +720,7 @@ function GgufVariantExpander({
|
|||
onSelect,
|
||||
gpuGb,
|
||||
systemRamGb,
|
||||
budgetKnown = false,
|
||||
hfToken,
|
||||
parentOptionKey,
|
||||
onNavigatePastStart,
|
||||
|
|
@ -735,6 +736,7 @@ function GgufVariantExpander({
|
|||
onSelect: (id: string, meta: ModelSelectorChangeMeta) => void;
|
||||
gpuGb?: number;
|
||||
systemRamGb?: number;
|
||||
budgetKnown?: boolean;
|
||||
/** HF token threaded into the variant fetch so private/gated repos resolve
|
||||
* their GGUF variants (and update badges). */
|
||||
hfToken?: string;
|
||||
|
|
@ -854,8 +856,9 @@ function GgufVariantExpander({
|
|||
|
||||
const getGgufFit = useCallback(
|
||||
(sizeBytes: number): "fits" | "tight" | "oom" => {
|
||||
// No device budget at all: can't classify, so don't show OOM badges.
|
||||
if (totalBudgetGb <= 0) return "fits";
|
||||
// Preserve permissive behavior only when no budget was measured. A known
|
||||
// zero Vulkan budget means every non-empty variant is OOM.
|
||||
if (totalBudgetGb <= 0) return budgetKnown ? "oom" : "fits";
|
||||
const gb = sizeBytes / 1024 ** 3;
|
||||
if (gb <= 0 || gb <= gpuBudgetGb) return "fits";
|
||||
// No-GPU / unified-memory hosts (Mac) have only the RAM budget, so the tier
|
||||
|
|
@ -864,13 +867,17 @@ function GgufVariantExpander({
|
|||
if (gb <= totalBudgetGb) return "tight";
|
||||
return "oom";
|
||||
},
|
||||
[gpuBudgetGb, totalBudgetGb],
|
||||
[budgetKnown, gpuBudgetGb, totalBudgetGb],
|
||||
);
|
||||
|
||||
// If the recommended variant is OOM, pick the largest fitting one;
|
||||
// if all are OOM, recommend the smallest.
|
||||
const effectiveRecommended = useMemo(() => {
|
||||
if (!variants || variants.length === 0 || totalBudgetGb <= 0) {
|
||||
if (
|
||||
!variants ||
|
||||
variants.length === 0 ||
|
||||
(totalBudgetGb <= 0 && !budgetKnown)
|
||||
) {
|
||||
return defaultVariant;
|
||||
}
|
||||
const defaultV = variants.find((v) => v.quant === defaultVariant);
|
||||
|
|
@ -885,7 +892,7 @@ function GgufVariantExpander({
|
|||
// All OOM -- recommend smallest (most likely to partially run)
|
||||
const sorted = [...variants].sort((a, b) => a.size_bytes - b.size_bytes);
|
||||
return sorted[0]?.quant ?? defaultVariant;
|
||||
}, [variants, defaultVariant, totalBudgetGb, getGgufFit]);
|
||||
}, [variants, defaultVariant, totalBudgetGb, budgetKnown, getGgufFit]);
|
||||
|
||||
const sortedVariants = useMemo(() => {
|
||||
if (!variants) return variants;
|
||||
|
|
@ -1396,6 +1403,7 @@ export function HubModelPicker({
|
|||
onEject?: () => void;
|
||||
}) {
|
||||
const gpu = useGpuInfo();
|
||||
const inferenceGpu = useInferenceGpuInfo();
|
||||
// Live model id from the runtime store (backend-mirrored active_model), not the dropdown
|
||||
// highlight which can be a staged pick. Disables the update action for it.
|
||||
const loadedModelId = useChatRuntimeStore((s) => s.params.checkpoint);
|
||||
|
|
@ -1854,7 +1862,7 @@ export function HubModelPicker({
|
|||
return rows.filter((r) => {
|
||||
// Downloaded models always show, regardless of device fit.
|
||||
if (downloadedSet.has(r.id.toLowerCase())) return true;
|
||||
return hfModelFitsDevice(r, gpu);
|
||||
return hfModelFitsDevice(r, r.isGguf ? inferenceGpu : gpu);
|
||||
});
|
||||
}, [
|
||||
recommendedSearch.results,
|
||||
|
|
@ -1864,6 +1872,7 @@ export function HubModelPicker({
|
|||
formatFilter,
|
||||
isMac,
|
||||
gpu,
|
||||
inferenceGpu,
|
||||
isChatSupported,
|
||||
]);
|
||||
|
||||
|
|
@ -1904,14 +1913,17 @@ export function HubModelPicker({
|
|||
r.estimatedSizeBytes ??
|
||||
(params ? estimateQuantBytes(params) : undefined);
|
||||
const hasDeviceBudget =
|
||||
gpu.memoryTotalGb > 0 || gpu.systemRamAvailableGb > 0;
|
||||
inferenceGpu.budgetKnown ||
|
||||
inferenceGpu.memoryTotalGb > 0 ||
|
||||
inferenceGpu.systemRamAvailableGb > 0;
|
||||
const exceeds =
|
||||
hasDeviceBudget &&
|
||||
sizeBytes != null &&
|
||||
!fitsDevice({
|
||||
sizeBytes,
|
||||
gpuGb: gpu.memoryTotalGb,
|
||||
systemRamGb: gpu.systemRamAvailableGb,
|
||||
gpuGb: inferenceGpu.memoryTotalGb,
|
||||
systemRamGb: inferenceGpu.systemRamAvailableGb,
|
||||
budgetKnown: inferenceGpu.budgetKnown,
|
||||
});
|
||||
map.set(r.id, {
|
||||
meta,
|
||||
|
|
@ -1928,7 +1940,7 @@ export function HubModelPicker({
|
|||
map.set(r.id, { meta, status, est });
|
||||
}
|
||||
return map;
|
||||
}, [recommendedSearch.results, isKnownGgufRepo, gpu]);
|
||||
}, [recommendedSearch.results, isKnownGgufRepo, gpu, inferenceGpu]);
|
||||
|
||||
// Tag-accurate capabilities keyed by repo id, pooled from both HF listings.
|
||||
// Rows look it up by id and fall back to name detection when absent.
|
||||
|
|
@ -2249,7 +2261,7 @@ export function HubModelPicker({
|
|||
totalParams: recommendedParamCountById.get(id),
|
||||
isGguf: isKnownGgufRepo(id),
|
||||
},
|
||||
gpu,
|
||||
isKnownGgufRepo(id) ? inferenceGpu : gpu,
|
||||
),
|
||||
)
|
||||
);
|
||||
|
|
@ -2263,6 +2275,7 @@ export function HubModelPicker({
|
|||
downloadedSet,
|
||||
recommendedParamCountById,
|
||||
gpu,
|
||||
inferenceGpu,
|
||||
]);
|
||||
|
||||
const recommendedSet = useMemo(
|
||||
|
|
@ -2280,7 +2293,7 @@ export function HubModelPicker({
|
|||
(r) =>
|
||||
!fitOnDeviceOnly ||
|
||||
downloadedSet.has(r.id.toLowerCase()) ||
|
||||
hfModelFitsDevice(r, gpu),
|
||||
hfModelFitsDevice(r, r.isGguf ? inferenceGpu : gpu),
|
||||
)
|
||||
.map((result) => result.id)
|
||||
.filter((id) => !isHiddenModelId(id))
|
||||
|
|
@ -2309,6 +2322,7 @@ export function HubModelPicker({
|
|||
fitOnDeviceOnly,
|
||||
downloadedSet,
|
||||
gpu,
|
||||
inferenceGpu,
|
||||
isMac,
|
||||
]);
|
||||
|
||||
|
|
@ -2905,8 +2919,9 @@ export function HubModelPicker({
|
|||
parentOptionKey={optionKey}
|
||||
onNavigatePastStart={() => hubModelList.focusOption(optionKey)}
|
||||
onNavigatePastEnd={() => hubModelList.moveFocus(optionKey, "next")}
|
||||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
systemRamGb={gpu.systemRamAvailableGb || undefined}
|
||||
gpuGb={inferenceGpu.available ? inferenceGpu.memoryTotalGb : undefined}
|
||||
systemRamGb={inferenceGpu.systemRamAvailableGb || undefined}
|
||||
budgetKnown={inferenceGpu.budgetKnown}
|
||||
variantActions={{
|
||||
onUpdate: (quant, expectedBytes) =>
|
||||
updateGgufVariant(c.repo_id, quant, expectedBytes),
|
||||
|
|
@ -3364,7 +3379,7 @@ export function HubModelPicker({
|
|||
loraModelList={hubModelList}
|
||||
expandedGguf={expandedGguf}
|
||||
setExpandedGguf={setExpandedGguf}
|
||||
gpu={gpu}
|
||||
gpu={inferenceGpu}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
|
|
@ -3691,13 +3706,14 @@ export function HubModelPicker({
|
|||
hubModelList.moveFocus(optionKey, "next")
|
||||
}
|
||||
gpuGb={
|
||||
gpu.available
|
||||
? gpu.memoryTotalGb
|
||||
inferenceGpu.available
|
||||
? inferenceGpu.memoryTotalGb
|
||||
: undefined
|
||||
}
|
||||
systemRamGb={
|
||||
gpu.systemRamAvailableGb || undefined
|
||||
inferenceGpu.systemRamAvailableGb || undefined
|
||||
}
|
||||
budgetKnown={inferenceGpu.budgetKnown}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
|
@ -3816,11 +3832,14 @@ export function HubModelPicker({
|
|||
hubModelList.moveFocus(optionKey, "next")
|
||||
}
|
||||
gpuGb={
|
||||
gpu.available ? gpu.memoryTotalGb : undefined
|
||||
inferenceGpu.available
|
||||
? inferenceGpu.memoryTotalGb
|
||||
: undefined
|
||||
}
|
||||
systemRamGb={
|
||||
gpu.systemRamAvailableGb || undefined
|
||||
inferenceGpu.systemRamAvailableGb || undefined
|
||||
}
|
||||
budgetKnown={inferenceGpu.budgetKnown}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
|
@ -3929,11 +3948,14 @@ export function HubModelPicker({
|
|||
hubModelList.moveFocus(optionKey, "next")
|
||||
}
|
||||
gpuGb={
|
||||
gpu.available ? gpu.memoryTotalGb : undefined
|
||||
inferenceGpu.available
|
||||
? inferenceGpu.memoryTotalGb
|
||||
: undefined
|
||||
}
|
||||
systemRamGb={
|
||||
gpu.systemRamAvailableGb || undefined
|
||||
inferenceGpu.systemRamAvailableGb || undefined
|
||||
}
|
||||
budgetKnown={inferenceGpu.budgetKnown}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
|
@ -3997,7 +4019,13 @@ export function HubModelPicker({
|
|||
vramStatus={info?.status ?? null}
|
||||
vramEst={info?.est}
|
||||
gpuGb={
|
||||
gpu.available ? gpu.memoryTotalGb : undefined
|
||||
isG
|
||||
? inferenceGpu.available
|
||||
? inferenceGpu.memoryTotalGb
|
||||
: undefined
|
||||
: gpu.available
|
||||
? gpu.memoryTotalGb
|
||||
: undefined
|
||||
}
|
||||
onArrowDownIntoChildren={
|
||||
expandedGguf === id
|
||||
|
|
@ -4019,11 +4047,14 @@ export function HubModelPicker({
|
|||
hubModelList.moveFocus(optionKey, "next")
|
||||
}
|
||||
gpuGb={
|
||||
gpu.available ? gpu.memoryTotalGb : undefined
|
||||
inferenceGpu.available
|
||||
? inferenceGpu.memoryTotalGb
|
||||
: undefined
|
||||
}
|
||||
systemRamGb={
|
||||
gpu.systemRamAvailableGb || undefined
|
||||
inferenceGpu.systemRamAvailableGb || undefined
|
||||
}
|
||||
budgetKnown={inferenceGpu.budgetKnown}
|
||||
variantActions={{
|
||||
onDelete: async (quant) => {
|
||||
await deleteCachedModel(
|
||||
|
|
@ -4102,7 +4133,13 @@ export function HubModelPicker({
|
|||
isKnownGgufRepo(id) ? undefined : vram?.est
|
||||
}
|
||||
gpuGb={
|
||||
gpu.available ? gpu.memoryTotalGb : undefined
|
||||
isKnownGgufRepo(id)
|
||||
? inferenceGpu.available
|
||||
? inferenceGpu.memoryTotalGb
|
||||
: undefined
|
||||
: gpu.available
|
||||
? gpu.memoryTotalGb
|
||||
: undefined
|
||||
}
|
||||
onArrowDownIntoChildren={
|
||||
expandedGguf === id
|
||||
|
|
@ -4128,11 +4165,14 @@ export function HubModelPicker({
|
|||
hubModelList.moveFocus(optionKey, "next")
|
||||
}
|
||||
gpuGb={
|
||||
gpu.available ? gpu.memoryTotalGb : undefined
|
||||
inferenceGpu.available
|
||||
? inferenceGpu.memoryTotalGb
|
||||
: undefined
|
||||
}
|
||||
systemRamGb={
|
||||
gpu.systemRamAvailableGb || undefined
|
||||
inferenceGpu.systemRamAvailableGb || undefined
|
||||
}
|
||||
budgetKnown={inferenceGpu.budgetKnown}
|
||||
variantActions={{
|
||||
onDelete: async (quant) => {
|
||||
await deleteCachedModel(
|
||||
|
|
@ -4207,7 +4247,13 @@ export function HubModelPicker({
|
|||
}
|
||||
vramEst={isSearchGguf ? undefined : vram?.est}
|
||||
gpuGb={
|
||||
gpu.available ? gpu.memoryTotalGb : undefined
|
||||
isSearchGguf
|
||||
? inferenceGpu.available
|
||||
? inferenceGpu.memoryTotalGb
|
||||
: undefined
|
||||
: gpu.available
|
||||
? gpu.memoryTotalGb
|
||||
: undefined
|
||||
}
|
||||
onArrowDownIntoChildren={
|
||||
expandedGguf === id
|
||||
|
|
@ -4233,11 +4279,14 @@ export function HubModelPicker({
|
|||
hubModelList.moveFocus(optionKey, "next")
|
||||
}
|
||||
gpuGb={
|
||||
gpu.available ? gpu.memoryTotalGb : undefined
|
||||
inferenceGpu.available
|
||||
? inferenceGpu.memoryTotalGb
|
||||
: undefined
|
||||
}
|
||||
systemRamGb={
|
||||
gpu.systemRamAvailableGb || undefined
|
||||
inferenceGpu.systemRamAvailableGb || undefined
|
||||
}
|
||||
budgetKnown={inferenceGpu.budgetKnown}
|
||||
variantActions={{
|
||||
onDelete: async (quant) => {
|
||||
await deleteCachedModel(
|
||||
|
|
@ -4320,6 +4369,7 @@ function FineTunedRows({
|
|||
setExpandedGguf: Dispatch<SetStateAction<string | null>>;
|
||||
gpu: {
|
||||
available: boolean;
|
||||
budgetKnown: boolean;
|
||||
memoryTotalGb: number;
|
||||
systemRamAvailableGb: number;
|
||||
};
|
||||
|
|
@ -4456,6 +4506,7 @@ function FineTunedRows({
|
|||
}
|
||||
gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
|
||||
systemRamGb={gpu.systemRamAvailableGb || undefined}
|
||||
budgetKnown={gpu.budgetKnown}
|
||||
sourceOverride={isExportedGguf ? "exported" : undefined}
|
||||
variantActions={{
|
||||
deleteTitle: "Delete exported GGUF variant?",
|
||||
|
|
|
|||
|
|
@ -97,13 +97,21 @@ export function fitsDevice(opts: {
|
|||
estimatedVramGb?: number;
|
||||
gpuGb?: number;
|
||||
systemRamGb?: number;
|
||||
budgetKnown?: boolean;
|
||||
requireKnown?: boolean;
|
||||
}): boolean {
|
||||
const { sizeBytes, estimatedVramGb, gpuGb, systemRamGb, requireKnown } = opts;
|
||||
const {
|
||||
sizeBytes,
|
||||
estimatedVramGb,
|
||||
gpuGb,
|
||||
systemRamGb,
|
||||
budgetKnown,
|
||||
requireKnown,
|
||||
} = opts;
|
||||
// Unified-memory hosts (Mac / no discrete GPU) report system RAM but no GPU,
|
||||
// so the budget must include RAM. Only an entirely unknown budget fits freely.
|
||||
const budgetGb = Math.max(0, gpuGb ?? 0) * 0.7 + Math.max(0, systemRamGb ?? 0) * 0.7;
|
||||
if (budgetGb <= 0) return true;
|
||||
if (budgetGb <= 0) return !budgetKnown;
|
||||
if (sizeBytes && sizeBytes > 0) {
|
||||
return sizeBytes / 1024 ** 3 <= budgetGb;
|
||||
}
|
||||
|
|
@ -129,9 +137,18 @@ export function hfModelFitsDevice(
|
|||
estimatedSizeBytes?: number;
|
||||
isGguf?: boolean;
|
||||
},
|
||||
gpu: { memoryTotalGb: number; systemRamAvailableGb: number },
|
||||
gpu: {
|
||||
memoryTotalGb: number;
|
||||
systemRamAvailableGb: number;
|
||||
budgetKnown?: boolean;
|
||||
},
|
||||
): boolean {
|
||||
if (gpu.memoryTotalGb <= 0 && gpu.systemRamAvailableGb <= 0) return true;
|
||||
if (
|
||||
gpu.memoryTotalGb <= 0 &&
|
||||
gpu.systemRamAvailableGb <= 0 &&
|
||||
!gpu.budgetKnown
|
||||
)
|
||||
return true;
|
||||
const params = model.totalParams ?? paramsFromId(model.id);
|
||||
const quantBytes = params ? estimateQuantBytes(params) : undefined;
|
||||
const sizeBytes = isGgufId(model.id, model.isGguf)
|
||||
|
|
@ -141,6 +158,7 @@ export function hfModelFitsDevice(
|
|||
sizeBytes,
|
||||
gpuGb: gpu.memoryTotalGb,
|
||||
systemRamGb: gpu.systemRamAvailableGb,
|
||||
budgetKnown: gpu.budgetKnown,
|
||||
requireKnown: true,
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,11 @@ import {
|
|||
openModelsDir,
|
||||
pickHuggingFaceCacheDir,
|
||||
} from "@/features/native-intents";
|
||||
import { useSystemInfo, type GpuDevice } from "@/hooks/use-system";
|
||||
import {
|
||||
aggregateGpuMemoryTotalGb,
|
||||
useSystemInfo,
|
||||
type GpuDevice,
|
||||
} from "@/hooks/use-system";
|
||||
import { isTauri } from "@/lib/api-base";
|
||||
import { copyToClipboard } from "@/lib/copy-to-clipboard";
|
||||
import { toast } from "@/lib/toast";
|
||||
|
|
@ -184,6 +188,18 @@ export function ResourcesTab() {
|
|||
const [hfCacheLoaded, setHfCacheLoaded] = useState(false);
|
||||
const [cacheBrowserOpen, setCacheBrowserOpen] = useState(false);
|
||||
const [cacheSaving, setCacheSaving] = useState(false);
|
||||
const displayedGpu = systemInfo.gpu?.available
|
||||
? systemInfo.gpu
|
||||
: (systemInfo.inference_gpu ?? systemInfo.gpu);
|
||||
const separateInferenceGpu =
|
||||
systemInfo.gpu?.available &&
|
||||
systemInfo.inference_gpu &&
|
||||
systemInfo.inference_gpu.backend !== systemInfo.gpu.backend
|
||||
? systemInfo.inference_gpu
|
||||
: null;
|
||||
const inferenceVramTotal = separateInferenceGpu
|
||||
? aggregateGpuMemoryTotalGb(separateInferenceGpu.devices)
|
||||
: 0;
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
|
|
@ -203,17 +219,14 @@ export function ResourcesTab() {
|
|||
}, []);
|
||||
|
||||
const metrics = useMemo(() => {
|
||||
const devices = systemInfo.gpu?.devices ?? [];
|
||||
const devices = displayedGpu?.devices ?? [];
|
||||
const ramTotal = systemInfo.memory?.total_gb ?? 0;
|
||||
const ramAvailable = systemInfo.memory?.available_gb ?? 0;
|
||||
const ramUsed = Math.max(0, ramTotal - ramAvailable);
|
||||
const diskTotal = systemInfo.disk?.total_gb ?? 0;
|
||||
const diskFree = systemInfo.disk?.free_gb ?? 0;
|
||||
const diskUsed = Math.max(0, diskTotal - diskFree);
|
||||
const vramTotal = devices.reduce(
|
||||
(sum, device) => sum + (device.memory_total_gb ?? 0),
|
||||
0,
|
||||
);
|
||||
const vramTotal = aggregateGpuMemoryTotalGb(devices);
|
||||
// null usage = unknown (e.g. Windows ROCm perf counter): treating it as 0
|
||||
// fabricates a 0-used total, so the aggregate is unknown if any device is.
|
||||
const vramUsageKnown =
|
||||
|
|
@ -252,7 +265,7 @@ export function ResourcesTab() {
|
|||
vramPercent,
|
||||
vramUsageKnown,
|
||||
};
|
||||
}, [systemInfo]);
|
||||
}, [displayedGpu, systemInfo]);
|
||||
|
||||
const handleCacheFolder = async () => {
|
||||
if (!hfCache) return;
|
||||
|
|
@ -312,9 +325,9 @@ export function ResourcesTab() {
|
|||
: t("settings.resources.environment.unknown");
|
||||
const cpuFrequencyLabel = formatFrequency(systemInfo.cpu?.frequency_mhz);
|
||||
const hasGpu =
|
||||
(systemInfo.gpu?.available ?? false) && metrics.devices.length > 0;
|
||||
(displayedGpu?.available ?? false) && metrics.devices.length > 0;
|
||||
const backendLabel = (
|
||||
systemInfo.gpu?.backend ?? systemInfo.device_backend ?? "cpu"
|
||||
displayedGpu?.backend ?? systemInfo.device_backend ?? "cpu"
|
||||
).toUpperCase();
|
||||
const modelsFolderPath = hfCache
|
||||
? hfCache.cacheHome
|
||||
|
|
@ -426,6 +439,19 @@ export function ResourcesTab() {
|
|||
</SettingsSection>
|
||||
|
||||
<SettingsSection title={t("settings.resources.gpu.title")}>
|
||||
{separateInferenceGpu && (
|
||||
<div className="flex items-center justify-between gap-4 border-b border-border/60 py-3 text-sm">
|
||||
<span className="text-muted-foreground">GGUF inference</span>
|
||||
<span className="text-right font-mono text-xs uppercase text-foreground">
|
||||
{separateInferenceGpu.backend ?? "GPU"}
|
||||
{separateInferenceGpu.available
|
||||
? inferenceVramTotal
|
||||
? ` · ${formatGiB(inferenceVramTotal)}`
|
||||
: ""
|
||||
: " · unavailable"}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
{hasGpu ? (
|
||||
metrics.devices.map((device, index) => {
|
||||
const ordinal = deviceOrdinal(device);
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
export { useDebouncedValue } from "./use-debounced-value";
|
||||
export { useGpuInfo } from "./use-gpu-info";
|
||||
export { useGpuInfo, useInferenceGpuInfo } from "./use-gpu-info";
|
||||
export { useGpuUtilization } from "./use-gpu-utilization";
|
||||
export { useHardwareInfo } from "./use-hardware-info";
|
||||
export { useHfDatasetSplits } from "./use-hf-dataset-splits";
|
||||
|
|
|
|||
|
|
@ -3,10 +3,14 @@
|
|||
|
||||
import { authFetch } from "@/features/auth";
|
||||
import { useEffect, useState } from "react";
|
||||
import type { SystemInfoResponse } from "./use-system";
|
||||
import {
|
||||
aggregateGpuMemoryTotalGb,
|
||||
type SystemInfoResponse,
|
||||
} from "./use-system";
|
||||
|
||||
export interface GpuInfo {
|
||||
available: boolean;
|
||||
budgetKnown: boolean;
|
||||
name: string;
|
||||
memoryTotalGb: number;
|
||||
cpuCore: number;
|
||||
|
|
@ -30,6 +34,7 @@ export interface SystemGpuDevice {
|
|||
|
||||
const DEFAULT_GPU: GpuInfo = {
|
||||
available: false,
|
||||
budgetKnown: false,
|
||||
name: "Unknown",
|
||||
memoryTotalGb: 0,
|
||||
cpuCore: 0,
|
||||
|
|
@ -42,8 +47,8 @@ const DEFAULT_GPU: GpuInfo = {
|
|||
let cachedSystem: SystemInfoResponse | null = null;
|
||||
let systemPromise: Promise<SystemInfoResponse | null> | null = null;
|
||||
|
||||
async function fetchSystemOnce(): Promise<SystemInfoResponse | null> {
|
||||
if (cachedSystem) return cachedSystem;
|
||||
async function fetchSystemOnce(force = false): Promise<SystemInfoResponse | null> {
|
||||
if (!force && cachedSystem) return cachedSystem;
|
||||
if (systemPromise) return systemPromise;
|
||||
systemPromise = (async () => {
|
||||
try {
|
||||
|
|
@ -52,14 +57,18 @@ async function fetchSystemOnce(): Promise<SystemInfoResponse | null> {
|
|||
cachedSystem = (await res.json()) as SystemInfoResponse;
|
||||
return cachedSystem;
|
||||
} catch {
|
||||
systemPromise = null; // reset so a later call retries (backend not ready)
|
||||
return null;
|
||||
} finally {
|
||||
systemPromise = null;
|
||||
}
|
||||
})();
|
||||
return systemPromise;
|
||||
}
|
||||
|
||||
function toGpuInfo(data: SystemInfoResponse | null): GpuInfo {
|
||||
function toGpuInfo(
|
||||
data: SystemInfoResponse | null,
|
||||
source: "gpu" | "inference_gpu" = "gpu",
|
||||
): GpuInfo {
|
||||
// CPU/RAM exist even on GPU-less hosts (e.g. Mac), so populate them on every
|
||||
// path: unified-memory math still needs a RAM budget to work with.
|
||||
const base = {
|
||||
|
|
@ -68,16 +77,25 @@ function toGpuInfo(data: SystemInfoResponse | null): GpuInfo {
|
|||
systemRamAvailableGb: data?.memory?.available_gb ?? 0,
|
||||
systemRamTotalGb: data?.memory?.total_gb ?? 0,
|
||||
};
|
||||
const gpuData = data?.gpu;
|
||||
const gpuData =
|
||||
source === "inference_gpu"
|
||||
? (data?.inference_gpu ?? data?.gpu)
|
||||
: data?.gpu;
|
||||
const devices = gpuData?.devices ?? [];
|
||||
if (!gpuData?.available || !devices.length) {
|
||||
return { ...DEFAULT_GPU, ...base };
|
||||
return { ...DEFAULT_GPU, ...base, budgetKnown: data !== null };
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
// A Vulkan iGPU's reported budget is capped shared system RAM, not an
|
||||
// independent VRAM pool. Do not offer the same RAM again for CPU offload.
|
||||
systemRamAvailableGb: devices.some((device) => device.shared_memory)
|
||||
? 0
|
||||
: base.systemRamAvailableGb,
|
||||
available: true,
|
||||
budgetKnown: true,
|
||||
name: devices[0]?.name ?? "Unknown",
|
||||
memoryTotalGb: devices.reduce((sum, d) => sum + (d.memory_total_gb ?? 0), 0),
|
||||
memoryTotalGb: aggregateGpuMemoryTotalGb(devices),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -104,24 +122,56 @@ function toGpuDevices(data: SystemInfoResponse | null): SystemGpuDevice[] {
|
|||
}
|
||||
|
||||
/** Aggregate GPU info from /api/system; shares one module-level fetch across all GPU hooks. */
|
||||
export function useGpuInfo(): GpuInfo {
|
||||
function useGpuInfoSource(source: "gpu" | "inference_gpu"): GpuInfo {
|
||||
const [gpu, setGpu] = useState<GpuInfo>(
|
||||
cachedSystem ? toGpuInfo(cachedSystem) : DEFAULT_GPU,
|
||||
cachedSystem ? toGpuInfo(cachedSystem, source) : DEFAULT_GPU,
|
||||
);
|
||||
useEffect(() => {
|
||||
// No early return on cachedSystem: a consumer mounting as the cache fills
|
||||
// (between render and effect) would otherwise stay stuck at the default.
|
||||
let cancelled = false;
|
||||
fetchSystemOnce().then((d) => {
|
||||
if (!cancelled) setGpu(toGpuInfo(d));
|
||||
});
|
||||
let retryId: number | undefined;
|
||||
const update = (force = false, retryVulkan = false) => {
|
||||
fetchSystemOnce(force).then((d) => {
|
||||
if (cancelled) return;
|
||||
if (!d) {
|
||||
// Once an unavailable Vulkan backend starts polling, a transient API
|
||||
// failure must preserve the current state and continue the same loop.
|
||||
if (retryVulkan) {
|
||||
retryId = window.setTimeout(() => update(true, true), 3000);
|
||||
}
|
||||
return;
|
||||
}
|
||||
setGpu(toGpuInfo(d, source));
|
||||
const inferenceGpu = d.inference_gpu;
|
||||
if (
|
||||
source === "inference_gpu" &&
|
||||
inferenceGpu?.backend === "vulkan" &&
|
||||
!inferenceGpu.available
|
||||
) {
|
||||
retryId = window.setTimeout(() => update(true, true), 3000);
|
||||
}
|
||||
});
|
||||
};
|
||||
update();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
if (retryId !== undefined) window.clearTimeout(retryId);
|
||||
};
|
||||
}, []);
|
||||
}, [source]);
|
||||
return gpu;
|
||||
}
|
||||
|
||||
/** Training-capable GPU info from the PyTorch/MLX hardware detector. */
|
||||
export function useGpuInfo(): GpuInfo {
|
||||
return useGpuInfoSource("gpu");
|
||||
}
|
||||
|
||||
/** GGUF inference GPU info, including a separately installed Vulkan backend. */
|
||||
export function useInferenceGpuInfo(): GpuInfo {
|
||||
return useGpuInfoSource("inference_gpu");
|
||||
}
|
||||
|
||||
/** All backend-visible GPUs (index, name, total VRAM); shares the same fetch. */
|
||||
export function useGpuDevices(): SystemGpuDevice[] {
|
||||
const [devices, setDevices] = useState<SystemGpuDevice[]>(
|
||||
|
|
|
|||
|
|
@ -13,6 +13,33 @@ export interface GpuDevice {
|
|||
vram_used_gb?: number;
|
||||
vram_free_gb?: number;
|
||||
vram_utilization_pct?: number | null;
|
||||
/** True when the reported GPU budget comes from shared system memory. */
|
||||
shared_memory?: boolean;
|
||||
}
|
||||
|
||||
export interface SystemGpuInfo {
|
||||
available: boolean;
|
||||
backend?: string;
|
||||
/** Whether GGUF loads accept explicit physical GPU IDs. */
|
||||
gguf_gpu_ids_supported?: boolean;
|
||||
backend_cuda_visible_devices?: string | null;
|
||||
parent_visible_gpu_ids?: number[];
|
||||
index_kind?: string;
|
||||
devices: GpuDevice[];
|
||||
}
|
||||
|
||||
/** Sum dedicated VRAM while counting a shared host-memory pool only once. */
|
||||
export function aggregateGpuMemoryTotalGb(devices: GpuDevice[]): number {
|
||||
const dedicated = devices
|
||||
.filter((device) => !device.shared_memory)
|
||||
.reduce((sum, device) => sum + (device.memory_total_gb ?? 0), 0);
|
||||
const shared = Math.max(
|
||||
0,
|
||||
...devices
|
||||
.filter((device) => device.shared_memory)
|
||||
.map((device) => device.memory_total_gb ?? 0),
|
||||
);
|
||||
return dedicated + shared;
|
||||
}
|
||||
|
||||
export interface SystemInfoResponse {
|
||||
|
|
@ -37,17 +64,9 @@ export interface SystemInfoResponse {
|
|||
free_gb: number;
|
||||
percent_used: number;
|
||||
};
|
||||
gpu: {
|
||||
available: boolean;
|
||||
backend?: string;
|
||||
/** Whether GGUF loads accept an explicit gpu_ids pick (false on XPU hosts
|
||||
* and Vulkan-only builds, where /load and /validate 400 picks). */
|
||||
gguf_gpu_ids_supported?: boolean;
|
||||
backend_cuda_visible_devices?: string | null;
|
||||
parent_visible_gpu_ids?: number[];
|
||||
index_kind?: string;
|
||||
devices: GpuDevice[];
|
||||
};
|
||||
gpu: SystemGpuInfo;
|
||||
/** Devices available to GGUF inference; differs when llama.cpp uses Vulkan. */
|
||||
inference_gpu?: SystemGpuInfo;
|
||||
ml_packages: {
|
||||
torch?: string;
|
||||
transformers?: string;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue