From 88cf376dc4840a9c527cd892b72a04313f74f431 Mon Sep 17 00:00:00 2001 From: LeoBorcherding Date: Fri, 24 Jul 2026 00:55:57 -0500 Subject: [PATCH] studio: stamp GPU index kind into remount key; hold empty Vulkan GGUF budget - gpuFieldsSignature now folds selectedGpuIdsIndexKind into the pick, so when the reactive fallback stamps a cold-hydrated active pick after the GPU cache warms, an open Run-settings panel remounts and re-snapshots with the new stamp instead of reloading/saving it as a legacy physical pick. - /api/system reports gguf_backend_is_vulkan so the frontend can tell an empty gguf_devices on a Vulkan build (probe failed/masked, budget unknown) from a non-Vulkan build. Empty Vulkan inventory now budgets GGUF at 0 instead of the torch VRAM total, which on a mixed host would let fit checks pass against VRAM /load cannot place. --- studio/backend/main.py | 12 +++++++--- .../backend/tests/test_system_gguf_devices.py | 8 +++++++ .../model-config/apply-per-model-config.ts | 10 ++++++++- studio/frontend/src/hooks/use-gpu-info.ts | 22 +++++++++++++++---- studio/frontend/src/hooks/use-system.ts | 5 +++++ 5 files changed, 49 insertions(+), 8 deletions(-) diff --git a/studio/backend/main.py b/studio/backend/main.py index 730321711c..d4ed5090c3 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -1238,16 +1238,17 @@ def _get_cached_system_gpu_info(logger) -> dict[str, Any]: # enumerated devices (gguf_devices above): the pick then lives in the # same ggml ordinal space `--device Vulkan` pins. Without that # inventory the frontend has no valid ordinals to offer. + is_vulkan_build = False try: from core.inference.llama_cpp import LlamaCppBackend from utils.hardware import DeviceType, get_device - + is_vulkan_build = LlamaCppBackend._is_vulkan_backend() # Check the Vulkan build first: its picks live in ggml's own ordinal # space (--device Vulkan) and don't rely on torch-xpu ordinals, so # they're valid even on an Intel/XPU host. Only fall through to the # XPU ban for a non-Vulkan build (where a pick would need torch-xpu # ordinals no visibility mask can speak). - if LlamaCppBackend._is_vulkan_backend(): + if is_vulkan_build: gpu_ids_supported = bool(gguf_devices) elif get_device() == DeviceType.XPU: gpu_ids_supported = False @@ -1258,12 +1259,17 @@ def _get_cached_system_gpu_info(logger) -> dict[str, Any]: gpu_ids_supported = True # `available` stays the torch view: training consumers key GPU labels on # it, and a Vulkan-only inventory doesn't make training GPU-capable. - # GGUF surfaces key on gguf_devices instead. + # GGUF surfaces key on gguf_devices instead. gguf_backend_is_vulkan lets + # the frontend tell an empty gguf_devices on a Vulkan build (probe failed + # / masked to nothing -- GGUF budget is unknown, must NOT reuse the torch + # VRAM total) apart from a non-Vulkan build (where llama-server does run + # on the torch devices, so that total is the right GGUF budget). gpu_info = { "available": visibility_info.get("available", False), "devices": enriched_devices, "gguf_devices": gguf_devices, "gguf_gpu_ids_supported": gpu_ids_supported, + "gguf_backend_is_vulkan": is_vulkan_build, } _system_gpu_cache = (time.monotonic(), gpu_info) return gpu_info diff --git a/studio/backend/tests/test_system_gguf_devices.py b/studio/backend/tests/test_system_gguf_devices.py index 2313ae54ab..561d4328c0 100644 --- a/studio/backend/tests/test_system_gguf_devices.py +++ b/studio/backend/tests/test_system_gguf_devices.py @@ -99,6 +99,7 @@ def test_vulkan_build_surfaces_llama_server_devices(main_module, monkeypatch): # Picks are valid Vulkan ordinals now, so the picker may offer them. assert info["gguf_gpu_ids_supported"] is True + assert info["gguf_backend_is_vulkan"] is True def test_vulkan_build_with_failed_probe_keeps_picks_unsupported(main_module, monkeypatch): @@ -106,6 +107,10 @@ def test_vulkan_build_with_failed_probe_keeps_picks_unsupported(main_module, mon assert info["gguf_devices"] == [] # No enumerable ordinal space -> the frontend has no valid picks to offer. assert info["gguf_gpu_ids_supported"] is False + # ...but the frontend must still know this is a Vulkan build so it treats the + # empty inventory as an unknown GGUF budget (0) rather than reusing the torch + # VRAM total, which on a mixed host would overclaim VRAM /load can't place. + assert info["gguf_backend_is_vulkan"] is True def test_vulkan_inventory_survives_gpu_cache_refreshes_without_reprobing(main_module, monkeypatch): @@ -147,6 +152,9 @@ def test_non_vulkan_build_reports_no_gguf_inventory(main_module, monkeypatch): # CUDA/ROCm llama builds see the same devices torch does; no separate list. assert info["gguf_devices"] == [] assert info["gguf_gpu_ids_supported"] is True + # Not a Vulkan build: the frontend keeps budgeting GGUF against the torch + # total, since llama-server runs on those same devices. + assert info["gguf_backend_is_vulkan"] is False if __name__ == "__main__": diff --git a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts index 3094a50f8c..5e0d24db8c 100644 --- a/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts +++ b/studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts @@ -141,9 +141,17 @@ export function gpuFieldsSignature(config: PerModelConfig): string { config.gpuMemoryMode ?? "auto", config.gpuLayers == null || config.gpuLayers < 0 ? -1 : config.gpuLayers, config.nCpuMoe ?? 0, + // Include the pick's index space (physical vs Vulkan ordinals): the same ids + // mean different cards across a backend swap, so a kind change is a real + // config change. It also drives the SidebarModelConfig remount key, so when + // the reactive fallback stamps a cold-hydrated active pick after the GPU + // cache warms, an open Run-settings panel re-snapshots with the new stamp + // instead of reloading/saving it as a legacy physical pick. config.selectedGpuIds == null ? "all" - : [...config.selectedGpuIds].sort((a, b) => a - b).join(","), + : `${[...config.selectedGpuIds].sort((a, b) => a - b).join(",")}@${ + config.selectedGpuIdsIndexKind ?? "physical" + }`, ].join("|"); } diff --git a/studio/frontend/src/hooks/use-gpu-info.ts b/studio/frontend/src/hooks/use-gpu-info.ts index 2e63cc775b..e1e2c9fb84 100644 --- a/studio/frontend/src/hooks/use-gpu-info.ts +++ b/studio/frontend/src/hooks/use-gpu-info.ts @@ -11,9 +11,12 @@ export interface GpuInfo { memoryTotalGb: number; /** VRAM budget for GGUF/llama-server workloads. On a Vulkan build this sums * the devices llama-server actually uses (gguf_devices), which can include - * cards the torch backend can't see (e.g. a pre-ROCm AMD card); otherwise - * identical to memoryTotalGb. GGUF fit labels must use this; torch-based - * (training / safetensors) estimates must stay on memoryTotalGb. */ + * cards the torch backend can't see (e.g. a pre-ROCm AMD card), and is 0 when + * that inventory is empty (probe failed/masked -- budget unknown, so labels + * stay conservative). On a non-Vulkan build llama-server runs on the torch + * devices, so this is identical to memoryTotalGb. GGUF fit labels must use + * this; torch-based (training / safetensors) estimates must stay on + * memoryTotalGb. */ ggufMemoryTotalGb: number; cpuCore: number; cpuThread: number; @@ -113,6 +116,13 @@ function toGpuInfo(data: SystemInfoResponse | null): GpuInfo { (sum, d) => sum + (d.is_igpu ? 0 : (d.memory_total_gb ?? 0)), 0, ); + // A Vulkan build budgets GGUF against gguf_devices, not the torch view. When + // that inventory is empty (probe failed / masked to no discrete device) the + // GGUF budget is genuinely unknown, so fall back to 0 (labels stay + // conservative) rather than the torch total: on a mixed host torch may still + // see a dGPU llama-server never enumerated, and reusing that total would let + // fit checks pass against VRAM /load can't actually place. + const isVulkanBuild = gpuData?.gguf_backend_is_vulkan === true; if (!gpuData?.available || !devices.length) { // Torch sees no GPU (training stays CPU-bound / unavailable), but a Vulkan // llama.cpp build may still drive GPUs for GGUF: surface that budget alone. @@ -127,7 +137,11 @@ function toGpuInfo(data: SystemInfoResponse | null): GpuInfo { available: true, name: devices[0]?.name ?? "Unknown", memoryTotalGb, - ggufMemoryTotalGb: ggufDevices.length ? ggufDeviceTotalGb : memoryTotalGb, + ggufMemoryTotalGb: ggufDevices.length + ? ggufDeviceTotalGb + : isVulkanBuild + ? 0 + : memoryTotalGb, }; } diff --git a/studio/frontend/src/hooks/use-system.ts b/studio/frontend/src/hooks/use-system.ts index da1c5101d3..e358fad5f6 100644 --- a/studio/frontend/src/hooks/use-system.ts +++ b/studio/frontend/src/hooks/use-system.ts @@ -46,6 +46,11 @@ export interface SystemInfoResponse { * and on Vulkan-only builds only when the device probe found nothing -- * with gguf_devices present, picks are Vulkan ordinals and supported). */ gguf_gpu_ids_supported?: boolean; + /** True when the llama.cpp build is Vulkan. Lets the frontend tell an empty + * gguf_devices on a Vulkan build (probe failed/masked -- GGUF budget + * unknown) apart from a non-Vulkan build (llama-server runs on the torch + * devices, so the torch total is the right GGUF budget). */ + gguf_backend_is_vulkan?: boolean; backend_cuda_visible_devices?: string | null; parent_visible_gpu_ids?: number[]; index_kind?: string;