studio: stamp GPU index kind into remount key; hold empty Vulkan GGUF budget

- gpuFieldsSignature now folds selectedGpuIdsIndexKind into the pick, so when
  the reactive fallback stamps a cold-hydrated active pick after the GPU cache
  warms, an open Run-settings panel remounts and re-snapshots with the new
  stamp instead of reloading/saving it as a legacy physical pick.
- /api/system reports gguf_backend_is_vulkan so the frontend can tell an empty
  gguf_devices on a Vulkan build (probe failed/masked, budget unknown) from a
  non-Vulkan build. Empty Vulkan inventory now budgets GGUF at 0 instead of the
  torch VRAM total, which on a mixed host would let fit checks pass against VRAM
  /load cannot place.
This commit is contained in:
LeoBorcherding 2026-07-24 00:55:57 -05:00
commit 88cf376dc4
5 changed files with 49 additions and 8 deletions

View file

@ -1238,16 +1238,17 @@ def _get_cached_system_gpu_info(logger) -> dict[str, Any]:
# enumerated devices (gguf_devices above): the pick then lives in the
# same ggml ordinal space `--device Vulkan<i>` pins. Without that
# inventory the frontend has no valid ordinals to offer.
is_vulkan_build = False
try:
from core.inference.llama_cpp import LlamaCppBackend
from utils.hardware import DeviceType, get_device
is_vulkan_build = LlamaCppBackend._is_vulkan_backend()
# Check the Vulkan build first: its picks live in ggml's own ordinal
# space (--device Vulkan<i>) and don't rely on torch-xpu ordinals, so
# they're valid even on an Intel/XPU host. Only fall through to the
# XPU ban for a non-Vulkan build (where a pick would need torch-xpu
# ordinals no visibility mask can speak).
if LlamaCppBackend._is_vulkan_backend():
if is_vulkan_build:
gpu_ids_supported = bool(gguf_devices)
elif get_device() == DeviceType.XPU:
gpu_ids_supported = False
@ -1258,12 +1259,17 @@ def _get_cached_system_gpu_info(logger) -> dict[str, Any]:
gpu_ids_supported = True
# `available` stays the torch view: training consumers key GPU labels on
# it, and a Vulkan-only inventory doesn't make training GPU-capable.
# GGUF surfaces key on gguf_devices instead.
# GGUF surfaces key on gguf_devices instead. gguf_backend_is_vulkan lets
# the frontend tell an empty gguf_devices on a Vulkan build (probe failed
# / masked to nothing -- GGUF budget is unknown, must NOT reuse the torch
# VRAM total) apart from a non-Vulkan build (where llama-server does run
# on the torch devices, so that total is the right GGUF budget).
gpu_info = {
"available": visibility_info.get("available", False),
"devices": enriched_devices,
"gguf_devices": gguf_devices,
"gguf_gpu_ids_supported": gpu_ids_supported,
"gguf_backend_is_vulkan": is_vulkan_build,
}
_system_gpu_cache = (time.monotonic(), gpu_info)
return gpu_info

View file

@ -99,6 +99,7 @@ def test_vulkan_build_surfaces_llama_server_devices(main_module, monkeypatch):
# Picks are valid Vulkan ordinals now, so the picker may offer them.
assert info["gguf_gpu_ids_supported"] is True
assert info["gguf_backend_is_vulkan"] is True
def test_vulkan_build_with_failed_probe_keeps_picks_unsupported(main_module, monkeypatch):
@ -106,6 +107,10 @@ def test_vulkan_build_with_failed_probe_keeps_picks_unsupported(main_module, mon
assert info["gguf_devices"] == []
# No enumerable ordinal space -> the frontend has no valid picks to offer.
assert info["gguf_gpu_ids_supported"] is False
# ...but the frontend must still know this is a Vulkan build so it treats the
# empty inventory as an unknown GGUF budget (0) rather than reusing the torch
# VRAM total, which on a mixed host would overclaim VRAM /load can't place.
assert info["gguf_backend_is_vulkan"] is True
def test_vulkan_inventory_survives_gpu_cache_refreshes_without_reprobing(main_module, monkeypatch):
@ -147,6 +152,9 @@ def test_non_vulkan_build_reports_no_gguf_inventory(main_module, monkeypatch):
# CUDA/ROCm llama builds see the same devices torch does; no separate list.
assert info["gguf_devices"] == []
assert info["gguf_gpu_ids_supported"] is True
# Not a Vulkan build: the frontend keeps budgeting GGUF against the torch
# total, since llama-server runs on those same devices.
assert info["gguf_backend_is_vulkan"] is False
if __name__ == "__main__":

View file

@ -141,9 +141,17 @@ export function gpuFieldsSignature(config: PerModelConfig): string {
config.gpuMemoryMode ?? "auto",
config.gpuLayers == null || config.gpuLayers < 0 ? -1 : config.gpuLayers,
config.nCpuMoe ?? 0,
// Include the pick's index space (physical vs Vulkan ordinals): the same ids
// mean different cards across a backend swap, so a kind change is a real
// config change. It also drives the SidebarModelConfig remount key, so when
// the reactive fallback stamps a cold-hydrated active pick after the GPU
// cache warms, an open Run-settings panel re-snapshots with the new stamp
// instead of reloading/saving it as a legacy physical pick.
config.selectedGpuIds == null
? "all"
: [...config.selectedGpuIds].sort((a, b) => a - b).join(","),
: `${[...config.selectedGpuIds].sort((a, b) => a - b).join(",")}@${
config.selectedGpuIdsIndexKind ?? "physical"
}`,
].join("|");
}

View file

@ -11,9 +11,12 @@ export interface GpuInfo {
memoryTotalGb: number;
/** VRAM budget for GGUF/llama-server workloads. On a Vulkan build this sums
* the devices llama-server actually uses (gguf_devices), which can include
* cards the torch backend can't see (e.g. a pre-ROCm AMD card); otherwise
* identical to memoryTotalGb. GGUF fit labels must use this; torch-based
* (training / safetensors) estimates must stay on memoryTotalGb. */
* cards the torch backend can't see (e.g. a pre-ROCm AMD card), and is 0 when
* that inventory is empty (probe failed/masked -- budget unknown, so labels
* stay conservative). On a non-Vulkan build llama-server runs on the torch
* devices, so this is identical to memoryTotalGb. GGUF fit labels must use
* this; torch-based (training / safetensors) estimates must stay on
* memoryTotalGb. */
ggufMemoryTotalGb: number;
cpuCore: number;
cpuThread: number;
@ -113,6 +116,13 @@ function toGpuInfo(data: SystemInfoResponse | null): GpuInfo {
(sum, d) => sum + (d.is_igpu ? 0 : (d.memory_total_gb ?? 0)),
0,
);
// A Vulkan build budgets GGUF against gguf_devices, not the torch view. When
// that inventory is empty (probe failed / masked to no discrete device) the
// GGUF budget is genuinely unknown, so fall back to 0 (labels stay
// conservative) rather than the torch total: on a mixed host torch may still
// see a dGPU llama-server never enumerated, and reusing that total would let
// fit checks pass against VRAM /load can't actually place.
const isVulkanBuild = gpuData?.gguf_backend_is_vulkan === true;
if (!gpuData?.available || !devices.length) {
// Torch sees no GPU (training stays CPU-bound / unavailable), but a Vulkan
// llama.cpp build may still drive GPUs for GGUF: surface that budget alone.
@ -127,7 +137,11 @@ function toGpuInfo(data: SystemInfoResponse | null): GpuInfo {
available: true,
name: devices[0]?.name ?? "Unknown",
memoryTotalGb,
ggufMemoryTotalGb: ggufDevices.length ? ggufDeviceTotalGb : memoryTotalGb,
ggufMemoryTotalGb: ggufDevices.length
? ggufDeviceTotalGb
: isVulkanBuild
? 0
: memoryTotalGb,
};
}

View file

@ -46,6 +46,11 @@ export interface SystemInfoResponse {
* and on Vulkan-only builds only when the device probe found nothing --
* with gguf_devices present, picks are Vulkan ordinals and supported). */
gguf_gpu_ids_supported?: boolean;
/** True when the llama.cpp build is Vulkan. Lets the frontend tell an empty
* gguf_devices on a Vulkan build (probe failed/masked -- GGUF budget
* unknown) apart from a non-Vulkan build (llama-server runs on the torch
* devices, so the torch total is the right GGUF budget). */
gguf_backend_is_vulkan?: boolean;
backend_cuda_visible_devices?: string | null;
parent_visible_gpu_ids?: number[];
index_kind?: string;