studio: stamp GPU index kind into remount key; hold empty Vulkan GGUF budget
- gpuFieldsSignature now folds selectedGpuIdsIndexKind into the pick, so when the reactive fallback stamps a cold-hydrated active pick after the GPU cache warms, an open Run-settings panel remounts and re-snapshots with the new stamp instead of reloading/saving it as a legacy physical pick. - /api/system reports gguf_backend_is_vulkan so the frontend can tell an empty gguf_devices on a Vulkan build (probe failed/masked, budget unknown) from a non-Vulkan build. Empty Vulkan inventory now budgets GGUF at 0 instead of the torch VRAM total, which on a mixed host would let fit checks pass against VRAM /load cannot place.
This commit is contained in:
parent
715b20561f
commit
88cf376dc4
5 changed files with 49 additions and 8 deletions
|
|
@ -1238,16 +1238,17 @@ def _get_cached_system_gpu_info(logger) -> dict[str, Any]:
|
|||
# enumerated devices (gguf_devices above): the pick then lives in the
|
||||
# same ggml ordinal space `--device Vulkan<i>` pins. Without that
|
||||
# inventory the frontend has no valid ordinals to offer.
|
||||
is_vulkan_build = False
|
||||
try:
|
||||
from core.inference.llama_cpp import LlamaCppBackend
|
||||
from utils.hardware import DeviceType, get_device
|
||||
|
||||
is_vulkan_build = LlamaCppBackend._is_vulkan_backend()
|
||||
# Check the Vulkan build first: its picks live in ggml's own ordinal
|
||||
# space (--device Vulkan<i>) and don't rely on torch-xpu ordinals, so
|
||||
# they're valid even on an Intel/XPU host. Only fall through to the
|
||||
# XPU ban for a non-Vulkan build (where a pick would need torch-xpu
|
||||
# ordinals no visibility mask can speak).
|
||||
if LlamaCppBackend._is_vulkan_backend():
|
||||
if is_vulkan_build:
|
||||
gpu_ids_supported = bool(gguf_devices)
|
||||
elif get_device() == DeviceType.XPU:
|
||||
gpu_ids_supported = False
|
||||
|
|
@ -1258,12 +1259,17 @@ def _get_cached_system_gpu_info(logger) -> dict[str, Any]:
|
|||
gpu_ids_supported = True
|
||||
# `available` stays the torch view: training consumers key GPU labels on
|
||||
# it, and a Vulkan-only inventory doesn't make training GPU-capable.
|
||||
# GGUF surfaces key on gguf_devices instead.
|
||||
# GGUF surfaces key on gguf_devices instead. gguf_backend_is_vulkan lets
|
||||
# the frontend tell an empty gguf_devices on a Vulkan build (probe failed
|
||||
# / masked to nothing -- GGUF budget is unknown, must NOT reuse the torch
|
||||
# VRAM total) apart from a non-Vulkan build (where llama-server does run
|
||||
# on the torch devices, so that total is the right GGUF budget).
|
||||
gpu_info = {
|
||||
"available": visibility_info.get("available", False),
|
||||
"devices": enriched_devices,
|
||||
"gguf_devices": gguf_devices,
|
||||
"gguf_gpu_ids_supported": gpu_ids_supported,
|
||||
"gguf_backend_is_vulkan": is_vulkan_build,
|
||||
}
|
||||
_system_gpu_cache = (time.monotonic(), gpu_info)
|
||||
return gpu_info
|
||||
|
|
|
|||
|
|
@ -99,6 +99,7 @@ def test_vulkan_build_surfaces_llama_server_devices(main_module, monkeypatch):
|
|||
|
||||
# Picks are valid Vulkan ordinals now, so the picker may offer them.
|
||||
assert info["gguf_gpu_ids_supported"] is True
|
||||
assert info["gguf_backend_is_vulkan"] is True
|
||||
|
||||
|
||||
def test_vulkan_build_with_failed_probe_keeps_picks_unsupported(main_module, monkeypatch):
|
||||
|
|
@ -106,6 +107,10 @@ def test_vulkan_build_with_failed_probe_keeps_picks_unsupported(main_module, mon
|
|||
assert info["gguf_devices"] == []
|
||||
# No enumerable ordinal space -> the frontend has no valid picks to offer.
|
||||
assert info["gguf_gpu_ids_supported"] is False
|
||||
# ...but the frontend must still know this is a Vulkan build so it treats the
|
||||
# empty inventory as an unknown GGUF budget (0) rather than reusing the torch
|
||||
# VRAM total, which on a mixed host would overclaim VRAM /load can't place.
|
||||
assert info["gguf_backend_is_vulkan"] is True
|
||||
|
||||
|
||||
def test_vulkan_inventory_survives_gpu_cache_refreshes_without_reprobing(main_module, monkeypatch):
|
||||
|
|
@ -147,6 +152,9 @@ def test_non_vulkan_build_reports_no_gguf_inventory(main_module, monkeypatch):
|
|||
# CUDA/ROCm llama builds see the same devices torch does; no separate list.
|
||||
assert info["gguf_devices"] == []
|
||||
assert info["gguf_gpu_ids_supported"] is True
|
||||
# Not a Vulkan build: the frontend keeps budgeting GGUF against the torch
|
||||
# total, since llama-server runs on those same devices.
|
||||
assert info["gguf_backend_is_vulkan"] is False
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -141,9 +141,17 @@ export function gpuFieldsSignature(config: PerModelConfig): string {
|
|||
config.gpuMemoryMode ?? "auto",
|
||||
config.gpuLayers == null || config.gpuLayers < 0 ? -1 : config.gpuLayers,
|
||||
config.nCpuMoe ?? 0,
|
||||
// Include the pick's index space (physical vs Vulkan ordinals): the same ids
|
||||
// mean different cards across a backend swap, so a kind change is a real
|
||||
// config change. It also drives the SidebarModelConfig remount key, so when
|
||||
// the reactive fallback stamps a cold-hydrated active pick after the GPU
|
||||
// cache warms, an open Run-settings panel re-snapshots with the new stamp
|
||||
// instead of reloading/saving it as a legacy physical pick.
|
||||
config.selectedGpuIds == null
|
||||
? "all"
|
||||
: [...config.selectedGpuIds].sort((a, b) => a - b).join(","),
|
||||
: `${[...config.selectedGpuIds].sort((a, b) => a - b).join(",")}@${
|
||||
config.selectedGpuIdsIndexKind ?? "physical"
|
||||
}`,
|
||||
].join("|");
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -11,9 +11,12 @@ export interface GpuInfo {
|
|||
memoryTotalGb: number;
|
||||
/** VRAM budget for GGUF/llama-server workloads. On a Vulkan build this sums
|
||||
* the devices llama-server actually uses (gguf_devices), which can include
|
||||
* cards the torch backend can't see (e.g. a pre-ROCm AMD card); otherwise
|
||||
* identical to memoryTotalGb. GGUF fit labels must use this; torch-based
|
||||
* (training / safetensors) estimates must stay on memoryTotalGb. */
|
||||
* cards the torch backend can't see (e.g. a pre-ROCm AMD card), and is 0 when
|
||||
* that inventory is empty (probe failed/masked -- budget unknown, so labels
|
||||
* stay conservative). On a non-Vulkan build llama-server runs on the torch
|
||||
* devices, so this is identical to memoryTotalGb. GGUF fit labels must use
|
||||
* this; torch-based (training / safetensors) estimates must stay on
|
||||
* memoryTotalGb. */
|
||||
ggufMemoryTotalGb: number;
|
||||
cpuCore: number;
|
||||
cpuThread: number;
|
||||
|
|
@ -113,6 +116,13 @@ function toGpuInfo(data: SystemInfoResponse | null): GpuInfo {
|
|||
(sum, d) => sum + (d.is_igpu ? 0 : (d.memory_total_gb ?? 0)),
|
||||
0,
|
||||
);
|
||||
// A Vulkan build budgets GGUF against gguf_devices, not the torch view. When
|
||||
// that inventory is empty (probe failed / masked to no discrete device) the
|
||||
// GGUF budget is genuinely unknown, so fall back to 0 (labels stay
|
||||
// conservative) rather than the torch total: on a mixed host torch may still
|
||||
// see a dGPU llama-server never enumerated, and reusing that total would let
|
||||
// fit checks pass against VRAM /load can't actually place.
|
||||
const isVulkanBuild = gpuData?.gguf_backend_is_vulkan === true;
|
||||
if (!gpuData?.available || !devices.length) {
|
||||
// Torch sees no GPU (training stays CPU-bound / unavailable), but a Vulkan
|
||||
// llama.cpp build may still drive GPUs for GGUF: surface that budget alone.
|
||||
|
|
@ -127,7 +137,11 @@ function toGpuInfo(data: SystemInfoResponse | null): GpuInfo {
|
|||
available: true,
|
||||
name: devices[0]?.name ?? "Unknown",
|
||||
memoryTotalGb,
|
||||
ggufMemoryTotalGb: ggufDevices.length ? ggufDeviceTotalGb : memoryTotalGb,
|
||||
ggufMemoryTotalGb: ggufDevices.length
|
||||
? ggufDeviceTotalGb
|
||||
: isVulkanBuild
|
||||
? 0
|
||||
: memoryTotalGb,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -46,6 +46,11 @@ export interface SystemInfoResponse {
|
|||
* and on Vulkan-only builds only when the device probe found nothing --
|
||||
* with gguf_devices present, picks are Vulkan ordinals and supported). */
|
||||
gguf_gpu_ids_supported?: boolean;
|
||||
/** True when the llama.cpp build is Vulkan. Lets the frontend tell an empty
|
||||
* gguf_devices on a Vulkan build (probe failed/masked -- GGUF budget
|
||||
* unknown) apart from a non-Vulkan build (llama-server runs on the torch
|
||||
* devices, so the torch total is the right GGUF budget). */
|
||||
gguf_backend_is_vulkan?: boolean;
|
||||
backend_cuda_visible_devices?: string | null;
|
||||
parent_visible_gpu_ids?: number[];
|
||||
index_kind?: string;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue