ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Vulkan GPU picks: off-loop probe, XPU-order, and reliable active/status kind

Browse source 
Four Codex follow-ups on the Vulkan GPU-selection path:

- /load + /validate: run validate_vulkan_gpu_ids off the event loop
  (asyncio.to_thread). It can spawn a blocking Vulkan probe subprocess, which
  would otherwise freeze status/progress/unload for up to the probe timeout.

- /api/system, /load, /validate: check the Vulkan build BEFORE the XPU ban. A
  Vulkan pick uses ggml ordinals (--device Vulkan<i>), not torch-xpu ordinals,
  so an Intel/XPU host on a Vulkan build must still get the picker and accept
  gpu_ids instead of being rejected.

- Active/status GPU-kind stamp: an active GGUF hydrated from /status before the
  GPU cache warms had no index-space stamp, so a later Reload/Remember treated
  its live Vulkan pick as a legacy physical one and reconciled it away. Carry
  selectedGpuIdsKind through useActiveModelConfig (via the reactive
  useCurrentGpuIndexKind) and fall back to the current kind in
  currentRuntimePerModelConfig when the store stamp is missing -- an active pick
  is by definition in the current backend's space.
This commit is contained in:
LeoBorcherding 2026-07-23 23:33:39 -05:00
commit a1c1aa7fed
5 changed files with 86 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

11
studio/backend/main.py
View file

@ -1241,10 +1241,15 @@ def _get_cached_system_gpu_info(logger) -> dict[str, Any]:
try:
from core.inference.llama_cpp import LlamaCppBackend
from utils.hardware import DeviceType, get_device
if get_device() == DeviceType.XPU:
gpu_ids_supported = False
elif LlamaCppBackend._is_vulkan_backend():
# Check the Vulkan build first: its picks live in ggml's own ordinal
# space (--device Vulkan<i>) and don't rely on torch-xpu ordinals, so
# they're valid even on an Intel/XPU host. Only fall through to the
# XPU ban for a non-Vulkan build (where a pick would need torch-xpu
# ordinals no visibility mask can speak).
if LlamaCppBackend._is_vulkan_backend():
gpu_ids_supported = bool(gguf_devices)
elif get_device() == DeviceType.XPU:
gpu_ids_supported = False
else:
gpu_ids_supported = True
except Exception as e:

60
studio/backend/routes/inference.py
View file

@ -4443,18 +4443,12 @@ async def _load_model_impl(
from utils.hardware import DeviceType, get_device
from utils.hardware.hardware import resolve_requested_gpu_ids
if get_device() == DeviceType.XPU:
raise HTTPException(
status_code = 400,
detail = (
"GPU selection (gpu_ids) is not supported on Intel XPU. "
"Omit gpu_ids to use all devices."
),
)
# A Vulkan-only build validates against ggml's own Vulkan ordinals
# instead: /api/system reports gguf_devices in that space, load_model
# pins the pick with --device Vulkan<i>, so probe, picker, and pin
# share one index space (physical ids are never involved).
# A Vulkan build validates against ggml's own Vulkan ordinals:
# /api/system reports gguf_devices in that space, load_model pins the
# pick with --device Vulkan<i>, so probe, picker, and pin share one
# index space (physical/torch ids are never involved). Check it
# BEFORE the XPU ban -- a Vulkan pick on an Intel/XPU host does not
# rely on torch-xpu ordinals, so the ban must not reject it.
if LlamaCppBackend._is_vulkan_backend():
# Diffusion GGUFs bypass llama-server: the diffusion runner
# forwards gpu_ids[0] as a CUDA/DG_GPU device token, NOT
@ -4474,10 +4468,24 @@ async def _load_model_impl(
"runner cannot map ggml Vulkan ordinals. Omit gpu_ids."
),
)
# validate_vulkan_gpu_ids may spawn the Vulkan device probe
# (blocking subprocess.run). Run it off the event loop so a
# stalled driver/probe can't freeze status/progress/unload for
# up to the probe timeout.
try:
LlamaCppBackend.validate_vulkan_gpu_ids(effective_gpu_ids)
await asyncio.to_thread(
LlamaCppBackend.validate_vulkan_gpu_ids, effective_gpu_ids
)
except ValueError as exc:
raise HTTPException(status_code = 400, detail = str(exc)) from exc
elif get_device() == DeviceType.XPU:
raise HTTPException(
status_code = 400,
detail = (
"GPU selection (gpu_ids) is not supported on Intel XPU. "
"Omit gpu_ids to use all devices."
),
)
else:
try:
resolve_requested_gpu_ids(effective_gpu_ids)
@ -5082,21 +5090,15 @@ async def validate_model(
from utils.hardware import DeviceType, get_device
from utils.hardware.hardware import resolve_requested_gpu_ids
if get_device() == DeviceType.XPU:
raise HTTPException(
status_code = 400,
detail = (
"GPU selection (gpu_ids) is not supported on Intel XPU. "
"Omit gpu_ids to use all devices."
),
)
# Mirror /load: a Vulkan build validates the pick in ggml's own
# Vulkan ordinal space (the space the --device pin uses), and rejects
# picks for CONFIRMED diffusion GGUFs (their runner takes a CUDA/DG_GPU
# token, not --device Vulkan<i>, so an ordinal targets the wrong card).
# `None` (uncached, unclassifiable) is allowed through so first-time
# remote GGUF loads still work; the spawn-time backstop catches an
# uncached model that turns out to be diffusion after download.
# uncached model that turns out to be diffusion after download. Check
# the Vulkan path BEFORE the XPU ban: a Vulkan pick on an XPU host
# uses ggml ordinals, not torch-xpu ones, so the ban must not hide it.
if LlamaCppBackend._is_vulkan_backend():
if _classify_diffusion_gguf(config) is True:
raise HTTPException(
@ -5107,10 +5109,22 @@ async def validate_model(
"runner cannot map ggml Vulkan ordinals. Omit gpu_ids."
),
)
# Off-loop: validate_vulkan_gpu_ids may spawn the blocking Vulkan
# probe subprocess (see /load).
try:
LlamaCppBackend.validate_vulkan_gpu_ids(effective_gpu_ids)
await asyncio.to_thread(
LlamaCppBackend.validate_vulkan_gpu_ids, effective_gpu_ids
)
except ValueError as exc:
raise HTTPException(status_code = 400, detail = str(exc)) from exc
elif get_device() == DeviceType.XPU:
raise HTTPException(
status_code = 400,
detail = (
"GPU selection (gpu_ids) is not supported on Intel XPU. "
"Omit gpu_ids to use all devices."
),
)
else:
try:
resolve_requested_gpu_ids(effective_gpu_ids)

14
studio/frontend/src/features/model-picker/hooks/use-active-model-config.ts
View file

@ -2,6 +2,7 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { isExternalModelId, useChatRuntimeStore } from "@/features/chat";
import { useCurrentGpuIndexKind } from "@/hooks/use-gpu-info";
import { useMemo } from "react";
import type { PerModelConfig } from "../model-config/per-model-config";
@ -28,6 +29,13 @@ export function useActiveModelConfig(): ActiveModelConfigState {
const gpuLayers = useChatRuntimeStore((s) => s.gpuLayers);
const nCpuMoe = useChatRuntimeStore((s) => s.nCpuMoe);
const selectedGpuIds = useChatRuntimeStore((s) => s.selectedGpuIds);
const selectedGpuIdsKind = useChatRuntimeStore((s) => s.selectedGpuIdsKind);
// The active model is running under the current backend, so its pick is in
// the current index space. Prefer the store's stamp, but fall back to the
// current kind (reactive, so it fills in once the cache warms) -- else a
// /status hydration before the cache warmed leaves no stamp, and a later
// Reload/Remember would treat the live Vulkan pick as a legacy physical one.
const currentKind = useCurrentGpuIndexKind();
const isGguf =
activeGgufVariant != null ||
@ -56,6 +64,10 @@ export function useActiveModelConfig(): ActiveModelConfigState {
gpuLayers,
nCpuMoe,
selectedGpuIds,
selectedGpuIdsIndexKind:
selectedGpuIds == null
? undefined
: ((selectedGpuIdsKind ?? currentKind) ?? undefined),
};
}, [
checkpoint,
@ -71,6 +83,8 @@ export function useActiveModelConfig(): ActiveModelConfigState {
gpuLayers,
nCpuMoe,
selectedGpuIds,
selectedGpuIdsKind,
currentKind,
]);
return { checkpoint, isGguf, config };

10
studio/frontend/src/features/model-picker/model-config/apply-per-model-config.ts
View file

@ -10,7 +10,7 @@ import {
reconcilePersistedGpuIds,
useChatRuntimeStore,
} from "@/features/chat";
import type { GpuIndexKind } from "@/hooks/use-gpu-info";
import { type GpuIndexKind, currentGpuIndexKind } from "@/hooks/use-gpu-info";
import {
DEFAULT_PER_MODEL_CONFIG,
type PerModelConfig,
@ -103,8 +103,14 @@ export function currentRuntimePerModelConfig(
selectedGpuIds: s.selectedGpuIds,
// Carry the index space the live pick is in so a save/restore round-trip
// (and a cancel-restore of this snapshot) can drop it after a backend swap.
// Fall back to the current kind when the store stamp is missing (a /status
// hydration before the GPU cache warmed leaves it null): a live/active pick
// is by definition in the current backend's space, so this is the right
// stamp and avoids persisting it as an unstamped (legacy physical) pick.
selectedGpuIdsIndexKind:
s.selectedGpuIds == null ? undefined : (s.selectedGpuIdsKind ?? undefined),
s.selectedGpuIds == null
? undefined
: ((s.selectedGpuIdsKind ?? currentGpuIndexKind()) ?? undefined),
};
}

19
studio/frontend/src/hooks/use-gpu-info.ts
View file

@ -206,6 +206,25 @@ export function useGpuDevices(): SystemGpuDevice[] {
return devices;
}
/** Reactive currentGpuIndexKind(): re-renders when /api/system loads, so a
* component reading a pick's index space (e.g. the active model's) updates
* once the cache warms instead of being stuck at the cold-cache null. */
export function useCurrentGpuIndexKind(): GpuIndexKind | null {
const [kind, setKind] = useState<GpuIndexKind | null>(() =>
systemGpuIndexKind(cachedSystem),
);
useEffect(() => {
let cancelled = false;
fetchSystemOnce().then((d) => {
if (!cancelled) setKind(systemGpuIndexKind(d));
});
return () => {
cancelled = true;
};
}, []);
return kind;
}
/**
* Await the shared /api/system fetch so cachedPinnableGpuIndices (and the
* store's reconcilePersistedGpuIds) can validate a persisted pick before a