diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py index 13df2c1327..5bf59d85d2 100644 --- a/studio/backend/routes/training.py +++ b/studio/backend/routes/training.py @@ -56,6 +56,22 @@ if not logger.handlers: logger.setLevel(logging.INFO) +@router.get("/hardware") +async def get_hardware_utilization( + current_subject: str = Depends(get_current_subject), +): + """ + Get a live snapshot of GPU hardware utilization. + + Designed to be polled by the frontend during training. + Returns GPU utilization %, temperature, VRAM usage, and power draw + via nvidia-smi for maximum accuracy. + """ + from utils.hardware import get_gpu_utilization + + return get_gpu_utilization() + + @router.post("/start") async def start_training( request: TrainingStartRequest, diff --git a/studio/backend/utils/hardware/__init__.py b/studio/backend/utils/hardware/__init__.py index df6df6d34c..667466efea 100644 --- a/studio/backend/utils/hardware/__init__.py +++ b/studio/backend/utils/hardware/__init__.py @@ -12,6 +12,7 @@ from .hardware import ( log_gpu_memory, get_gpu_summary, get_package_versions, + get_gpu_utilization, ) __all__ = [ @@ -25,4 +26,5 @@ __all__ = [ 'log_gpu_memory', 'get_gpu_summary', 'get_package_versions', + 'get_gpu_utilization', ] diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py index 2effae4814..754ef6fdae 100644 --- a/studio/backend/utils/hardware/hardware.py +++ b/studio/backend/utils/hardware/hardware.py @@ -256,3 +256,131 @@ def get_package_versions() -> Dict[str, Optional[str]]: versions["cuda"] = None return versions + + +# ========== Live GPU Utilization (nvidia-smi) ========== + +def get_gpu_utilization() -> Dict[str, Any]: + """ + Return a live snapshot of GPU utilization via ``nvidia-smi``. + + Designed to be polled by the frontend during training (not streaming). + Uses ``nvidia-smi --query-gpu`` which is the most accurate source for + utilization %, temperature, and power draw – stats that PyTorch does + not expose. + + Returns dict with keys: + available – bool, whether stats could be retrieved + gpu_utilization_pct – GPU core utilization % + temperature_c – GPU temperature in °C + vram_used_gb – VRAM currently used (GiB) + vram_total_gb – VRAM total (GiB) + vram_utilization_pct – VRAM used / total * 100 + power_draw_w – current power draw (W) + power_limit_w – power limit (W) + power_utilization_pct – power draw / limit * 100 + """ + device = get_device() + + if device != DeviceType.CUDA: + return {"available": False, "backend": device.value} + + def _parse_smi_value(raw: str): + """Parse a single nvidia-smi CSV value. Returns float or None for [N/A].""" + raw = raw.strip() + if not raw or raw == "[N/A]": + return None + try: + return float(raw) + except (ValueError, TypeError): + return None + + # ── nvidia-smi (most complete source) ─────────────────────── + smi_data = {} + try: + import subprocess + + result = subprocess.run( + [ + "nvidia-smi", + "--query-gpu=utilization.gpu,temperature.gpu," + "memory.used,memory.total,power.draw,power.limit", + "--format=csv,noheader,nounits", + ], + capture_output=True, + text=True, + timeout=5, + ) + + if result.returncode == 0 and result.stdout.strip(): + # nvidia-smi outputs one line per GPU; take GPU 0 + first_line = result.stdout.strip().splitlines()[0] + parts = [p.strip() for p in first_line.split(",")] + if len(parts) >= 6: + smi_data = { + "gpu_util": _parse_smi_value(parts[0]), + "temp": _parse_smi_value(parts[1]), + "vram_used_mb": _parse_smi_value(parts[2]), + "vram_total_mb": _parse_smi_value(parts[3]), + "power_draw": _parse_smi_value(parts[4]), + "power_limit": _parse_smi_value(parts[5]), + } + + except FileNotFoundError: + logger.debug("nvidia-smi not found, falling back to torch.cuda") + except Exception as e: + logger.warning(f"nvidia-smi query failed: {e}") + + # ── Backfill VRAM from torch.cuda if nvidia-smi returned [N/A] ── + vram_used_mb = smi_data.get("vram_used_mb") + vram_total_mb = smi_data.get("vram_total_mb") + + if vram_used_mb is None or vram_total_mb is None: + try: + import torch + + idx = torch.cuda.current_device() + props = torch.cuda.get_device_properties(idx) + if vram_total_mb is None: + vram_total_mb = props.total_memory / (1024**2) # bytes → MiB + if vram_used_mb is None: + vram_used_mb = torch.cuda.memory_allocated(idx) / (1024**2) + except Exception as e: + logger.debug(f"torch.cuda VRAM backfill failed: {e}") + + # ── Build response ────────────────────────────────────────── + gpu_util = smi_data.get("gpu_util") + temp = smi_data.get("temp") + power_draw = smi_data.get("power_draw") + power_limit = smi_data.get("power_limit") + + vram_used_gb = round(vram_used_mb / 1024, 2) if vram_used_mb is not None else None + vram_total_gb = round(vram_total_mb / 1024, 2) if vram_total_mb is not None else None + vram_pct = ( + round((vram_used_mb / vram_total_mb) * 100, 1) + if vram_used_mb is not None and vram_total_mb and vram_total_mb > 0 + else None + ) + power_pct = ( + round((power_draw / power_limit) * 100, 1) + if power_draw is not None and power_limit and power_limit > 0 + else None + ) + + # If we got at least something useful, report available + has_any = any(v is not None for v in [gpu_util, temp, vram_used_gb, power_draw]) + if not has_any: + return {"available": False, "backend": device.value} + + return { + "available": True, + "backend": device.value, + "gpu_utilization_pct": gpu_util, + "temperature_c": temp, + "vram_used_gb": vram_used_gb, + "vram_total_gb": vram_total_gb, + "vram_utilization_pct": vram_pct, + "power_draw_w": power_draw, + "power_limit_w": power_limit, + "power_utilization_pct": power_pct, + } diff --git a/studio/frontend/src/features/studio/sections/progress-section.tsx b/studio/frontend/src/features/studio/sections/progress-section.tsx index 23ae7adb87..140b9ead66 100644 --- a/studio/frontend/src/features/studio/sections/progress-section.tsx +++ b/studio/frontend/src/features/studio/sections/progress-section.tsx @@ -32,6 +32,7 @@ import { import { HugeiconsIcon } from "@hugeicons/react"; import { useEffect, useRef, useState, type ReactElement, type ReactNode } from "react"; import { useShallow } from "zustand/react/shallow"; +import { useGpuUtilization } from "@/hooks"; import { formatDuration, formatNumber, phaseColors, phaseLabel } from "./progress-section-lib"; export function ProgressSection(): ReactElement { @@ -72,6 +73,7 @@ export function ProgressSection(): ReactElement { ); const { stopTrainingRun } = useTrainingActions(); + const gpu = useGpuUtilization(runtime.isTrainingRunning); const [stopDialogOpen, setStopDialogOpen] = useState(false); const localStartAtRef = useRef(null); const [, setLocalTick] = useState(0); @@ -79,12 +81,12 @@ export function ProgressSection(): ReactElement { const pct = runtime.totalSteps > 0 ? Math.min( - 100, - Math.max( - 0, - Math.round((runtime.currentStep / runtime.totalSteps) * 100), - ), - ) + 100, + Math.max( + 0, + Math.round((runtime.currentStep / runtime.totalSteps) * 100), + ), + ) : Math.round(runtime.progressPercent); useEffect(() => { @@ -137,16 +139,16 @@ export function ProgressSection(): ReactElement { }, ...(config.trainingMethod !== "full" ? [ - { - section: "LoRA", - rows: [ - ["Rank", config.loraRank], - ["Alpha", config.loraAlpha], - ["Dropout", config.loraDropout], - ["Variant", config.loraVariant], - ], - }, - ] + { + section: "LoRA", + rows: [ + ["Rank", config.loraRank], + ["Alpha", config.loraAlpha], + ["Dropout", config.loraDropout], + ["Variant", config.loraVariant], + ], + }, + ] : []), ]; @@ -321,27 +323,27 @@ export function ProgressSection(): ReactElement { className="size-3.5" /> } - value="--" - pct={0} + value={gpu.gpu_utilization_pct != null ? `${gpu.gpu_utilization_pct}%` : "--"} + pct={gpu.gpu_utilization_pct ?? 0} /> } - value="--" - pct={0} + value={gpu.temperature_c != null ? `${gpu.temperature_c}°C` : "--"} + pct={gpu.temperature_c ?? 0} max={100} /> } - value="--" - pct={0} + value={gpu.vram_used_gb != null && gpu.vram_total_gb != null ? `${gpu.vram_used_gb} / ${gpu.vram_total_gb} GB` : "--"} + pct={gpu.vram_utilization_pct ?? 0} /> } - value="--" - pct={0} + value={gpu.power_draw_w != null ? (gpu.power_limit_w != null ? `${gpu.power_draw_w} / ${gpu.power_limit_w} W` : `${gpu.power_draw_w} W`) : "--"} + pct={gpu.power_utilization_pct ?? 0} /> diff --git a/studio/frontend/src/hooks/index.ts b/studio/frontend/src/hooks/index.ts index 1ada378651..da0d595d9e 100644 --- a/studio/frontend/src/hooks/index.ts +++ b/studio/frontend/src/hooks/index.ts @@ -1,5 +1,6 @@ export { useDebouncedValue } from "./use-debounced-value"; export { useGpuInfo } from "./use-gpu-info"; +export { useGpuUtilization } from "./use-gpu-utilization"; export { useHfModelSearch } from "./use-hf-model-search"; export { useHfDatasetSearch } from "./use-hf-dataset-search"; export { useHfDatasetSplits } from "./use-hf-dataset-splits"; diff --git a/studio/frontend/src/hooks/use-gpu-utilization.ts b/studio/frontend/src/hooks/use-gpu-utilization.ts new file mode 100644 index 0000000000..785104489b --- /dev/null +++ b/studio/frontend/src/hooks/use-gpu-utilization.ts @@ -0,0 +1,74 @@ +import { authFetch } from "@/features/auth"; +import { useEffect, useRef, useState } from "react"; + +export interface GpuUtilization { + available: boolean; + backend: string | null; + gpu_utilization_pct: number | null; + temperature_c: number | null; + vram_used_gb: number | null; + vram_total_gb: number | null; + vram_utilization_pct: number | null; + power_draw_w: number | null; + power_limit_w: number | null; + power_utilization_pct: number | null; +} + +const DEFAULT: GpuUtilization = { + available: false, + backend: null, + gpu_utilization_pct: null, + temperature_c: null, + vram_used_gb: null, + vram_total_gb: null, + vram_utilization_pct: null, + power_draw_w: null, + power_limit_w: null, + power_utilization_pct: null, +}; + +/** + * Poll `GET /api/train/hardware` for live GPU utilization stats. + * + * Only polls while `enabled` is true (i.e. training is running). + * Polling interval defaults to 10 000 ms. + */ +export function useGpuUtilization( + enabled: boolean, + intervalMs = 10_000, +): GpuUtilization { + const [data, setData] = useState(DEFAULT); + const timerRef = useRef | null>(null); + + useEffect(() => { + if (!enabled) { + // Reset when training stops so the cards show "--" again + setData(DEFAULT); + return; + } + + let cancelled = false; + + async function poll() { + try { + const res = await authFetch("/api/train/hardware"); + if (!res.ok || cancelled) return; + const json = (await res.json()) as GpuUtilization; + if (!cancelled) setData(json); + } catch { + // Silently ignore — next poll will retry + } + } + + // Fetch immediately, then set up interval + void poll(); + timerRef.current = setInterval(() => void poll(), intervalMs); + + return () => { + cancelled = true; + if (timerRef.current) clearInterval(timerRef.current); + }; + }, [enabled, intervalMs]); + + return data; +}