Merge pull request #116 from unslothai/feature/gpu-monitor-training

feat: add live GPU monitor with nvidia-smi polling during training

studio/backend/routes/training.py

@@ -56,6 +56,22 @@ if not logger.handlers:
     logger.setLevel(logging.INFO)
 
 
+@router.get("/hardware")
+async def get_hardware_utilization(
+    current_subject: str = Depends(get_current_subject),
+):
+    """
+    Get a live snapshot of GPU hardware utilization.
+
+    Designed to be polled by the frontend during training.
+    Returns GPU utilization %, temperature, VRAM usage, and power draw
+    via nvidia-smi for maximum accuracy.
+    """
+    from utils.hardware import get_gpu_utilization
+
+    return get_gpu_utilization()
+
+
 @router.post("/start")
 async def start_training(
     request: TrainingStartRequest,

studio/backend/utils/hardware/__init__.py

@@ -12,6 +12,7 @@ from .hardware import (
     log_gpu_memory,
     get_gpu_summary,
     get_package_versions,
+    get_gpu_utilization,
 )
 
 __all__ = [
@@ -25,4 +26,5 @@ __all__ = [
     'log_gpu_memory',
     'get_gpu_summary',
     'get_package_versions',
+    'get_gpu_utilization',
 ]

studio/backend/utils/hardware/hardware.py

@@ -256,3 +256,131 @@ def get_package_versions() -> Dict[str, Optional[str]]:
         versions["cuda"] = None
 
     return versions
+
+
+# ========== Live GPU Utilization (nvidia-smi) ==========
+
+def get_gpu_utilization() -> Dict[str, Any]:
+    """
+    Return a live snapshot of GPU utilization via ``nvidia-smi``.
+
+    Designed to be polled by the frontend during training (not streaming).
+    Uses ``nvidia-smi --query-gpu`` which is the most accurate source for
+    utilization %, temperature, and power draw – stats that PyTorch does
+    not expose.
+
+    Returns dict with keys:
+        available          – bool, whether stats could be retrieved
+        gpu_utilization_pct – GPU core utilization %
+        temperature_c      – GPU temperature in °C
+        vram_used_gb       – VRAM currently used (GiB)
+        vram_total_gb      – VRAM total (GiB)
+        vram_utilization_pct – VRAM used / total * 100
+        power_draw_w       – current power draw (W)
+        power_limit_w      – power limit (W)
+        power_utilization_pct – power draw / limit * 100
+    """
+    device = get_device()
+
+    if device != DeviceType.CUDA:
+        return {"available": False, "backend": device.value}
+
+    def _parse_smi_value(raw: str):
+        """Parse a single nvidia-smi CSV value. Returns float or None for [N/A]."""
+        raw = raw.strip()
+        if not raw or raw == "[N/A]":
+            return None
+        try:
+            return float(raw)
+        except (ValueError, TypeError):
+            return None
+
+    # ── nvidia-smi (most complete source) ───────────────────────
+    smi_data = {}
+    try:
+        import subprocess
+
+        result = subprocess.run(
+            [
+                "nvidia-smi",
+                "--query-gpu=utilization.gpu,temperature.gpu,"
+                "memory.used,memory.total,power.draw,power.limit",
+                "--format=csv,noheader,nounits",
+            ],
+            capture_output=True,
+            text=True,
+            timeout=5,
+        )
+
+        if result.returncode == 0 and result.stdout.strip():
+            # nvidia-smi outputs one line per GPU; take GPU 0
+            first_line = result.stdout.strip().splitlines()[0]
+            parts = [p.strip() for p in first_line.split(",")]
+            if len(parts) >= 6:
+                smi_data = {
+                    "gpu_util": _parse_smi_value(parts[0]),
+                    "temp": _parse_smi_value(parts[1]),
+                    "vram_used_mb": _parse_smi_value(parts[2]),
+                    "vram_total_mb": _parse_smi_value(parts[3]),
+                    "power_draw": _parse_smi_value(parts[4]),
+                    "power_limit": _parse_smi_value(parts[5]),
+                }
+
+    except FileNotFoundError:
+        logger.debug("nvidia-smi not found, falling back to torch.cuda")
+    except Exception as e:
+        logger.warning(f"nvidia-smi query failed: {e}")
+
+    # ── Backfill VRAM from torch.cuda if nvidia-smi returned [N/A] ──
+    vram_used_mb = smi_data.get("vram_used_mb")
+    vram_total_mb = smi_data.get("vram_total_mb")
+
+    if vram_used_mb is None or vram_total_mb is None:
+        try:
+            import torch
+
+            idx = torch.cuda.current_device()
+            props = torch.cuda.get_device_properties(idx)
+            if vram_total_mb is None:
+                vram_total_mb = props.total_memory / (1024**2)  # bytes → MiB
+            if vram_used_mb is None:
+                vram_used_mb = torch.cuda.memory_allocated(idx) / (1024**2)
+        except Exception as e:
+            logger.debug(f"torch.cuda VRAM backfill failed: {e}")
+
+    # ── Build response ──────────────────────────────────────────
+    gpu_util = smi_data.get("gpu_util")
+    temp = smi_data.get("temp")
+    power_draw = smi_data.get("power_draw")
+    power_limit = smi_data.get("power_limit")
+
+    vram_used_gb = round(vram_used_mb / 1024, 2) if vram_used_mb is not None else None
+    vram_total_gb = round(vram_total_mb / 1024, 2) if vram_total_mb is not None else None
+    vram_pct = (
+        round((vram_used_mb / vram_total_mb) * 100, 1)
+        if vram_used_mb is not None and vram_total_mb and vram_total_mb > 0
+        else None
+    )
+    power_pct = (
+        round((power_draw / power_limit) * 100, 1)
+        if power_draw is not None and power_limit and power_limit > 0
+        else None
+    )
+
+    # If we got at least something useful, report available
+    has_any = any(v is not None for v in [gpu_util, temp, vram_used_gb, power_draw])
+    if not has_any:
+        return {"available": False, "backend": device.value}
+
+    return {
+        "available": True,
+        "backend": device.value,
+        "gpu_utilization_pct": gpu_util,
+        "temperature_c": temp,
+        "vram_used_gb": vram_used_gb,
+        "vram_total_gb": vram_total_gb,
+        "vram_utilization_pct": vram_pct,
+        "power_draw_w": power_draw,
+        "power_limit_w": power_limit,
+        "power_utilization_pct": power_pct,
+    }

studio/frontend/src/features/studio/sections/progress-section.tsx

@@ -32,6 +32,7 @@ import {
 import { HugeiconsIcon } from "@hugeicons/react";
 import { useEffect, useRef, useState, type ReactElement, type ReactNode } from "react";
 import { useShallow } from "zustand/react/shallow";
+import { useGpuUtilization } from "@/hooks";
 import { formatDuration, formatNumber, phaseColors, phaseLabel } from "./progress-section-lib";
 
 export function ProgressSection(): ReactElement {
@@ -72,6 +73,7 @@ export function ProgressSection(): ReactElement {
   );
 
   const { stopTrainingRun } = useTrainingActions();
+  const gpu = useGpuUtilization(runtime.isTrainingRunning);
   const [stopDialogOpen, setStopDialogOpen] = useState(false);
   const localStartAtRef = useRef<number | null>(null);
   const [, setLocalTick] = useState(0);
@@ -79,12 +81,12 @@ export function ProgressSection(): ReactElement {
   const pct =
     runtime.totalSteps > 0
       ? Math.min(
-          100,
-          Math.max(
-            0,
-            Math.round((runtime.currentStep / runtime.totalSteps) * 100),
-          ),
-        )
+        100,
+        Math.max(
+          0,
+          Math.round((runtime.currentStep / runtime.totalSteps) * 100),
+        ),
+      )
       : Math.round(runtime.progressPercent);
 
   useEffect(() => {
@@ -137,16 +139,16 @@ export function ProgressSection(): ReactElement {
     },
     ...(config.trainingMethod !== "full"
       ? [
-          {
-            section: "LoRA",
-            rows: [
-              ["Rank", config.loraRank],
-              ["Alpha", config.loraAlpha],
-              ["Dropout", config.loraDropout],
-              ["Variant", config.loraVariant],
-            ],
-          },
-        ]
+        {
+          section: "LoRA",
+          rows: [
+            ["Rank", config.loraRank],
+            ["Alpha", config.loraAlpha],
+            ["Dropout", config.loraDropout],
+            ["Variant", config.loraVariant],
+          ],
+        },
+      ]
       : []),
   ];
 
@@ -321,27 +323,27 @@ export function ProgressSection(): ReactElement {
                   className="size-3.5"
                 />
               }
-              value="--"
-              pct={0}
+              value={gpu.gpu_utilization_pct != null ? `${gpu.gpu_utilization_pct}%` : "--"}
+              pct={gpu.gpu_utilization_pct ?? 0}
             />
             <GpuStat
               label="Temperature"
               icon={<HugeiconsIcon icon={TemperatureIcon} className="size-3.5" />}
-              value="--"
-              pct={0}
+              value={gpu.temperature_c != null ? `${gpu.temperature_c}°C` : "--"}
+              pct={gpu.temperature_c ?? 0}
               max={100}
             />
             <GpuStat
               label="VRAM"
               icon={<HugeiconsIcon icon={RamMemoryIcon} className="size-3.5" />}
-              value="--"
-              pct={0}
+              value={gpu.vram_used_gb != null && gpu.vram_total_gb != null ? `${gpu.vram_used_gb} / ${gpu.vram_total_gb} GB` : "--"}
+              pct={gpu.vram_utilization_pct ?? 0}
             />
             <GpuStat
               label="Power"
               icon={<HugeiconsIcon icon={ZapIcon} className="size-3.5" />}
-              value="--"
-              pct={0}
+              value={gpu.power_draw_w != null ? (gpu.power_limit_w != null ? `${gpu.power_draw_w} / ${gpu.power_limit_w} W` : `${gpu.power_draw_w} W`) : "--"}
+              pct={gpu.power_utilization_pct ?? 0}
             />
           </div>
         </div>

studio/frontend/src/hooks/index.ts

@@ -1,5 +1,6 @@
 export { useDebouncedValue } from "./use-debounced-value";
 export { useGpuInfo } from "./use-gpu-info";
+export { useGpuUtilization } from "./use-gpu-utilization";
 export { useHfModelSearch } from "./use-hf-model-search";
 export { useHfDatasetSearch } from "./use-hf-dataset-search";
 export { useHfDatasetSplits } from "./use-hf-dataset-splits";

studio/frontend/src/hooks/use-gpu-utilization.ts

@@ -0,0 +1,74 @@
+import { authFetch } from "@/features/auth";
+import { useEffect, useRef, useState } from "react";
+
+export interface GpuUtilization {
+    available: boolean;
+    backend: string | null;
+    gpu_utilization_pct: number | null;
+    temperature_c: number | null;
+    vram_used_gb: number | null;
+    vram_total_gb: number | null;
+    vram_utilization_pct: number | null;
+    power_draw_w: number | null;
+    power_limit_w: number | null;
+    power_utilization_pct: number | null;
+}
+
+const DEFAULT: GpuUtilization = {
+    available: false,
+    backend: null,
+    gpu_utilization_pct: null,
+    temperature_c: null,
+    vram_used_gb: null,
+    vram_total_gb: null,
+    vram_utilization_pct: null,
+    power_draw_w: null,
+    power_limit_w: null,
+    power_utilization_pct: null,
+};
+
+/**
+ * Poll `GET /api/train/hardware` for live GPU utilization stats.
+ *
+ * Only polls while `enabled` is true (i.e. training is running).
+ * Polling interval defaults to 10 000 ms.
+ */
+export function useGpuUtilization(
+    enabled: boolean,
+    intervalMs = 10_000,
+): GpuUtilization {
+    const [data, setData] = useState<GpuUtilization>(DEFAULT);
+    const timerRef = useRef<ReturnType<typeof setInterval> | null>(null);
+
+    useEffect(() => {
+        if (!enabled) {
+            // Reset when training stops so the cards show "--" again
+            setData(DEFAULT);
+            return;
+        }
+
+        let cancelled = false;
+
+        async function poll() {
+            try {
+                const res = await authFetch("/api/train/hardware");
+                if (!res.ok || cancelled) return;
+                const json = (await res.json()) as GpuUtilization;
+                if (!cancelled) setData(json);
+            } catch {
+                // Silently ignore — next poll will retry
+            }
+        }
+
+        // Fetch immediately, then set up interval
+        void poll();
+        timerRef.current = setInterval(() => void poll(), intervalMs);
+
+        return () => {
+            cancelled = true;
+            if (timerRef.current) clearInterval(timerRef.current);
+        };
+    }, [enabled, intervalMs]);
+
+    return data;
+}